2024-08-17T16:39:21.795 INFO:root:teuthology version: 1.2.0 2024-08-17T16:39:21.822 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T16:39:21.949 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322 branch: main description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7858322' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - PG_AVAILABILITY sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 17 sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 11/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 targets: smithi142.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPPHcB8eX7HmkZ9PjWzokE1WWTvliDuiucSaArPHpX49fNhYde8Z1LQPd11XQD/rusjQiRE1kaL0ZPVs0taLqSk= smithi152.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGv8Hyg/zInwHLfxufbl5w0wNrvBeOjMvv+CSwoyhDbWHyXjty3nEf1qZb9z7HLA8gwhdaMQ/7n/wd73566IRoY= tasks: - pexec: all: - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 67db276914d29505c9707d04f3a3b8fb9d8aee4b timestamp: 2024-08-17_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-17T16:39:21.950 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa; will attempt to use it 2024-08-17T16:39:21.951 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks 2024-08-17T16:39:21.951 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-17T16:39:21.953 INFO:teuthology.task.internal:Checking packages... 2024-08-17T16:39:21.975 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453' 2024-08-17T16:39:21.976 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:39:21.976 INFO:teuthology.packaging:ref: None 2024-08-17T16:39:21.976 INFO:teuthology.packaging:tag: None 2024-08-17T16:39:21.976 INFO:teuthology.packaging:branch: main 2024-08-17T16:39:21.976 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:39:21.976 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-08-17T16:39:22.199 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-5842.g4887e20f 2024-08-17T16:39:22.201 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-17T16:39:22.207 INFO:teuthology.task.internal:no buildpackages task found 2024-08-17T16:39:22.207 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-17T16:39:22.214 INFO:teuthology.task.internal:Saving configuration 2024-08-17T16:39:22.239 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-17T16:39:22.246 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-17T16:39:22.279 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi142.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:34:03.054292', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPPHcB8eX7HmkZ9PjWzokE1WWTvliDuiucSaArPHpX49fNhYde8Z1LQPd11XQD/rusjQiRE1kaL0ZPVs0taLqSk='} 2024-08-17T16:39:22.308 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:34:03.056803', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGv8Hyg/zInwHLfxufbl5w0wNrvBeOjMvv+CSwoyhDbWHyXjty3nEf1qZb9z7HLA8gwhdaMQ/7n/wd73566IRoY='} 2024-08-17T16:39:22.308 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-17T16:39:22.314 INFO:teuthology.task.internal:roles: ubuntu@smithi142.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-08-17T16:39:22.314 INFO:teuthology.task.internal:roles: ubuntu@smithi152.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-08-17T16:39:22.314 INFO:teuthology.run_tasks:Running task console_log... 2024-08-17T16:39:22.387 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8f3df07ac0>, signals=[15]) 2024-08-17T16:39:22.387 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-17T16:39:22.393 INFO:teuthology.task.internal:Opening connections... 2024-08-17T16:39:22.393 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi142.front.sepia.ceph.com 2024-08-17T16:39:22.394 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi142.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:39:22.479 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi152.front.sepia.ceph.com 2024-08-17T16:39:22.480 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:39:22.564 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-17T16:39:22.571 DEBUG:teuthology.orchestra.run.smithi142:> uname -m 2024-08-17T16:39:22.590 INFO:teuthology.orchestra.run.smithi142.stdout:x86_64 2024-08-17T16:39:22.590 DEBUG:teuthology.orchestra.run.smithi142:> cat /etc/os-release 2024-08-17T16:39:22.647 INFO:teuthology.orchestra.run.smithi142.stdout:NAME="CentOS Stream" 2024-08-17T16:39:22.647 INFO:teuthology.orchestra.run.smithi142.stdout:VERSION="9" 2024-08-17T16:39:22.647 INFO:teuthology.orchestra.run.smithi142.stdout:ID="centos" 2024-08-17T16:39:22.647 INFO:teuthology.orchestra.run.smithi142.stdout:ID_LIKE="rhel fedora" 2024-08-17T16:39:22.647 INFO:teuthology.orchestra.run.smithi142.stdout:VERSION_ID="9" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:PLATFORM_ID="platform:el9" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:ANSI_COLOR="0;31" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:LOGO="fedora-logo-icon" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:HOME_URL="https://centos.org/" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T16:39:22.648 INFO:teuthology.orchestra.run.smithi142.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T16:39:22.649 INFO:teuthology.lock.ops:Updating smithi142.front.sepia.ceph.com on lock server 2024-08-17T16:39:22.679 DEBUG:teuthology.orchestra.run.smithi152:> uname -m 2024-08-17T16:39:22.696 INFO:teuthology.orchestra.run.smithi152.stdout:x86_64 2024-08-17T16:39:22.696 DEBUG:teuthology.orchestra.run.smithi152:> cat /etc/os-release 2024-08-17T16:39:22.753 INFO:teuthology.orchestra.run.smithi152.stdout:NAME="CentOS Stream" 2024-08-17T16:39:22.753 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION="9" 2024-08-17T16:39:22.753 INFO:teuthology.orchestra.run.smithi152.stdout:ID="centos" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:ID_LIKE="rhel fedora" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION_ID="9" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:PLATFORM_ID="platform:el9" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:ANSI_COLOR="0;31" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:LOGO="fedora-logo-icon" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:HOME_URL="https://centos.org/" 2024-08-17T16:39:22.754 INFO:teuthology.orchestra.run.smithi152.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T16:39:22.755 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T16:39:22.755 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T16:39:22.755 INFO:teuthology.lock.ops:Updating smithi152.front.sepia.ceph.com on lock server 2024-08-17T16:39:22.787 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-17T16:39:22.796 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-17T16:39:22.802 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-17T16:39:22.802 DEBUG:teuthology.orchestra.run.smithi142:> test '!' -e /home/ubuntu/cephtest 2024-08-17T16:39:22.804 DEBUG:teuthology.orchestra.run.smithi152:> test '!' -e /home/ubuntu/cephtest 2024-08-17T16:39:22.819 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-17T16:39:22.825 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-17T16:39:22.826 DEBUG:teuthology.orchestra.run.smithi142:> test -z $(ls -A /var/lib/ceph) 2024-08-17T16:39:22.862 DEBUG:teuthology.orchestra.run.smithi152:> test -z $(ls -A /var/lib/ceph) 2024-08-17T16:39:22.976 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-17T16:39:23.044 INFO:teuthology.run_tasks:Running task kernel... 2024-08-17T16:39:23.059 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-17T16:39:23.060 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-17T16:39:23.060 DEBUG:teuthology.orchestra.run.smithi142:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T16:39:23.060 DEBUG:teuthology.orchestra.run.smithi152:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T16:39:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:39:23.077 DEBUG:teuthology.orchestra.run.smithi142:> uname -r 2024-08-17T16:39:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:39:23.078 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-08-17T16:39:23.135 INFO:teuthology.orchestra.run.smithi142.stdout:5.14.0-437.el9.x86_64 2024-08-17T16:39:23.135 INFO:teuthology.task.kernel:Running kernel on smithi142: 5.14.0-437.el9.x86_64 2024-08-17T16:39:23.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum install -y kernel 2024-08-17T16:39:23.136 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-437.el9.x86_64 2024-08-17T16:39:23.136 INFO:teuthology.task.kernel:Running kernel on smithi152: 5.14.0-437.el9.x86_64 2024-08-17T16:39:23.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum install -y kernel 2024-08-17T16:39:26.061 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - BaseOS 6.5 MB/s | 8.2 MB 00:01 2024-08-17T16:39:27.503 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 3.0 MB/s | 8.2 MB 00:02 2024-08-17T16:39:31.445 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - AppStream 4.7 MB/s | 20 MB 00:04 2024-08-17T16:39:31.954 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 6.0 MB/s | 20 MB 00:03 2024-08-17T16:39:35.729 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.6 MB 00:00 2024-08-17T16:39:37.021 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 5.3 MB/s | 6.6 MB 00:01 2024-08-17T16:39:37.254 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - Extras packages 59 kB/s | 18 kB 00:00 2024-08-17T16:39:38.300 INFO:teuthology.orchestra.run.smithi142.stdout:Extra Packages for Enterprise Linux 24 MB/s | 22 MB 00:00 2024-08-17T16:39:38.797 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 18 kB 00:00 2024-08-17T16:39:40.151 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 18 MB/s | 22 MB 00:01 2024-08-17T16:39:43.672 INFO:teuthology.orchestra.run.smithi142.stdout:lab-extras 47 kB/s | 1.7 kB 00:00 2024-08-17T16:39:45.490 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-08-17T16:39:45.637 INFO:teuthology.orchestra.run.smithi142.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T16:39:45.637 INFO:teuthology.orchestra.run.smithi142.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T16:39:45.745 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout: Package Architecture Version Repository Size 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T16:39:45.757 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:39:45.758 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:39:45.758 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:39:45.758 INFO:teuthology.orchestra.run.smithi142.stdout:Install 4 Packages 2024-08-17T16:39:45.758 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:39:45.758 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 84 M 2024-08-17T16:39:45.759 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 126 M 2024-08-17T16:39:45.759 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:39:46.365 INFO:teuthology.orchestra.run.smithi142.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 4.5 MB/s | 1.5 MB 00:00 2024-08-17T16:39:47.488 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T16:39:47.488 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T16:39:47.582 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:39:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:39:47.587 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-08-17T16:39:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:Install 4 Packages 2024-08-17T16:39:47.588 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:39:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 84 M 2024-08-17T16:39:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 126 M 2024-08-17T16:39:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:39:47.774 INFO:teuthology.orchestra.run.smithi142.stdout:(2/4): kernel-modules-core-5.14.0-480.el9.x86_6 21 MB/s | 30 MB 00:01 2024-08-17T16:39:48.208 INFO:teuthology.orchestra.run.smithi152.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 4.4 MB/s | 1.5 MB 00:00 2024-08-17T16:39:49.983 INFO:teuthology.orchestra.run.smithi152.stdout:(2/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 17 MB/s | 36 MB 00:02 2024-08-17T16:39:50.200 INFO:teuthology.orchestra.run.smithi152.stdout:(3/4): kernel-core-5.14.0-480.el9.x86_64.rpm 7.3 MB/s | 17 MB 00:02 2024-08-17T16:39:50.642 INFO:teuthology.orchestra.run.smithi152.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 12 MB/s | 30 MB 00:02 2024-08-17T16:39:50.642 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:39:50.642 INFO:teuthology.orchestra.run.smithi152.stdout:Total 28 MB/s | 84 MB 00:03 2024-08-17T16:39:51.376 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:39:51.704 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:39:51.704 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:39:52.697 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:39:52.698 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:39:53.790 INFO:teuthology.orchestra.run.smithi142.stdout:(3/4): kernel-core-5.14.0-480.el9.x86_64.rpm 2.2 MB/s | 17 MB 00:07 2024-08-17T16:39:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:39:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:39:55.936 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:39:58.588 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:39:58.731 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:40:02.864 INFO:teuthology.orchestra.run.smithi142.stdout:(4/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 2.1 MB/s | 36 MB 00:16 2024-08-17T16:40:02.864 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:40:02.864 INFO:teuthology.orchestra.run.smithi142.stdout:Total 4.9 MB/s | 84 MB 00:17 2024-08-17T16:40:03.586 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:40:03.603 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:40:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:03.947 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:40:03.947 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:40:04.898 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:40:04.899 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:40:07.323 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:40:07.986 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:40:08.082 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:40:08.594 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:10.791 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:40:10.955 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:40:15.790 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:40:15.864 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:20.720 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:38.595 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:38.644 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:39.039 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:39.039 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:40:39.039 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:40:39.039 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:40:39.838 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:39.838 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:40:39.838 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:40:39.839 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T16:40:39.839 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T16:40:39.839 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T16:40:39.839 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T16:40:39.839 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:40:39.839 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:40:40.226 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:40:40.307 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:40:40.308 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:40:40.308 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T16:40:40.308 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T16:40:40.308 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T16:40:40.308 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-08-17T16:40:40.308 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi152.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T16:40:40.308 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:40:40.365 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:40:40.366 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:40:40.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo rpm -qi grub2-tools 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Name : grub2-tools 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Epoch : 1 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Version : 2.06 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Release : 68.el9 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Architecture: x86_64 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Group : Unspecified 2024-08-17T16:40:40.397 INFO:teuthology.orchestra.run.smithi152.stdout:Size : 8264254 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:License : GPLv3+ 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Packager : builder@centos.org 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Vendor : CentOS 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Summary : Support tools for GRUB. 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:Description : 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:40:40.398 INFO:teuthology.orchestra.run.smithi152.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T16:40:40.399 INFO:teuthology.orchestra.run.smithi152.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T16:40:40.399 INFO:teuthology.orchestra.run.smithi152.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T16:40:40.399 INFO:teuthology.orchestra.run.smithi152.stdout:hardware devices. 2024-08-17T16:40:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:40:40.399 INFO:teuthology.orchestra.run.smithi152.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T16:40:40.400 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T16:40:40.400 INFO:teuthology.task.kernel:Updating grub on smithi152 to boot 5.14.0-480.el9.x86_64 2024-08-17T16:40:40.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T16:40:41.027 INFO:teuthology.orchestra.run.smithi152.stderr:Generating grub configuration file ... 2024-08-17T16:40:42.458 INFO:teuthology.orchestra.run.smithi152.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T16:40:42.485 INFO:teuthology.orchestra.run.smithi152.stderr:done 2024-08-17T16:40:42.487 DEBUG:teuthology.orchestra.run.smithi152:> mktemp 2024-08-17T16:40:42.502 INFO:teuthology.orchestra.run.smithi152.stdout:/tmp/tmp.7ZiFi7hrAw 2024-08-17T16:40:42.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.7ZiFi7hrAw 2024-08-17T16:40:42.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0666 /tmp/tmp.7ZiFi7hrAw 2024-08-17T16:40:42.701 DEBUG:teuthology.orchestra.remote:smithi152:/tmp/tmp.7ZiFi7hrAw is 6KB 2024-08-17T16:40:42.753 DEBUG:teuthology.orchestra.run.smithi152:> rm -fr /tmp/tmp.7ZiFi7hrAw 2024-08-17T16:40:42.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T16:40:42.837 INFO:teuthology.orchestra.run.smithi152.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T16:40:42.837 INFO:teuthology.orchestra.run.smithi152.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T16:40:42.838 INFO:teuthology.orchestra.run.smithi152.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T16:40:42.838 INFO:teuthology.orchestra.run.smithi152.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T16:40:42.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T16:40:42.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo shutdown -r now 2024-08-17T16:40:50.355 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:50.395 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:50.718 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:50.718 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:40:50.718 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:40:50.718 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:40:51.485 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:40:51.485 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:40:51.485 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:40:51.486 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T16:40:51.486 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T16:40:51.486 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T16:40:51.486 INFO:teuthology.orchestra.run.smithi142.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T16:40:51.486 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:40:51.486 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:40:51.900 DEBUG:teuthology.orchestra.run.smithi142:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:40:51.987 INFO:teuthology.orchestra.run.smithi142.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:40:51.988 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:40:51.988 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T16:40:51.988 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T16:40:51.988 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T16:40:51.988 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-08-17T16:40:51.988 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi142.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T16:40:51.988 DEBUG:teuthology.orchestra.run.smithi142:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:40:52.045 INFO:teuthology.orchestra.run.smithi142.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:40:52.045 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:40:52.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo rpm -qi grub2-tools 2024-08-17T16:40:52.082 INFO:teuthology.orchestra.run.smithi142.stdout:Name : grub2-tools 2024-08-17T16:40:52.083 INFO:teuthology.orchestra.run.smithi142.stdout:Epoch : 1 2024-08-17T16:40:52.083 INFO:teuthology.orchestra.run.smithi142.stdout:Version : 2.06 2024-08-17T16:40:52.083 INFO:teuthology.orchestra.run.smithi142.stdout:Release : 68.el9 2024-08-17T16:40:52.083 INFO:teuthology.orchestra.run.smithi142.stdout:Architecture: x86_64 2024-08-17T16:40:52.083 INFO:teuthology.orchestra.run.smithi142.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Group : Unspecified 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Size : 8264254 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:License : GPLv3+ 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Packager : builder@centos.org 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Vendor : CentOS 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Summary : Support tools for GRUB. 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout:Description : 2024-08-17T16:40:52.084 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:40:52.085 INFO:teuthology.orchestra.run.smithi142.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T16:40:52.085 INFO:teuthology.orchestra.run.smithi142.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T16:40:52.085 INFO:teuthology.orchestra.run.smithi142.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T16:40:52.085 INFO:teuthology.orchestra.run.smithi142.stdout:hardware devices. 2024-08-17T16:40:52.085 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:40:52.085 INFO:teuthology.orchestra.run.smithi142.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T16:40:52.086 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T16:40:52.086 INFO:teuthology.task.kernel:Updating grub on smithi142 to boot 5.14.0-480.el9.x86_64 2024-08-17T16:40:52.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T16:40:52.681 INFO:teuthology.orchestra.run.smithi142.stderr:Generating grub configuration file ... 2024-08-17T16:40:54.050 INFO:teuthology.orchestra.run.smithi142.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T16:40:54.077 INFO:teuthology.orchestra.run.smithi142.stderr:done 2024-08-17T16:40:54.079 DEBUG:teuthology.orchestra.run.smithi142:> mktemp 2024-08-17T16:40:54.094 INFO:teuthology.orchestra.run.smithi142.stdout:/tmp/tmp.wE5SgIe6gI 2024-08-17T16:40:54.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wE5SgIe6gI 2024-08-17T16:40:54.171 DEBUG:teuthology.orchestra.run.smithi142:> sudo chmod 0666 /tmp/tmp.wE5SgIe6gI 2024-08-17T16:40:54.306 DEBUG:teuthology.orchestra.remote:smithi142:/tmp/tmp.wE5SgIe6gI is 6KB 2024-08-17T16:40:54.326 DEBUG:teuthology.orchestra.run.smithi142:> rm -fr /tmp/tmp.wE5SgIe6gI 2024-08-17T16:40:54.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T16:40:54.402 INFO:teuthology.orchestra.run.smithi142.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T16:40:54.402 INFO:teuthology.orchestra.run.smithi142.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T16:40:54.403 INFO:teuthology.orchestra.run.smithi142.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T16:40:54.403 INFO:teuthology.orchestra.run.smithi142.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T16:40:54.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T16:40:54.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo shutdown -r now 2024-08-17T16:41:13.011 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-08-17T16:41:13.012 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:41:13.013 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:41:24.554 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-08-17T16:41:24.555 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi142.front.sepia.ceph.com' 2024-08-17T16:41:24.556 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi142.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:41:31.358 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.152 2024-08-17T16:41:40.367 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:41:40.367 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:41:42.878 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.142 2024-08-17T16:41:43.454 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.152 2024-08-17T16:41:51.888 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi142.front.sepia.ceph.com' 2024-08-17T16:41:51.889 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi142.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:41:54.974 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.142 2024-08-17T16:41:55.455 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:41:55.456 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:42:06.986 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi142.front.sepia.ceph.com' 2024-08-17T16:42:06.987 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi142.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:42:38.750 DEBUG:teuthology.orchestra.run.smithi142:> true 2024-08-17T16:42:39.208 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi142.front.sepia.ceph.com' 2024-08-17T16:42:39.208 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-480.el9.x86_64"... 2024-08-17T16:42:39.209 DEBUG:teuthology.orchestra.run.smithi142:> uname -r 2024-08-17T16:42:39.268 INFO:teuthology.orchestra.run.smithi142.stdout:5.14.0-480.el9.x86_64 2024-08-17T16:42:39.268 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T16:42:39.268 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T16:42:39.268 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T16:42:40.269 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-08-17T16:42:40.269 DEBUG:teuthology.orchestra.run.smithi142:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T16:42:40.328 INFO:teuthology.orchestra.run.smithi142.stdout:ttyS1 2024-08-17T16:42:40.355 DEBUG:teuthology.parallel:result is None 2024-08-17T16:42:55.466 DEBUG:teuthology.orchestra.remote:timed out 2024-08-17T16:43:10.468 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:43:10.469 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:43:10.794 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-08-17T16:43:11.152 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:43:11.152 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-480.el9.x86_64"... 2024-08-17T16:43:11.152 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-08-17T16:43:11.168 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-480.el9.x86_64 2024-08-17T16:43:11.168 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T16:43:11.168 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T16:43:11.168 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T16:43:12.170 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-08-17T16:43:12.170 DEBUG:teuthology.orchestra.run.smithi152:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T16:43:12.531 INFO:teuthology.orchestra.run.smithi152.stdout:ttyS1 2024-08-17T16:43:12.564 DEBUG:teuthology.parallel:result is None 2024-08-17T16:43:12.564 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-17T16:43:12.571 INFO:teuthology.task.internal:Creating test directory... 2024-08-17T16:43:12.571 DEBUG:teuthology.orchestra.run.smithi142:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T16:43:12.574 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T16:43:12.592 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-17T16:43:12.629 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-17T16:43:12.639 INFO:teuthology.task.internal:Creating archive directory... 2024-08-17T16:43:12.639 DEBUG:teuthology.orchestra.run.smithi142:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T16:43:12.642 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T16:43:12.782 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-17T16:43:12.789 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-17T16:43:12.789 DEBUG:teuthology.orchestra.run.smithi142:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T16:43:12.792 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T16:43:12.824 INFO:teuthology.orchestra.run.smithi142.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:43:12.829 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:43:12.835 INFO:teuthology.orchestra.run.smithi142.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:43:12.841 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:43:12.843 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-17T16:43:12.849 INFO:teuthology.task.internal:Configuring sudo... 2024-08-17T16:43:12.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T16:43:12.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T16:43:12.911 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-17T16:43:12.920 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-17T16:43:12.921 DEBUG:teuthology.orchestra.run.smithi142:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T16:43:12.947 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T16:43:12.967 DEBUG:teuthology.orchestra.run.smithi142:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:43:13.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:43:13.090 DEBUG:teuthology.orchestra.run.smithi142:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:43:13.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:43:13.214 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:43:13.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T16:43:13.280 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:43:13.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:43:13.379 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:43:13.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:43:13.505 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:43:13.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T16:43:13.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo service rsyslog restart 2024-08-17T16:43:13.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo service rsyslog restart 2024-08-17T16:43:13.635 INFO:teuthology.orchestra.run.smithi142.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T16:43:13.649 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T16:43:14.117 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-17T16:43:14.125 INFO:teuthology.task.internal:Starting timer... 2024-08-17T16:43:14.125 INFO:teuthology.run_tasks:Running task pcp... 2024-08-17T16:43:14.148 INFO:teuthology.run_tasks:Running task selinux... 2024-08-17T16:43:14.159 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-08-17T16:43:14.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo service auditd rotate 2024-08-17T16:43:14.224 INFO:teuthology.orchestra.run.smithi142.stdout:Rotating logs: 2024-08-17T16:43:14.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo service auditd rotate 2024-08-17T16:43:14.286 INFO:teuthology.orchestra.run.smithi152.stdout:Rotating logs: 2024-08-17T16:43:14.288 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-17T16:43:14.288 DEBUG:teuthology.orchestra.run.smithi142:> /usr/sbin/getenforce 2024-08-17T16:43:14.338 INFO:teuthology.orchestra.run.smithi142.stdout:Permissive 2024-08-17T16:43:14.339 DEBUG:teuthology.orchestra.run.smithi152:> /usr/sbin/getenforce 2024-08-17T16:43:14.373 INFO:teuthology.orchestra.run.smithi152.stdout:Permissive 2024-08-17T16:43:14.373 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi142.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi152.front.sepia.ceph.com': 'permissive'} 2024-08-17T16:43:14.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T16:43:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:43:14.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T16:43:14.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:43:14.444 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-17T16:43:14.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo /usr/sbin/setenforce permissive 2024-08-17T16:43:14.473 DEBUG:teuthology.orchestra.run.smithi152:> sudo /usr/sbin/setenforce permissive 2024-08-17T16:43:14.525 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-17T16:43:14.535 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-17T16:43:14.547 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-17T16:43:14.548 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi142.front.sepia.ceph.com,smithi152.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-17T16:47:52.133 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi142.front.sepia.ceph.com'), Remote(name='ubuntu@smithi152.front.sepia.ceph.com')] 2024-08-17T16:47:52.134 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi142.front.sepia.ceph.com' 2024-08-17T16:47:52.135 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi142.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:47:52.216 DEBUG:teuthology.orchestra.run.smithi142:> true 2024-08-17T16:47:52.298 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi142.front.sepia.ceph.com' 2024-08-17T16:47:52.298 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:47:52.299 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:47:52.379 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-08-17T16:47:52.462 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-08-17T16:47:52.462 INFO:teuthology.run_tasks:Running task clock... 2024-08-17T16:47:52.472 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-17T16:47:52.472 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T16:47:52.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T16:47:52.475 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T16:47:52.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T16:47:52.503 INFO:teuthology.orchestra.run.smithi142.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T16:47:52.517 INFO:teuthology.orchestra.run.smithi142.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T16:47:52.532 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T16:47:52.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: ntpd: command not found 2024-08-17T16:47:52.546 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T16:47:52.552 INFO:teuthology.orchestra.run.smithi142.stdout:506 Cannot talk to daemon 2024-08-17T16:47:52.565 INFO:teuthology.orchestra.run.smithi142.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T16:47:52.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: ntpd: command not found 2024-08-17T16:47:52.579 INFO:teuthology.orchestra.run.smithi142.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T16:47:52.582 INFO:teuthology.orchestra.run.smithi152.stdout:506 Cannot talk to daemon 2024-08-17T16:47:52.597 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T16:47:52.611 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T16:47:52.636 INFO:teuthology.orchestra.run.smithi142.stderr:bash: line 1: ntpq: command not found 2024-08-17T16:47:52.640 INFO:teuthology.orchestra.run.smithi142.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T16:47:52.640 INFO:teuthology.orchestra.run.smithi142.stdout:=============================================================================== 2024-08-17T16:47:52.640 INFO:teuthology.orchestra.run.smithi142.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.640 INFO:teuthology.orchestra.run.smithi142.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.640 INFO:teuthology.orchestra.run.smithi142.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.640 INFO:teuthology.orchestra.run.smithi142.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.666 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-08-17T16:47:52.669 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T16:47:52.669 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-08-17T16:47:52.669 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.669 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.669 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.669 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:47:52.670 INFO:teuthology.run_tasks:Running task pexec... 2024-08-17T16:47:52.680 INFO:teuthology.task.pexec:Executing custom commands... 2024-08-17T16:47:52.680 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi142.front.sepia.ceph.com 2024-08-17T16:47:52.680 DEBUG:teuthology.orchestra.run.smithi142:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T16:47:52.681 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi152.front.sepia.ceph.com 2024-08-17T16:47:52.681 DEBUG:teuthology.orchestra.run.smithi152:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T16:47:53.253 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:15 ago on Sat 17 Aug 2024 04:46:38 PM UTC. 2024-08-17T16:47:53.276 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:15 ago on Sat 17 Aug 2024 04:46:38 PM UTC. 2024-08-17T16:47:53.338 INFO:teuthology.orchestra.run.smithi142.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T16:47:53.361 INFO:teuthology.orchestra.run.smithi152.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T16:47:53.379 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:47:53.393 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:47:53.393 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout:Upgrading: 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T16:47:53.394 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:47:53.395 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:47:53.395 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:47:53.395 INFO:teuthology.orchestra.run.smithi142.stdout:Install 5 Packages 2024-08-17T16:47:53.395 INFO:teuthology.orchestra.run.smithi142.stdout:Upgrade 2 Packages 2024-08-17T16:47:53.395 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:47:53.403 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:47:53.403 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 5.2 M 2024-08-17T16:47:53.403 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:47:53.404 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:47:53.404 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrading: 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:47:53.405 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:47:53.406 INFO:teuthology.orchestra.run.smithi152.stdout:Install 5 Packages 2024-08-17T16:47:53.406 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrade 2 Packages 2024-08-17T16:47:53.406 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:47:53.418 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 5.2 M 2024-08-17T16:47:53.418 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:47:53.911 INFO:teuthology.orchestra.run.smithi142.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 443 kB/s | 44 kB 00:00 2024-08-17T16:47:53.936 INFO:teuthology.orchestra.run.smithi142.stdout:(2/7): python3-configshell-1.1.30-1.el9.noarch. 571 kB/s | 72 kB 00:00 2024-08-17T16:47:53.962 INFO:teuthology.orchestra.run.smithi142.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 559 kB/s | 84 kB 00:00 2024-08-17T16:47:54.020 INFO:teuthology.orchestra.run.smithi142.stdout:(4/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.5 MB/s | 837 kB 00:00 2024-08-17T16:47:54.054 INFO:teuthology.orchestra.run.smithi142.stdout:(5/7): libnvme-1.9-2.el9.x86_64.rpm 1.1 MB/s | 106 kB 00:00 2024-08-17T16:47:54.138 INFO:teuthology.orchestra.run.smithi142.stdout:(6/7): nvme-cli-2.9.1-3.el9.x86_64.rpm 7.6 MB/s | 903 kB 00:00 2024-08-17T16:47:54.205 INFO:teuthology.orchestra.run.smithi142.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 12 MB/s | 3.2 MB 00:00 2024-08-17T16:47:54.205 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:47:54.205 INFO:teuthology.orchestra.run.smithi142.stdout:Total 6.4 MB/s | 5.2 MB 00:00 2024-08-17T16:47:54.316 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:47:54.317 INFO:teuthology.orchestra.run.smithi152.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 127 kB/s | 44 kB 00:00 2024-08-17T16:47:54.333 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:47:54.333 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:47:54.342 INFO:teuthology.orchestra.run.smithi152.stdout:(2/7): python3-configshell-1.1.30-1.el9.noarch. 191 kB/s | 72 kB 00:00 2024-08-17T16:47:54.392 INFO:teuthology.orchestra.run.smithi152.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 197 kB/s | 84 kB 00:00 2024-08-17T16:47:54.576 INFO:teuthology.orchestra.run.smithi152.stdout:(4/7): libnvme-1.9-2.el9.x86_64.rpm 579 kB/s | 106 kB 00:00 2024-08-17T16:47:54.578 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:47:54.578 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:47:54.843 INFO:teuthology.orchestra.run.smithi152.stdout:(5/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 1.6 MB/s | 837 kB 00:00 2024-08-17T16:47:54.974 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:47:55.044 INFO:teuthology.orchestra.run.smithi152.stdout:(6/7): nvme-cli-2.9.1-3.el9.x86_64.rpm 1.9 MB/s | 903 kB 00:00 2024-08-17T16:47:55.119 INFO:teuthology.orchestra.run.smithi152.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 4.1 MB/s | 3.2 MB 00:00 2024-08-17T16:47:55.119 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:47:55.119 INFO:teuthology.orchestra.run.smithi152.stdout:Total 3.0 MB/s | 5.2 MB 00:01 2024-08-17T16:47:55.199 INFO:teuthology.orchestra.run.smithi142.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/9 2024-08-17T16:47:55.236 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:47:55.254 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:47:55.254 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:47:55.259 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/9 2024-08-17T16:47:55.294 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/9 2024-08-17T16:47:55.328 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/9 2024-08-17T16:47:55.352 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:47:55.495 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:47:55.495 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:47:55.629 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:47:55.688 INFO:teuthology.orchestra.run.smithi142.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:47:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:47:56.083 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:47:56.115 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/9 2024-08-17T16:47:56.137 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/9 2024-08-17T16:47:56.148 INFO:teuthology.orchestra.run.smithi142.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 8/9 2024-08-17T16:47:56.190 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/9 2024-08-17T16:47:56.209 INFO:teuthology.orchestra.run.smithi142.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:47:56.225 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/9 2024-08-17T16:47:56.258 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/9 2024-08-17T16:47:56.284 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:47:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:47:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:47:57.021 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:47:57.048 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/9 2024-08-17T16:47:57.081 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 8/9 2024-08-17T16:47:57.141 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 5/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libnvme-1.9-2.el9.x86_64 6/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libnvme-1.6-1.el9.x86_64 7/9 2024-08-17T16:47:57.201 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 8/9 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 9/9 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout:Upgraded: 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: runc-4:1.1.13-4.el9.x86_64 2024-08-17T16:47:57.585 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:47:57.586 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:47:57.953 DEBUG:teuthology.parallel:result is None 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 5/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libnvme-1.9-2.el9.x86_64 6/9 2024-08-17T16:47:58.156 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libnvme-1.6-1.el9.x86_64 7/9 2024-08-17T16:47:58.157 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 8/9 2024-08-17T16:47:58.534 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 9/9 2024-08-17T16:47:58.534 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:47:58.534 INFO:teuthology.orchestra.run.smithi152.stdout:Upgraded: 2024-08-17T16:47:58.534 INFO:teuthology.orchestra.run.smithi152.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T16:47:58.535 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:47:58.536 INFO:teuthology.orchestra.run.smithi152.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T16:47:58.536 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T16:47:58.536 INFO:teuthology.orchestra.run.smithi152.stdout: runc-4:1.1.13-4.el9.x86_64 2024-08-17T16:47:58.536 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:47:58.536 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:47:58.905 DEBUG:teuthology.parallel:result is None 2024-08-17T16:47:58.905 INFO:teuthology.run_tasks:Running task install... 2024-08-17T16:47:58.915 DEBUG:teuthology.task.install:project ceph 2024-08-17T16:47:58.915 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}} 2024-08-17T16:47:58.916 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T16:47:58.916 INFO:teuthology.task.install:Using flavor: default 2024-08-17T16:47:58.921 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-17T16:47:58.921 INFO:teuthology.task.install:extra packages: [] 2024-08-17T16:47:58.922 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T16:47:58.922 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:47:58.922 INFO:teuthology.packaging:ref: None 2024-08-17T16:47:58.922 INFO:teuthology.packaging:tag: None 2024-08-17T16:47:58.922 INFO:teuthology.packaging:branch: quincy 2024-08-17T16:47:58.923 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:47:58.923 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T16:47:58.925 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T16:47:58.925 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:47:58.925 INFO:teuthology.packaging:ref: None 2024-08-17T16:47:58.925 INFO:teuthology.packaging:tag: None 2024-08-17T16:47:58.925 INFO:teuthology.packaging:branch: quincy 2024-08-17T16:47:58.925 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:47:58.926 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T16:47:59.151 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/ 2024-08-17T16:47:59.152 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1404.gf2c39d4d 2024-08-17T16:47:59.193 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/ 2024-08-17T16:47:59.193 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1404.gf2c39d4d 2024-08-17T16:47:59.277 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T16:47:59.277 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:47:59.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T16:47:59.309 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T16:47:59.309 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:47:59.309 INFO:teuthology.packaging:ref: None 2024-08-17T16:47:59.309 INFO:teuthology.packaging:tag: None 2024-08-17T16:47:59.309 INFO:teuthology.packaging:branch: quincy 2024-08-17T16:47:59.309 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:47:59.310 DEBUG:teuthology.orchestra.run.smithi142:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T16:47:59.317 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T16:47:59.317 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:47:59.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T16:47:59.343 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T16:47:59.343 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:47:59.343 INFO:teuthology.packaging:ref: None 2024-08-17T16:47:59.343 INFO:teuthology.packaging:tag: None 2024-08-17T16:47:59.344 INFO:teuthology.packaging:branch: quincy 2024-08-17T16:47:59.344 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:47:59.344 DEBUG:teuthology.orchestra.run.smithi152:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T16:47:59.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T16:47:59.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T16:47:59.460 DEBUG:teuthology.orchestra.run.smithi142:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T16:47:59.486 DEBUG:teuthology.orchestra.run.smithi152:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T16:47:59.528 INFO:teuthology.orchestra.run.smithi142.stdout:check_obsoletes = 1 2024-08-17T16:47:59.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum clean all 2024-08-17T16:47:59.552 INFO:teuthology.orchestra.run.smithi152.stdout:check_obsoletes = 1 2024-08-17T16:47:59.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum clean all 2024-08-17T16:47:59.776 INFO:teuthology.orchestra.run.smithi142.stdout:45 files removed 2024-08-17T16:47:59.777 INFO:teuthology.orchestra.run.smithi152.stdout:45 files removed 2024-08-17T16:47:59.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-radosgw 2024-08-17T16:47:59.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-radosgw 2024-08-17T16:48:00.396 INFO:teuthology.orchestra.run.smithi142.stdout:ceph packages for x86_64 249 kB/s | 77 kB 00:00 2024-08-17T16:48:00.402 INFO:teuthology.orchestra.run.smithi152.stdout:ceph packages for x86_64 242 kB/s | 77 kB 00:00 2024-08-17T16:48:00.735 INFO:teuthology.orchestra.run.smithi152.stdout:ceph noarch packages 38 kB/s | 11 kB 00:00 2024-08-17T16:48:00.738 INFO:teuthology.orchestra.run.smithi142.stdout:ceph noarch packages 37 kB/s | 11 kB 00:00 2024-08-17T16:48:01.055 INFO:teuthology.orchestra.run.smithi142.stdout:ceph source packages 6.6 kB/s | 1.8 kB 00:00 2024-08-17T16:48:01.069 INFO:teuthology.orchestra.run.smithi152.stdout:ceph source packages 6.2 kB/s | 1.8 kB 00:00 2024-08-17T16:48:01.611 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.2 MB 00:00 2024-08-17T16:48:01.839 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.2 MB 00:00 2024-08-17T16:48:03.304 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 32 MB/s | 20 MB 00:00 2024-08-17T16:48:03.523 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - AppStream 34 MB/s | 20 MB 00:00 2024-08-17T16:48:07.680 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.6 MB 00:00 2024-08-17T16:48:07.691 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.6 MB 00:00 2024-08-17T16:48:09.222 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 58 kB/s | 18 kB 00:00 2024-08-17T16:48:09.975 INFO:teuthology.orchestra.run.smithi142.stdout:CentOS Stream 9 - Extras packages 18 kB/s | 18 kB 00:01 2024-08-17T16:48:10.419 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 21 MB/s | 22 MB 00:01 2024-08-17T16:48:11.253 INFO:teuthology.orchestra.run.smithi142.stdout:Extra Packages for Enterprise Linux 20 MB/s | 22 MB 00:01 2024-08-17T16:48:15.883 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-08-17T16:48:16.809 INFO:teuthology.orchestra.run.smithi142.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-17T16:48:18.247 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:48:18.248 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:18.248 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 14 M 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrading: 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: librados2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.1 M 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 5.0 M 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 22 M 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T16:48:18.249 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 637 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 457 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 4.9 M 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 45 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 120 k 2024-08-17T16:48:18.250 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 182 k 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 368 k 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 343 k 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 116 k 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout:Install 22 Packages 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrade 2 Packages 2024-08-17T16:48:18.251 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:18.252 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 58 M 2024-08-17T16:48:18.252 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:48:18.861 INFO:teuthology.orchestra.run.smithi152.stdout:(1/24): ceph-base-17.2.7-1404.gf2c39d4d.el9.x86 11 MB/s | 5.0 MB 00:00 2024-08-17T16:48:18.926 INFO:teuthology.orchestra.run.smithi152.stdout:(2/24): ceph-selinux-17.2.7-1404.gf2c39d4d.el9. 378 kB/s | 25 kB 00:00 2024-08-17T16:48:19.031 INFO:teuthology.orchestra.run.smithi152.stdout:(3/24): libcephfs2-17.2.7-1404.gf2c39d4d.el9.x8 5.9 MB/s | 637 kB 00:00 2024-08-17T16:48:19.099 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:48:19.101 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:19.101 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:48:19.101 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:19.101 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:48:19.101 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-radosgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 14 M 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout:Upgrading: 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: librados2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.1 M 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: librbd1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-base x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 5.0 M 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 22 M 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-selinux x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: libcephfs2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 637 k 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T16:48:19.102 INFO:teuthology.orchestra.run.smithi142.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: libradosstriper1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 457 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: librgw2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 4.9 M 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-ceph-argparse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 45 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 120 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cephfs x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 182 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rados x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 368 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 343 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 116 k 2024-08-17T16:48:19.103 INFO:teuthology.orchestra.run.smithi142.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:Install 22 Packages 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:Upgrade 2 Packages 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 58 M 2024-08-17T16:48:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:48:19.118 INFO:teuthology.orchestra.run.smithi152.stdout:(4/24): libradosstriper1-17.2.7-1404.gf2c39d4d. 5.2 MB/s | 457 kB 00:00 2024-08-17T16:48:19.402 INFO:teuthology.orchestra.run.smithi152.stdout:(5/24): librgw2-17.2.7-1404.gf2c39d4d.el9.x86_6 17 MB/s | 4.9 MB 00:00 2024-08-17T16:48:19.468 INFO:teuthology.orchestra.run.smithi152.stdout:(6/24): python3-ceph-argparse-17.2.7-1404.gf2c3 680 kB/s | 45 kB 00:00 2024-08-17T16:48:19.702 INFO:teuthology.orchestra.run.smithi152.stdout:(7/24): ceph-common-17.2.7-1404.gf2c39d4d.el9.x 17 MB/s | 22 MB 00:01 2024-08-17T16:48:19.727 INFO:teuthology.orchestra.run.smithi152.stdout:(8/24): python3-ceph-common-17.2.7-1404.gf2c39d 464 kB/s | 120 kB 00:00 2024-08-17T16:48:19.878 INFO:teuthology.orchestra.run.smithi152.stdout:(9/24): ceph-radosgw-17.2.7-1404.gf2c39d4d.el9. 9.7 MB/s | 14 MB 00:01 2024-08-17T16:48:19.903 INFO:teuthology.orchestra.run.smithi152.stdout:(10/24): python3-cephfs-17.2.7-1404.gf2c39d4d.e 907 kB/s | 182 kB 00:00 2024-08-17T16:48:19.928 INFO:teuthology.orchestra.run.smithi152.stdout:(11/24): python3-rados-17.2.7-1404.gf2c39d4d.el 1.8 MB/s | 368 kB 00:00 2024-08-17T16:48:19.979 INFO:teuthology.orchestra.run.smithi152.stdout:(12/24): python3-rbd-17.2.7-1404.gf2c39d4d.el9. 3.3 MB/s | 343 kB 00:00 2024-08-17T16:48:20.004 INFO:teuthology.orchestra.run.smithi152.stdout:(13/24): python3-rgw-17.2.7-1404.gf2c39d4d.el9. 1.1 MB/s | 116 kB 00:00 2024-08-17T16:48:20.079 INFO:teuthology.orchestra.run.smithi152.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.0 MB/s | 160 kB 00:00 2024-08-17T16:48:20.105 INFO:teuthology.orchestra.run.smithi152.stdout:(15/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 360 kB/s | 45 kB 00:00 2024-08-17T16:48:20.163 INFO:teuthology.orchestra.run.smithi152.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-08-17T16:48:20.189 INFO:teuthology.orchestra.run.smithi152.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 504 kB/s | 42 kB 00:00 2024-08-17T16:48:20.273 INFO:teuthology.orchestra.run.smithi152.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.4 MB/s | 662 kB 00:00 2024-08-17T16:48:20.331 INFO:teuthology.orchestra.run.smithi152.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 342 kB/s | 49 kB 00:00 2024-08-17T16:48:20.365 INFO:teuthology.orchestra.run.smithi152.stdout:(20/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.5 MB/s | 309 kB 00:00 2024-08-17T16:48:20.390 INFO:teuthology.orchestra.run.smithi152.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 587 kB/s | 67 kB 00:00 2024-08-17T16:48:20.427 INFO:teuthology.orchestra.run.smithi142.stdout:(1/24): ceph-common-17.2.7-1404.gf2c39d4d.el9.x 20 MB/s | 22 MB 00:01 2024-08-17T16:48:20.482 INFO:teuthology.orchestra.run.smithi152.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-08-17T16:48:20.494 INFO:teuthology.orchestra.run.smithi142.stdout:(2/24): ceph-base-17.2.7-1404.gf2c39d4d.el9.x86 4.2 MB/s | 5.0 MB 00:01 2024-08-17T16:48:20.519 INFO:teuthology.orchestra.run.smithi142.stdout:(3/24): ceph-selinux-17.2.7-1404.gf2c39d4d.el9. 275 kB/s | 25 kB 00:00 2024-08-17T16:48:20.557 INFO:teuthology.orchestra.run.smithi152.stdout:(23/24): librados2-17.2.7-1404.gf2c39d4d.el9.x8 16 MB/s | 3.1 MB 00:00 2024-08-17T16:48:20.569 INFO:teuthology.orchestra.run.smithi142.stdout:(4/24): libcephfs2-17.2.7-1404.gf2c39d4d.el9.x8 8.3 MB/s | 637 kB 00:00 2024-08-17T16:48:20.603 INFO:teuthology.orchestra.run.smithi142.stdout:(5/24): libradosstriper1-17.2.7-1404.gf2c39d4d. 5.3 MB/s | 457 kB 00:00 2024-08-17T16:48:20.624 INFO:teuthology.orchestra.run.smithi152.stdout:(24/24): librbd1-17.2.7-1404.gf2c39d4d.el9.x86_ 13 MB/s | 2.9 MB 00:00 2024-08-17T16:48:20.629 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:20.629 INFO:teuthology.orchestra.run.smithi152.stdout:Total 24 MB/s | 58 MB 00:02 2024-08-17T16:48:20.690 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:48:20.722 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:48:20.722 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:48:20.745 INFO:teuthology.orchestra.run.smithi142.stdout:(6/24): ceph-radosgw-17.2.7-1404.gf2c39d4d.el9. 10 MB/s | 14 MB 00:01 2024-08-17T16:48:20.770 INFO:teuthology.orchestra.run.smithi142.stdout:(7/24): python3-ceph-argparse-17.2.7-1404.gf2c3 270 kB/s | 45 kB 00:00 2024-08-17T16:48:20.795 INFO:teuthology.orchestra.run.smithi142.stdout:(8/24): python3-ceph-common-17.2.7-1404.gf2c39d 2.4 MB/s | 120 kB 00:00 2024-08-17T16:48:20.820 INFO:teuthology.orchestra.run.smithi142.stdout:(9/24): python3-cephfs-17.2.7-1404.gf2c39d4d.el 3.5 MB/s | 182 kB 00:00 2024-08-17T16:48:20.846 INFO:teuthology.orchestra.run.smithi142.stdout:(10/24): python3-rados-17.2.7-1404.gf2c39d4d.el 7.2 MB/s | 368 kB 00:00 2024-08-17T16:48:20.871 INFO:teuthology.orchestra.run.smithi142.stdout:(11/24): python3-rbd-17.2.7-1404.gf2c39d4d.el9. 6.7 MB/s | 343 kB 00:00 2024-08-17T16:48:20.896 INFO:teuthology.orchestra.run.smithi142.stdout:(12/24): python3-rgw-17.2.7-1404.gf2c39d4d.el9. 2.3 MB/s | 116 kB 00:00 2024-08-17T16:48:20.988 INFO:teuthology.orchestra.run.smithi142.stdout:(13/24): librgw2-17.2.7-1404.gf2c39d4d.el9.x86_ 12 MB/s | 4.9 MB 00:00 2024-08-17T16:48:21.005 INFO:teuthology.orchestra.run.smithi142.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 416 kB/s | 45 kB 00:00 2024-08-17T16:48:21.039 INFO:teuthology.orchestra.run.smithi142.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 957 kB/s | 160 kB 00:00 2024-08-17T16:48:21.081 INFO:teuthology.orchestra.run.smithi142.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-08-17T16:48:21.131 INFO:teuthology.orchestra.run.smithi142.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.3 MB/s | 292 kB 00:00 2024-08-17T16:48:21.180 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:48:21.180 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:48:21.189 INFO:teuthology.orchestra.run.smithi142.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 3.2 MB/s | 662 kB 00:00 2024-08-17T16:48:21.240 INFO:teuthology.orchestra.run.smithi142.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 457 kB/s | 49 kB 00:00 2024-08-17T16:48:21.265 INFO:teuthology.orchestra.run.smithi142.stdout:(20/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.6 MB/s | 309 kB 00:00 2024-08-17T16:48:21.290 INFO:teuthology.orchestra.run.smithi142.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 685 kB/s | 67 kB 00:00 2024-08-17T16:48:21.374 INFO:teuthology.orchestra.run.smithi142.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 12 MB/s | 1.6 MB 00:00 2024-08-17T16:48:21.466 INFO:teuthology.orchestra.run.smithi142.stdout:(23/24): librbd1-17.2.7-1404.gf2c39d4d.el9.x86_ 17 MB/s | 2.9 MB 00:00 2024-08-17T16:48:21.516 INFO:teuthology.orchestra.run.smithi142.stdout:(24/24): librados2-17.2.7-1404.gf2c39d4d.el9.x8 12 MB/s | 3.1 MB 00:00 2024-08-17T16:48:21.520 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:21.520 INFO:teuthology.orchestra.run.smithi142.stdout:Total 24 MB/s | 58 MB 00:02 2024-08-17T16:48:21.578 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:48:21.609 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:48:21.609 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:48:22.063 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:48:22.063 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:48:22.367 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:48:22.749 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-08-17T16:48:22.818 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T16:48:22.901 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T16:48:22.988 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T16:48:23.019 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T16:48:23.105 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T16:48:23.158 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-08-17T16:48:23.201 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-08-17T16:48:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:48:23.625 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-08-17T16:48:23.643 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-08-17T16:48:23.673 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T16:48:23.702 INFO:teuthology.orchestra.run.smithi142.stdout: Upgrading : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T16:48:23.736 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T16:48:23.773 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-08-17T16:48:23.815 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T16:48:23.818 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 10/26 2024-08-17T16:48:23.854 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T16:48:23.897 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T16:48:23.915 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T16:48:23.919 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T16:48:23.942 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T16:48:24.008 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T16:48:24.012 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T16:48:24.049 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-08-17T16:48:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-08-17T16:48:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-08-17T16:48:24.101 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-08-17T16:48:24.133 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-08-17T16:48:24.434 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-08-17T16:48:24.480 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T16:48:24.550 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-08-17T16:48:24.573 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T16:48:24.580 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T16:48:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T16:48:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 19/26 2024-08-17T16:48:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-08-17T16:48:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 20/26 2024-08-17T16:48:24.716 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 10/26 2024-08-17T16:48:24.753 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T16:48:24.814 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T16:48:24.841 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T16:48:24.907 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T16:48:24.948 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-08-17T16:48:24.999 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-08-17T16:48:25.032 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-08-17T16:48:25.334 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-08-17T16:48:25.379 INFO:teuthology.orchestra.run.smithi142.stdout: Upgrading : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T16:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T16:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 19/26 2024-08-17T16:48:25.572 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 20/26 2024-08-17T16:48:27.059 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T16:48:27.097 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T16:48:27.628 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T16:48:27.667 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T16:48:27.778 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T16:48:27.778 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T16:48:27.778 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T16:48:27.778 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:27.810 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T16:48:28.044 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T16:48:28.080 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T16:48:28.599 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T16:48:28.633 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T16:48:28.726 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T16:48:28.726 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T16:48:28.726 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T16:48:28.726 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:28.751 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T16:48:37.507 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T16:48:37.507 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /sys 2024-08-17T16:48:37.507 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /proc 2024-08-17T16:48:37.507 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /mnt 2024-08-17T16:48:37.508 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /var/tmp 2024-08-17T16:48:37.508 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /home 2024-08-17T16:48:37.508 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /root 2024-08-17T16:48:37.508 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /tmp 2024-08-17T16:48:37.508 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:38.418 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T16:48:38.418 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /sys 2024-08-17T16:48:38.418 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /proc 2024-08-17T16:48:38.418 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /mnt 2024-08-17T16:48:38.418 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /var/tmp 2024-08-17T16:48:38.419 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /home 2024-08-17T16:48:38.419 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /root 2024-08-17T16:48:38.419 INFO:teuthology.orchestra.run.smithi142.stdout:skipping the directory /tmp 2024-08-17T16:48:38.419 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:38.856 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T16:48:38.934 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T16:48:38.934 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:48:38.935 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T16:48:38.935 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:48:38.935 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:48:38.935 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:38.936 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T16:48:39.026 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T16:48:39.026 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T16:48:39.715 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T16:48:39.801 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T16:48:39.801 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:48:39.801 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T16:48:39.801 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:48:39.801 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:48:39.801 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:39.802 INFO:teuthology.orchestra.run.smithi142.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T16:48:39.893 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T16:48:39.893 INFO:teuthology.orchestra.run.smithi142.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T16:48:40.398 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T16:48:40.398 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/26 2024-08-17T16:48:40.398 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T16:48:40.398 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T16:48:40.398 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 6/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 7/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 8/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 9/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 10/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 13/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-08-17T16:48:40.399 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-08-17T16:48:40.400 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 25/26 2024-08-17T16:48:41.157 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T16:48:41.157 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout:Upgraded: 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T16:48:41.158 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.159 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T16:48:41.160 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.160 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.160 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:41.160 INFO:teuthology.orchestra.run.smithi152.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T16:48:41.160 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:41.160 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 6/26 2024-08-17T16:48:41.211 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 7/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 8/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 9/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 10/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 13/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-08-17T16:48:41.212 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-08-17T16:48:41.213 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-08-17T16:48:41.213 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-08-17T16:48:41.213 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T16:48:41.213 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-08-17T16:48:41.213 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 25/26 2024-08-17T16:48:41.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-test 2024-08-17T16:48:42.033 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:27 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:48:42.069 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T16:48:42.069 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout:Upgraded: 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.070 INFO:teuthology.orchestra.run.smithi142.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.071 INFO:teuthology.orchestra.run.smithi142.stdout: python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:42.072 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:48:42.160 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 48 M 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 162 k 2024-08-17T16:48:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T16:48:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T16:48:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:42.162 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:48:42.162 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:42.162 INFO:teuthology.orchestra.run.smithi152.stdout:Install 4 Packages 2024-08-17T16:48:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:42.163 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 48 M 2024-08-17T16:48:42.163 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 193 M 2024-08-17T16:48:42.163 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:48:42.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-test 2024-08-17T16:48:42.547 INFO:teuthology.orchestra.run.smithi152.stdout:(1/4): libcephsqlite-17.2.7-1404.gf2c39d4d.el9. 618 kB/s | 162 kB 00:00 2024-08-17T16:48:42.789 INFO:teuthology.orchestra.run.smithi152.stdout:(2/4): socat-1.7.4.1-6.el9.x86_64.rpm 602 kB/s | 304 kB 00:00 2024-08-17T16:48:42.889 INFO:teuthology.orchestra.run.smithi152.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 186 kB/s | 64 kB 00:00 2024-08-17T16:48:42.989 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:26 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:48:43.116 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:48:43.116 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-test x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 48 M 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: libcephsqlite x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 162 k 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:43.117 INFO:teuthology.orchestra.run.smithi142.stdout:Install 4 Packages 2024-08-17T16:48:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:43.118 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 48 M 2024-08-17T16:48:43.118 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 193 M 2024-08-17T16:48:43.118 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:48:43.415 INFO:teuthology.orchestra.run.smithi142.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.7 MB/s | 304 kB 00:00 2024-08-17T16:48:43.481 INFO:teuthology.orchestra.run.smithi142.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 971 kB/s | 64 kB 00:00 2024-08-17T16:48:43.567 INFO:teuthology.orchestra.run.smithi142.stdout:(3/4): libcephsqlite-17.2.7-1404.gf2c39d4d.el9. 501 kB/s | 162 kB 00:00 2024-08-17T16:48:43.772 INFO:teuthology.orchestra.run.smithi152.stdout:(4/4): ceph-test-17.2.7-1404.gf2c39d4d.el9.x86_ 32 MB/s | 48 MB 00:01 2024-08-17T16:48:43.774 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:43.775 INFO:teuthology.orchestra.run.smithi152.stdout:Total 30 MB/s | 48 MB 00:01 2024-08-17T16:48:43.802 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:48:43.823 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:48:43.823 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:48:44.188 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:48:44.188 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:48:44.722 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:48:44.792 INFO:teuthology.orchestra.run.smithi142.stdout:(4/4): ceph-test-17.2.7-1404.gf2c39d4d.el9.x86_ 31 MB/s | 48 MB 00:01 2024-08-17T16:48:44.794 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:44.794 INFO:teuthology.orchestra.run.smithi142.stdout:Total 29 MB/s | 48 MB 00:01 2024-08-17T16:48:44.798 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T16:48:44.822 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:48:44.838 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T16:48:44.841 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:48:44.841 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:48:44.858 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T16:48:45.209 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:48:45.209 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:48:45.728 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:48:45.809 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T16:48:45.859 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T16:48:45.885 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T16:48:49.251 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T16:48:49.323 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T16:48:50.281 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T16:48:50.358 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T16:48:51.605 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T16:48:51.606 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/4 2024-08-17T16:48:51.606 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/4 2024-08-17T16:48:51.606 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T16:48:51.886 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T16:48:51.886 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:51.886 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:48:51.886 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:51.886 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:51.887 INFO:teuthology.orchestra.run.smithi152.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T16:48:51.887 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T16:48:51.887 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:51.887 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:48:52.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph 2024-08-17T16:48:52.638 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T16:48:52.638 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/4 2024-08-17T16:48:52.638 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/4 2024-08-17T16:48:52.638 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T16:48:52.717 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:37 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:48:52.845 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:48:52.847 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: ceph x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 6.5 k 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.1 M 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 1.4 M 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 240 k 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.9 M 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 15 M 2024-08-17T16:48:52.848 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T16:48:52.849 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T16:48:52.850 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout:Install 34 Packages 2024-08-17T16:48:52.851 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:52.852 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 27 M 2024-08-17T16:48:52.852 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 96 M 2024-08-17T16:48:52.852 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T16:48:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:52.964 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:48:53.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph 2024-08-17T16:48:53.634 INFO:teuthology.orchestra.run.smithi152.stdout:(1/34): ceph-17.2.7-1404.gf2c39d4d.el9.x86_64.r 41 kB/s | 6.5 kB 00:00 2024-08-17T16:48:53.814 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:37 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:48:53.843 INFO:teuthology.orchestra.run.smithi152.stdout:(2/34): ceph-mds-17.2.7-1404.gf2c39d4d.el9.x86_ 5.6 MB/s | 2.1 MB 00:00 2024-08-17T16:48:53.901 INFO:teuthology.orchestra.run.smithi152.stdout:(3/34): ceph-mgr-17.2.7-1404.gf2c39d4d.el9.x86_ 3.3 MB/s | 1.4 MB 00:00 2024-08-17T16:48:53.945 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:48:53.947 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:48:53.947 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:53.947 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:48:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: ceph x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 6.5 k 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mds x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.1 M 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 1.4 M 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 240 k 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mon x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.9 M 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-osd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 15 M 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T16:48:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-collections 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T16:48:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T16:48:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout:Install 34 Packages 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:48:53.951 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 27 M 2024-08-17T16:48:53.952 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 96 M 2024-08-17T16:48:53.952 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:48:53.985 INFO:teuthology.orchestra.run.smithi152.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1404.gf2c3 2.8 MB/s | 240 kB 00:00 2024-08-17T16:48:54.227 INFO:teuthology.orchestra.run.smithi152.stdout:(5/34): ceph-mon-17.2.7-1404.gf2c39d4d.el9.x86_ 6.6 MB/s | 3.9 MB 00:00 2024-08-17T16:48:54.252 INFO:teuthology.orchestra.run.smithi152.stdout:(6/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 948 kB/s | 253 kB 00:00 2024-08-17T16:48:54.428 INFO:teuthology.orchestra.run.smithi152.stdout:(7/34): ceph-osd-17.2.7-1404.gf2c39d4d.el9.x86_ 26 MB/s | 15 MB 00:00 2024-08-17T16:48:54.453 INFO:teuthology.orchestra.run.smithi152.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 531 kB/s | 106 kB 00:00 2024-08-17T16:48:54.503 INFO:teuthology.orchestra.run.smithi152.stdout:(9/34): python3-requests-2.25.1-8.el9.noarch.rp 2.4 MB/s | 125 kB 00:00 2024-08-17T16:48:54.562 INFO:teuthology.orchestra.run.smithi152.stdout:(10/34): python3-urllib3-1.26.5-6.el9.noarch.rp 3.6 MB/s | 215 kB 00:00 2024-08-17T16:48:54.621 INFO:teuthology.orchestra.run.smithi152.stdout:(11/34): python3-pycparser-2.20-6.el9.noarch.rp 703 kB/s | 135 kB 00:00 2024-08-17T16:48:54.662 INFO:teuthology.orchestra.run.smithi152.stdout:(12/34): python3-cryptography-36.0.1-4.el9.x86_ 2.8 MB/s | 1.2 MB 00:00 2024-08-17T16:48:54.746 INFO:teuthology.orchestra.run.smithi152.stdout:(13/34): python3-mako-1.1.4-6.el9.noarch.rpm 936 kB/s | 172 kB 00:00 2024-08-17T16:48:54.771 INFO:teuthology.orchestra.run.smithi152.stdout:(14/34): python3-markupsafe-1.1.1-12.el9.x86_64 232 kB/s | 35 kB 00:00 2024-08-17T16:48:54.822 INFO:teuthology.orchestra.run.smithi152.stdout:(15/34): python3-packaging-20.9-5.el9.noarch.rp 575 kB/s | 77 kB 00:00 2024-08-17T16:48:54.852 INFO:teuthology.orchestra.run.smithi142.stdout:(1/34): ceph-17.2.7-1404.gf2c39d4d.el9.x86_64.r 33 kB/s | 6.5 kB 00:00 2024-08-17T16:48:54.897 INFO:teuthology.orchestra.run.smithi152.stdout:(16/34): python3-pytz-2021.1-5.el9.noarch.rpm 341 kB/s | 51 kB 00:00 2024-08-17T16:48:54.956 INFO:teuthology.orchestra.run.smithi152.stdout:(17/34): python3-toml-0.10.2-6.el9.noarch.rpm 261 kB/s | 42 kB 00:00 2024-08-17T16:48:55.006 INFO:teuthology.orchestra.run.smithi152.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 273 kB/s | 43 kB 00:00 2024-08-17T16:48:55.018 INFO:teuthology.orchestra.run.smithi142.stdout:(2/34): ceph-mds-17.2.7-1404.gf2c39d4d.el9.x86_ 5.7 MB/s | 2.1 MB 00:00 2024-08-17T16:48:55.056 INFO:teuthology.orchestra.run.smithi152.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 443 kB/s | 11 kB 00:00 2024-08-17T16:48:55.060 INFO:teuthology.orchestra.run.smithi142.stdout:(3/34): ceph-mgr-17.2.7-1404.gf2c39d4d.el9.x86_ 3.5 MB/s | 1.4 MB 00:00 2024-08-17T16:48:55.081 INFO:teuthology.orchestra.run.smithi152.stdout:(20/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 944 kB/s | 172 kB 00:00 2024-08-17T16:48:55.107 INFO:teuthology.orchestra.run.smithi152.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 367 kB/s | 18 kB 00:00 2024-08-17T16:48:55.132 INFO:teuthology.orchestra.run.smithi152.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-08-17T16:48:55.152 INFO:teuthology.orchestra.run.smithi142.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1404.gf2c3 2.6 MB/s | 240 kB 00:00 2024-08-17T16:48:55.158 INFO:teuthology.orchestra.run.smithi152.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-08-17T16:48:55.183 INFO:teuthology.orchestra.run.smithi152.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 391 kB/s | 20 kB 00:00 2024-08-17T16:48:55.208 INFO:teuthology.orchestra.run.smithi152.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 919 kB/s | 46 kB 00:00 2024-08-17T16:48:55.210 INFO:teuthology.orchestra.run.smithi142.stdout:(5/34): ceph-mon-17.2.7-1404.gf2c39d4d.el9.x86_ 11 MB/s | 3.9 MB 00:00 2024-08-17T16:48:55.233 INFO:teuthology.orchestra.run.smithi152.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-17T16:48:55.258 INFO:teuthology.orchestra.run.smithi152.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-08-17T16:48:55.284 INFO:teuthology.orchestra.run.smithi152.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 326 kB/s | 16 kB 00:00 2024-08-17T16:48:55.309 INFO:teuthology.orchestra.run.smithi152.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-08-17T16:48:55.319 INFO:teuthology.orchestra.run.smithi142.stdout:(6/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.5 MB/s | 253 kB 00:00 2024-08-17T16:48:55.334 INFO:teuthology.orchestra.run.smithi152.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 713 kB/s | 36 kB 00:00 2024-08-17T16:48:55.361 INFO:teuthology.orchestra.run.smithi142.stdout:(7/34): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-08-17T16:48:55.391 INFO:teuthology.orchestra.run.smithi152.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.5 MB/s | 272 kB 00:00 2024-08-17T16:48:55.403 INFO:teuthology.orchestra.run.smithi142.stdout:(8/34): python3-cryptography-36.0.1-4.el9.x86_6 6.4 MB/s | 1.2 MB 00:00 2024-08-17T16:48:55.418 INFO:teuthology.orchestra.run.smithi152.stdout:(32/34): python3-webob-1.8.7-6.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-08-17T16:48:55.420 INFO:teuthology.orchestra.run.smithi142.stdout:(9/34): python3-pycparser-2.20-6.el9.noarch.rpm 2.3 MB/s | 135 kB 00:00 2024-08-17T16:48:55.443 INFO:teuthology.orchestra.run.smithi152.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 385 kB/s | 20 kB 00:00 2024-08-17T16:48:55.445 INFO:teuthology.orchestra.run.smithi142.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 2.9 MB/s | 125 kB 00:00 2024-08-17T16:48:55.470 INFO:teuthology.orchestra.run.smithi142.stdout:(11/34): python3-urllib3-1.26.5-6.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-08-17T16:48:55.477 INFO:teuthology.orchestra.run.smithi152.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.9 MB/s | 427 kB 00:00 2024-08-17T16:48:55.481 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:55.481 INFO:teuthology.orchestra.run.smithi152.stdout:Total 10 MB/s | 27 MB 00:02 2024-08-17T16:48:55.602 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:48:55.621 INFO:teuthology.orchestra.run.smithi142.stdout:(12/34): ceph-osd-17.2.7-1404.gf2c39d4d.el9.x86 25 MB/s | 15 MB 00:00 2024-08-17T16:48:55.640 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:48:55.640 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:48:55.804 INFO:teuthology.orchestra.run.smithi142.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 104 kB/s | 35 kB 00:00 2024-08-17T16:48:55.971 INFO:teuthology.orchestra.run.smithi142.stdout:(14/34): python3-packaging-20.9-5.el9.noarch.rp 220 kB/s | 77 kB 00:00 2024-08-17T16:48:55.997 INFO:teuthology.orchestra.run.smithi142.stdout:(15/34): python3-pytz-2021.1-5.el9.noarch.rpm 265 kB/s | 51 kB 00:00 2024-08-17T16:48:56.040 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:48:56.040 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:48:56.055 INFO:teuthology.orchestra.run.smithi142.stdout:(16/34): python3-mako-1.1.4-6.el9.noarch.rpm 282 kB/s | 172 kB 00:00 2024-08-17T16:48:56.080 INFO:teuthology.orchestra.run.smithi142.stdout:(17/34): python3-toml-0.10.2-6.el9.noarch.rpm 383 kB/s | 42 kB 00:00 2024-08-17T16:48:56.131 INFO:teuthology.orchestra.run.smithi142.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 325 kB/s | 43 kB 00:00 2024-08-17T16:48:56.181 INFO:teuthology.orchestra.run.smithi142.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 477 kB/s | 11 kB 00:00 2024-08-17T16:48:56.206 INFO:teuthology.orchestra.run.smithi142.stdout:(20/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-08-17T16:48:56.231 INFO:teuthology.orchestra.run.smithi142.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 369 kB/s | 18 kB 00:00 2024-08-17T16:48:56.257 INFO:teuthology.orchestra.run.smithi142.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-08-17T16:48:56.282 INFO:teuthology.orchestra.run.smithi142.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-08-17T16:48:56.308 INFO:teuthology.orchestra.run.smithi142.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 390 kB/s | 20 kB 00:00 2024-08-17T16:48:56.333 INFO:teuthology.orchestra.run.smithi142.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 916 kB/s | 46 kB 00:00 2024-08-17T16:48:56.358 INFO:teuthology.orchestra.run.smithi142.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-17T16:48:56.386 INFO:teuthology.orchestra.run.smithi142.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-08-17T16:48:56.408 INFO:teuthology.orchestra.run.smithi142.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 327 kB/s | 16 kB 00:00 2024-08-17T16:48:56.478 INFO:teuthology.orchestra.run.smithi142.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 958 kB/s | 90 kB 00:00 2024-08-17T16:48:56.501 INFO:teuthology.orchestra.run.smithi142.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 389 kB/s | 36 kB 00:00 2024-08-17T16:48:56.526 INFO:teuthology.orchestra.run.smithi142.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-08-17T16:48:56.551 INFO:teuthology.orchestra.run.smithi142.stdout:(32/34): python3-webob-1.8.7-6.el9.noarch.rpm 3.1 MB/s | 230 kB 00:00 2024-08-17T16:48:56.576 INFO:teuthology.orchestra.run.smithi142.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 400 kB/s | 20 kB 00:00 2024-08-17T16:48:56.602 INFO:teuthology.orchestra.run.smithi142.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.1 MB/s | 427 kB 00:00 2024-08-17T16:48:56.606 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:48:56.606 INFO:teuthology.orchestra.run.smithi142.stdout:Total 10 MB/s | 27 MB 00:02 2024-08-17T16:48:56.731 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:48:56.768 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:48:56.768 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:48:57.126 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:48:57.127 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:48:57.232 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:48:57.297 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-08-17T16:48:57.329 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-08-17T16:48:57.362 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-08-17T16:48:57.455 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-08-17T16:48:57.533 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-08-17T16:48:57.572 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/34 2024-08-17T16:48:57.608 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-08-17T16:48:57.637 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-08-17T16:48:57.672 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-08-17T16:48:57.706 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-08-17T16:48:57.746 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-08-17T16:48:57.781 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-08-17T16:48:57.808 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-08-17T16:48:57.856 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-08-17T16:48:57.888 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T16:48:57.970 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-08-17T16:48:58.117 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-08-17T16:48:58.196 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-08-17T16:48:58.249 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-08-17T16:48:58.288 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-08-17T16:48:58.332 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:48:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-08-17T16:48:58.393 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-08-17T16:48:58.396 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-08-17T16:48:58.428 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-08-17T16:48:58.461 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-08-17T16:48:58.563 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-08-17T16:48:58.584 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-08-17T16:48:58.631 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-08-17T16:48:58.634 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-08-17T16:48:58.680 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/34 2024-08-17T16:48:58.712 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-08-17T16:48:58.715 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-08-17T16:48:58.745 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-08-17T16:48:58.779 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-08-17T16:48:58.800 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-08-17T16:48:58.813 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-08-17T16:48:58.834 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-08-17T16:48:58.853 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-08-17T16:48:58.888 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-08-17T16:48:58.923 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-08-17T16:48:58.948 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-08-17T16:48:58.970 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-08-17T16:48:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T16:48:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-08-17T16:48:59.112 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 29/34 2024-08-17T16:48:59.136 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T16:48:59.161 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T16:48:59.161 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:48:59.161 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T16:48:59.161 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:48:59.161 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:48:59.161 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:48:59.222 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-08-17T16:48:59.303 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-08-17T16:48:59.354 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-08-17T16:48:59.394 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-08-17T16:48:59.442 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-08-17T16:48:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-08-17T16:48:59.698 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-08-17T16:48:59.748 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-08-17T16:48:59.817 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-08-17T16:48:59.906 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-08-17T16:48:59.946 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-08-17T16:49:00.059 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-08-17T16:49:00.218 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 29/34 2024-08-17T16:49:00.249 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T16:49:00.275 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T16:49:00.275 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:00.275 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T16:49:00.275 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:49:00.275 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:49:00.275 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:00.546 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T16:49:00.579 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T16:49:00.579 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:00.579 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T16:49:00.579 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:49:00.579 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:49:00.579 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:00.947 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T16:49:00.971 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T16:49:00.971 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:00.972 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T16:49:00.972 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:49:00.972 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:49:00.972 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:01.173 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T16:49:01.196 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T16:49:01.196 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:01.196 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T16:49:01.196 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:49:01.196 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:49:01.197 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:01.261 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T16:49:01.634 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T16:49:01.663 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T16:49:01.663 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:01.663 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T16:49:01.663 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:49:01.663 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:49:01.663 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:02.035 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T16:49:02.058 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T16:49:02.059 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:02.059 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T16:49:02.059 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:49:02.059 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:49:02.059 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:02.260 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T16:49:02.284 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T16:49:02.284 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:02.284 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T16:49:02.284 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:49:02.284 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:49:02.284 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:02.348 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T16:49:02.927 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T16:49:02.927 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/34 2024-08-17T16:49:02.927 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/34 2024-08-17T16:49:02.927 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/34 2024-08-17T16:49:02.927 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 6/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-08-17T16:49:02.928 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-08-17T16:49:02.929 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-08-17T16:49:02.930 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-08-17T16:49:02.930 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-08-17T16:49:02.930 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-08-17T16:49:02.931 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-08-17T16:49:02.931 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-08-17T16:49:02.931 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-08-17T16:49:02.931 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 32/34 2024-08-17T16:49:02.931 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-08-17T16:49:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T16:49:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-17T16:49:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T16:49:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:03.876 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/34 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/34 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/34 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/34 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/34 2024-08-17T16:49:03.991 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 6/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-08-17T16:49:03.992 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-08-17T16:49:03.993 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-08-17T16:49:03.994 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-08-17T16:49:03.994 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-08-17T16:49:03.994 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 32/34 2024-08-17T16:49:03.994 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-08-17T16:49:04.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-base 2024-08-17T16:49:04.796 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:04.841 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T16:49:04.842 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T16:49:04.843 INFO:teuthology.orchestra.run.smithi142.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T16:49:04.844 INFO:teuthology.orchestra.run.smithi142.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T16:49:04.844 INFO:teuthology.orchestra.run.smithi142.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T16:49:04.844 INFO:teuthology.orchestra.run.smithi142.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T16:49:04.844 INFO:teuthology.orchestra.run.smithi142.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T16:49:04.844 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:04.844 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:04.881 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:04.923 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:04.924 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:04.924 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:04.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install cephadm 2024-08-17T16:49:05.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-base 2024-08-17T16:49:05.544 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:50 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:05.673 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 83 k 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:05.674 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:05.675 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 83 k 2024-08-17T16:49:05.675 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 375 k 2024-08-17T16:49:05.675 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:05.746 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:05.831 INFO:teuthology.orchestra.run.smithi142.stdout:Package ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:05.874 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:05.875 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:05.875 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:05.883 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm-17.2.7-1404.gf2c39d4d.el9.noarch.rpm 397 kB/s | 83 kB 00:00 2024-08-17T16:49:05.883 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:05.884 INFO:teuthology.orchestra.run.smithi152.stdout:Total 394 kB/s | 83 kB 00:00 2024-08-17T16:49:05.884 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:05.889 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:05.889 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:05.891 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:05.891 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:05.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install cephadm 2024-08-17T16:49:05.993 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:06.458 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:06.499 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:50 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:06.553 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:06.630 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:06.630 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:06.630 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout: cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 83 k 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 83 k 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 375 k 2024-08-17T16:49:06.631 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:06.839 INFO:teuthology.orchestra.run.smithi142.stdout:cephadm-17.2.7-1404.gf2c39d4d.el9.noarch.rpm 398 kB/s | 83 kB 00:00 2024-08-17T16:49:06.840 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:06.840 INFO:teuthology.orchestra.run.smithi142.stdout:Total 395 kB/s | 83 kB 00:00 2024-08-17T16:49:06.840 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:06.845 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:06.845 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:06.847 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:06.847 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:06.895 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:06.973 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:07.150 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:07.150 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:07.150 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:07.150 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:07.151 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:07.151 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:07.355 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:07.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T16:49:07.435 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:07.796 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:07.953 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:52 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:08.045 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T16:49:08.045 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:08.045 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:08.045 INFO:teuthology.orchestra.run.smithi142.stdout: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:08.045 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:08.045 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:08.083 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 137 k 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:08.084 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:08.085 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 137 k 2024-08-17T16:49:08.085 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 379 k 2024-08-17T16:49:08.085 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:08.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T16:49:08.305 INFO:teuthology.orchestra.run.smithi152.stdout:ceph-immutable-object-cache-17.2.7-1404.gf2c39d 624 kB/s | 137 kB 00:00 2024-08-17T16:49:08.306 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:08.306 INFO:teuthology.orchestra.run.smithi152.stdout:Total 619 kB/s | 137 kB 00:00 2024-08-17T16:49:08.306 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:08.314 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:08.314 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:08.371 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:08.371 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:08.524 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:08.614 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T16:49:08.640 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T16:49:08.640 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:08.640 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T16:49:08.640 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:08.823 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:52 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:08.952 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repo Size 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 137 k 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:08.953 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:08.954 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 137 k 2024-08-17T16:49:08.954 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 379 k 2024-08-17T16:49:08.954 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:09.178 INFO:teuthology.orchestra.run.smithi142.stdout:ceph-immutable-object-cache-17.2.7-1404.gf2c39d 612 kB/s | 137 kB 00:00 2024-08-17T16:49:09.179 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:09.179 INFO:teuthology.orchestra.run.smithi142.stdout:Total 607 kB/s | 137 kB 00:00 2024-08-17T16:49:09.180 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:09.187 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:09.187 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:09.245 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:09.245 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:09.419 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:09.496 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T16:49:09.508 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T16:49:09.508 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:09.508 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:09.508 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:09.508 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:09.508 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:09.522 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T16:49:09.522 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:09.522 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T16:49:09.522 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:09.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr 2024-08-17T16:49:10.326 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:55 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:10.416 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:10.452 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T16:49:10.452 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:10.452 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:10.452 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:10.452 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:10.453 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:10.456 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:10.458 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:10.458 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:10.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T16:49:10.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-mgr 2024-08-17T16:49:11.093 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:56 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:11.221 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 3.4 M 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:49:11.222 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 24 k 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 15 k 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout:Installing weak dependencies: 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout:Install 7 Packages 2024-08-17T16:49:11.223 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:11.224 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 3.7 M 2024-08-17T16:49:11.224 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 80 M 2024-08-17T16:49:11.224 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:11.260 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:55 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:11.348 INFO:teuthology.orchestra.run.smithi142.stdout:Package ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:11.390 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:11.391 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:11.391 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:11.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T16:49:11.617 INFO:teuthology.orchestra.run.smithi152.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1404.gf2c 127 kB/s | 24 kB 00:00 2024-08-17T16:49:11.642 INFO:teuthology.orchestra.run.smithi152.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1404.gf2c3 67 kB/s | 15 kB 00:00 2024-08-17T16:49:11.726 INFO:teuthology.orchestra.run.smithi152.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 83 kB/s | 9.0 kB 00:00 2024-08-17T16:49:11.760 INFO:teuthology.orchestra.run.smithi152.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 987 kB/s | 31 kB 00:00 2024-08-17T16:49:11.785 INFO:teuthology.orchestra.run.smithi152.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 287 kB/s | 41 kB 00:00 2024-08-17T16:49:11.818 INFO:teuthology.orchestra.run.smithi152.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.2 MB/s | 188 kB 00:00 2024-08-17T16:49:11.868 INFO:teuthology.orchestra.run.smithi152.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1404.gf2c39d4d 7.6 MB/s | 3.4 MB 00:00 2024-08-17T16:49:11.872 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:11.872 INFO:teuthology.orchestra.run.smithi152.stdout:Total 5.6 MB/s | 3.7 MB 00:00 2024-08-17T16:49:11.872 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:11.880 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:11.880 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:11.943 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:11.943 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:12.008 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:56 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:12.137 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 3.4 M 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 24 k 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 15 k 2024-08-17T16:49:12.138 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout:Installing weak dependencies: 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout:Install 7 Packages 2024-08-17T16:49:12.139 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:12.140 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 3.7 M 2024-08-17T16:49:12.140 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 80 M 2024-08-17T16:49:12.140 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-08-17T16:49:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-08-17T16:49:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-08-17T16:49:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-08-17T16:49:12.444 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 5/7 2024-08-17T16:49:12.749 INFO:teuthology.orchestra.run.smithi142.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1404.gf2c 135 kB/s | 24 kB 00:00 2024-08-17T16:49:12.774 INFO:teuthology.orchestra.run.smithi142.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1404.gf2c3 71 kB/s | 15 kB 00:00 2024-08-17T16:49:12.858 INFO:teuthology.orchestra.run.smithi142.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 83 kB/s | 9.0 kB 00:00 2024-08-17T16:49:12.892 INFO:teuthology.orchestra.run.smithi142.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 986 kB/s | 31 kB 00:00 2024-08-17T16:49:12.917 INFO:teuthology.orchestra.run.smithi142.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 287 kB/s | 41 kB 00:00 2024-08-17T16:49:12.942 INFO:teuthology.orchestra.run.smithi142.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.7 MB/s | 188 kB 00:00 2024-08-17T16:49:13.026 INFO:teuthology.orchestra.run.smithi142.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1404.gf2c39d4d 7.3 MB/s | 3.4 MB 00:00 2024-08-17T16:49:13.029 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:13.030 INFO:teuthology.orchestra.run.smithi142.stdout:Total 4.1 MB/s | 3.7 MB 00:00 2024-08-17T16:49:13.030 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:13.038 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:13.038 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:13.102 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:13.102 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:13.405 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 6/7 2024-08-17T16:49:13.460 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-08-17T16:49:13.464 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T16:49:13.494 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-08-17T16:49:13.549 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-08-17T16:49:13.581 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-08-17T16:49:13.618 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 5/7 2024-08-17T16:49:14.522 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T16:49:14.522 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 1/7 2024-08-17T16:49:14.522 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 2/7 2024-08-17T16:49:14.523 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 3/7 2024-08-17T16:49:14.523 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-08-17T16:49:14.523 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-08-17T16:49:14.523 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-08-17T16:49:14.621 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 6/7 2024-08-17T16:49:14.655 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T16:49:14.782 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-08-17T16:49:14.782 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:14.782 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:14.782 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:14.782 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:14.783 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:14.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T16:49:15.533 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:00 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 1/7 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 2/7 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 3/7 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-08-17T16:49:15.658 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-08-17T16:49:15.662 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 7.4 M 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T16:49:15.663 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout:Install 12 Packages 2024-08-17T16:49:15.664 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:15.665 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 38 M 2024-08-17T16:49:15.665 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 209 M 2024-08-17T16:49:15.665 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:15.935 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-08-17T16:49:15.935 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:15.936 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:16.076 INFO:teuthology.orchestra.run.smithi152.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 904 kB/s | 195 kB 00:00 2024-08-17T16:49:16.135 INFO:teuthology.orchestra.run.smithi152.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.9 MB/s | 803 kB 00:00 2024-08-17T16:49:16.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T16:49:16.160 INFO:teuthology.orchestra.run.smithi152.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 392 kB/s | 33 kB 00:00 2024-08-17T16:49:16.219 INFO:teuthology.orchestra.run.smithi152.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 311 kB/s | 18 kB 00:00 2024-08-17T16:49:16.336 INFO:teuthology.orchestra.run.smithi152.stdout:(5/12): ceph-mgr-diskprediction-local-17.2.7-14 16 MB/s | 7.4 MB 00:00 2024-08-17T16:49:16.361 INFO:teuthology.orchestra.run.smithi152.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 248 kB/s | 35 kB 00:00 2024-08-17T16:49:16.428 INFO:teuthology.orchestra.run.smithi152.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 10 MB/s | 3.0 MB 00:00 2024-08-17T16:49:16.478 INFO:teuthology.orchestra.run.smithi152.stdout:(8/12): python3-devel-3.9.18-3.el9.x86_64.rpm 2.0 MB/s | 244 kB 00:00 2024-08-17T16:49:16.570 INFO:teuthology.orchestra.run.smithi152.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.7 MB/s | 250 kB 00:00 2024-08-17T16:49:16.654 INFO:teuthology.orchestra.run.smithi152.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 23 MB/s | 5.1 MB 00:00 2024-08-17T16:49:16.708 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:00 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:16.836 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:16.837 INFO:teuthology.orchestra.run.smithi142.stdout:======================================================================================= 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout:======================================================================================= 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 7.4 M 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T16:49:16.838 INFO:teuthology.orchestra.run.smithi142.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout:======================================================================================= 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout:Install 12 Packages 2024-08-17T16:49:16.839 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:16.840 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 38 M 2024-08-17T16:49:16.840 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 209 M 2024-08-17T16:49:16.840 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:17.079 INFO:teuthology.orchestra.run.smithi152.stdout:(11/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 6.4 MB/s | 4.7 MB 00:00 2024-08-17T16:49:17.312 INFO:teuthology.orchestra.run.smithi152.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 22 MB/s | 16 MB 00:00 2024-08-17T16:49:17.313 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:17.314 INFO:teuthology.orchestra.run.smithi152.stdout:Total 23 MB/s | 38 MB 00:01 2024-08-17T16:49:17.492 INFO:teuthology.orchestra.run.smithi142.stdout:(1/12): ceph-mgr-diskprediction-local-17.2.7-14 15 MB/s | 7.4 MB 00:00 2024-08-17T16:49:17.665 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:17.691 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:17.692 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:17.742 INFO:teuthology.orchestra.run.smithi142.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 131 kB/s | 33 kB 00:00 2024-08-17T16:49:17.876 INFO:teuthology.orchestra.run.smithi142.stdout:(3/12): libquadmath-11.4.1-3.el9.x86_64.rpm 222 kB/s | 195 kB 00:00 2024-08-17T16:49:17.967 INFO:teuthology.orchestra.run.smithi142.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 199 kB/s | 18 kB 00:00 2024-08-17T16:49:18.026 INFO:teuthology.orchestra.run.smithi142.stdout:(5/12): libgfortran-11.4.1-3.el9.x86_64.rpm 780 kB/s | 803 kB 00:01 2024-08-17T16:49:18.051 INFO:teuthology.orchestra.run.smithi142.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 421 kB/s | 35 kB 00:00 2024-08-17T16:49:18.102 INFO:teuthology.orchestra.run.smithi142.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 8.3 MB/s | 3.0 MB 00:00 2024-08-17T16:49:18.152 INFO:teuthology.orchestra.run.smithi142.stdout:(8/12): python3-devel-3.9.18-3.el9.x86_64.rpm 2.4 MB/s | 244 kB 00:00 2024-08-17T16:49:18.216 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:18.217 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:18.227 INFO:teuthology.orchestra.run.smithi142.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 3.3 MB/s | 250 kB 00:00 2024-08-17T16:49:18.369 INFO:teuthology.orchestra.run.smithi142.stdout:(10/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 14 MB/s | 4.7 MB 00:00 2024-08-17T16:49:18.444 INFO:teuthology.orchestra.run.smithi142.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 15 MB/s | 5.1 MB 00:00 2024-08-17T16:49:18.669 INFO:teuthology.orchestra.run.smithi142.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 37 MB/s | 16 MB 00:00 2024-08-17T16:49:18.670 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:18.671 INFO:teuthology.orchestra.run.smithi142.stdout:Total 21 MB/s | 38 MB 00:01 2024-08-17T16:49:18.971 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:19.049 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T16:49:19.056 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:19.056 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:19.103 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:49:19.201 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:49:19.236 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T16:49:19.412 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T16:49:19.443 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T16:49:19.533 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T16:49:19.573 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:19.573 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:20.179 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T16:49:20.297 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T16:49:20.329 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:20.400 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T16:49:20.452 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:49:20.551 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:49:20.586 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T16:49:20.755 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T16:49:20.784 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T16:49:20.872 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T16:49:21.530 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T16:49:21.541 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:49:21.656 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T16:49:22.586 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T16:49:22.625 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T16:49:22.905 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:49:23.938 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T16:49:23.975 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T16:49:25.236 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T16:49:25.236 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 1/12 2024-08-17T16:49:25.236 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:49:25.236 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:49:25.236 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T16:49:25.236 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T16:49:25.237 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T16:49:25.237 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T16:49:25.237 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T16:49:25.237 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T16:49:25.237 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:49:25.237 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T16:49:25.633 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T16:49:25.634 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T16:49:25.635 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T16:49:25.635 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T16:49:25.635 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T16:49:25.635 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:25.635 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:26.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-rook 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 1/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T16:49:26.476 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T16:49:26.477 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T16:49:26.477 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:49:26.477 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T16:49:26.593 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:11 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:26.722 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 47 k 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T16:49:26.724 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:26.725 INFO:teuthology.orchestra.run.smithi152.stdout:Install 13 Packages 2024-08-17T16:49:26.726 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:26.726 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.2 M 2024-08-17T16:49:26.726 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 26 M 2024-08-17T16:49:26.727 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:26.877 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T16:49:26.878 INFO:teuthology.orchestra.run.smithi142.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T16:49:26.879 INFO:teuthology.orchestra.run.smithi142.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T16:49:26.879 INFO:teuthology.orchestra.run.smithi142.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T16:49:26.879 INFO:teuthology.orchestra.run.smithi142.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T16:49:26.879 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:26.879 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:27.032 INFO:teuthology.orchestra.run.smithi152.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 208 kB/s | 19 kB 00:00 2024-08-17T16:49:27.049 INFO:teuthology.orchestra.run.smithi152.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 235 kB/s | 26 kB 00:00 2024-08-17T16:49:27.132 INFO:teuthology.orchestra.run.smithi152.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.2 MB/s | 222 kB 00:00 2024-08-17T16:49:27.149 INFO:teuthology.orchestra.run.smithi152.stdout:(4/13): ceph-mgr-rook-17.2.7-1404.gf2c39d4d.el9 226 kB/s | 47 kB 00:00 2024-08-17T16:49:27.174 INFO:teuthology.orchestra.run.smithi152.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-08-17T16:49:27.208 INFO:teuthology.orchestra.run.smithi152.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.6 MB/s | 279 kB 00:00 2024-08-17T16:49:27.250 INFO:teuthology.orchestra.run.smithi152.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 533 kB/s | 54 kB 00:00 2024-08-17T16:49:27.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-mgr-rook 2024-08-17T16:49:27.292 INFO:teuthology.orchestra.run.smithi152.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 275 kB/s | 32 kB 00:00 2024-08-17T16:49:27.334 INFO:teuthology.orchestra.run.smithi152.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 139 kB/s | 14 kB 00:00 2024-08-17T16:49:27.367 INFO:teuthology.orchestra.run.smithi152.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.9 MB/s | 59 kB 00:00 2024-08-17T16:49:27.392 INFO:teuthology.orchestra.run.smithi152.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.5 MB/s | 90 kB 00:00 2024-08-17T16:49:27.418 INFO:teuthology.orchestra.run.smithi152.stdout:(12/13): python3-google-auth-2.33.0-1.el9.noarc 1.5 MB/s | 220 kB 00:00 2024-08-17T16:49:27.460 INFO:teuthology.orchestra.run.smithi152.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 7.2 MB/s | 1.0 MB 00:00 2024-08-17T16:49:27.462 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:27.462 INFO:teuthology.orchestra.run.smithi152.stdout:Total 3.0 MB/s | 2.2 MB 00:00 2024-08-17T16:49:27.525 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:27.540 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:27.540 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:27.626 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:27.627 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:27.815 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:11 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:27.943 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-rook noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 47 k 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T16:49:27.945 INFO:teuthology.orchestra.run.smithi142.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout:Install 13 Packages 2024-08-17T16:49:27.946 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:27.947 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 2.2 M 2024-08-17T16:49:27.947 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 26 M 2024-08-17T16:49:27.947 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:28.131 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:28.291 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T16:49:28.352 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T16:49:28.400 INFO:teuthology.orchestra.run.smithi142.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 191 kB/s | 26 kB 00:00 2024-08-17T16:49:28.415 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T16:49:28.425 INFO:teuthology.orchestra.run.smithi142.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 120 kB/s | 19 kB 00:00 2024-08-17T16:49:28.450 INFO:teuthology.orchestra.run.smithi142.stdout:(3/13): ceph-mgr-rook-17.2.7-1404.gf2c39d4d.el9 257 kB/s | 47 kB 00:00 2024-08-17T16:49:28.451 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T16:49:28.493 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T16:49:28.534 INFO:teuthology.orchestra.run.smithi142.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 159 kB 00:00 2024-08-17T16:49:28.559 INFO:teuthology.orchestra.run.smithi142.stdout:(5/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.4 MB/s | 222 kB 00:00 2024-08-17T16:49:28.595 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T16:49:28.609 INFO:teuthology.orchestra.run.smithi142.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 712 kB/s | 54 kB 00:00 2024-08-17T16:49:28.660 INFO:teuthology.orchestra.run.smithi142.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 321 kB/s | 32 kB 00:00 2024-08-17T16:49:28.685 INFO:teuthology.orchestra.run.smithi142.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.2 MB/s | 279 kB 00:00 2024-08-17T16:49:28.703 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T16:49:28.710 INFO:teuthology.orchestra.run.smithi142.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 144 kB/s | 14 kB 00:00 2024-08-17T16:49:28.758 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T16:49:28.760 INFO:teuthology.orchestra.run.smithi142.stdout:(10/13): python3-google-auth-2.33.0-1.el9.noarc 2.2 MB/s | 220 kB 00:00 2024-08-17T16:49:28.785 INFO:teuthology.orchestra.run.smithi142.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-08-17T16:49:28.810 INFO:teuthology.orchestra.run.smithi142.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-08-17T16:49:28.853 INFO:teuthology.orchestra.run.smithi142.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.1 MB/s | 1.0 MB 00:00 2024-08-17T16:49:28.855 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:28.855 INFO:teuthology.orchestra.run.smithi142.stdout:Total 2.4 MB/s | 2.2 MB 00:00 2024-08-17T16:49:28.919 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:28.934 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:28.934 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:29.019 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:29.020 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:29.474 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T16:49:29.529 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T16:49:29.540 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:29.588 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T16:49:29.650 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T16:49:29.683 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T16:49:29.706 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T16:49:29.762 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T16:49:29.817 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T16:49:29.855 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T16:49:29.896 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T16:49:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T16:49:30.114 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T16:49:30.163 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T16:49:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T16:49:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T16:49:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T16:49:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T16:49:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T16:49:30.824 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T16:49:30.824 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:30.824 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:30.824 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:30.824 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T16:49:30.824 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:30.825 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:30.871 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T16:49:30.924 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T16:49:30.957 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T16:49:31.003 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T16:49:31.027 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T16:49:31.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T16:49:31.631 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:16 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:31.761 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 124 k 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T16:49:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2024-08-17T16:49:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:31.764 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 7.0 M 2024-08-17T16:49:31.764 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 33 M 2024-08-17T16:49:31.764 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:31.803 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T16:49:31.803 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/13 2024-08-17T16:49:31.803 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T16:49:31.803 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T16:49:31.804 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T16:49:32.133 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T16:49:32.133 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:32.133 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:32.133 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:32.133 INFO:teuthology.orchestra.run.smithi142.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T16:49:32.133 INFO:teuthology.orchestra.run.smithi142.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:32.134 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.6 MB/s | 248 kB 00:00 2024-08-17T16:49:32.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T16:49:32.473 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): ceph-mgr-cephadm-17.2.7-1404.gf2c39d4d.e 559 kB/s | 124 kB 00:00 2024-08-17T16:49:32.540 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 4.0 MB/s | 548 kB 00:00 2024-08-17T16:49:32.567 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 617 kB/s | 58 kB 00:00 2024-08-17T16:49:32.592 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.4 MB/s | 74 kB 00:00 2024-08-17T16:49:32.668 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-08-17T16:49:32.670 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:32.671 INFO:teuthology.orchestra.run.smithi152.stdout:Total 7.7 MB/s | 7.0 MB 00:00 2024-08-17T16:49:32.741 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:32.752 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:32.753 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:32.851 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:32.854 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:32.963 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:16 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:33.092 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:33.092 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:33.092 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:49:33.092 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 124 k 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout:Install 6 Packages 2024-08-17T16:49:33.093 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:33.094 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 7.0 M 2024-08-17T16:49:33.094 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 33 M 2024-08-17T16:49:33.094 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:33.160 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:33.319 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T16:49:33.369 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T16:49:33.785 INFO:teuthology.orchestra.run.smithi142.stdout:(1/6): ceph-mgr-cephadm-17.2.7-1404.gf2c39d4d.e 623 kB/s | 124 kB 00:00 2024-08-17T16:49:33.810 INFO:teuthology.orchestra.run.smithi142.stdout:(2/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-08-17T16:49:33.892 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T16:49:33.902 INFO:teuthology.orchestra.run.smithi142.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 627 kB/s | 58 kB 00:00 2024-08-17T16:49:33.927 INFO:teuthology.orchestra.run.smithi142.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.9 MB/s | 74 kB 00:00 2024-08-17T16:49:33.953 INFO:teuthology.orchestra.run.smithi142.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 3.2 MB/s | 548 kB 00:00 2024-08-17T16:49:33.973 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T16:49:34.036 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T16:49:34.065 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T16:49:34.881 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T16:49:34.881 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/6 2024-08-17T16:49:34.881 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T16:49:34.881 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T16:49:34.882 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T16:49:34.882 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T16:49:35.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T16:49:35.146 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:35.146 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:35.147 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:35.186 INFO:teuthology.orchestra.run.smithi142.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.7 MB/s | 6.0 MB 00:01 2024-08-17T16:49:35.188 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:35.188 INFO:teuthology.orchestra.run.smithi142.stdout:Total 3.3 MB/s | 7.0 MB 00:02 2024-08-17T16:49:35.259 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:35.270 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:35.270 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:35.368 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:35.369 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:35.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-fuse 2024-08-17T16:49:35.678 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:35.852 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T16:49:35.905 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T16:49:35.956 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:20 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:36.085 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:36.085 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:36.085 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 813 k 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout:Install 2 Packages 2024-08-17T16:49:36.086 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:36.087 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 893 k 2024-08-17T16:49:36.087 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 2.6 M 2024-08-17T16:49:36.087 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:36.245 INFO:teuthology.orchestra.run.smithi152.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 664 kB/s | 80 kB 00:00 2024-08-17T16:49:36.423 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T16:49:36.445 INFO:teuthology.orchestra.run.smithi152.stdout:(2/2): ceph-fuse-17.2.7-1404.gf2c39d4d.el9.x86_ 2.5 MB/s | 813 kB 00:00 2024-08-17T16:49:36.445 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:36.446 INFO:teuthology.orchestra.run.smithi152.stdout:Total 2.4 MB/s | 893 kB 00:00 2024-08-17T16:49:36.457 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:36.479 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:36.479 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:36.509 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T16:49:36.558 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:36.558 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:36.573 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T16:49:36.602 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T16:49:36.754 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:36.890 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T16:49:36.961 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T16:49:37.390 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T16:49:37.390 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/6 2024-08-17T16:49:37.390 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T16:49:37.390 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T16:49:37.390 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T16:49:37.390 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T16:49:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T16:49:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/2 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T16:49:37.665 INFO:teuthology.orchestra.run.smithi142.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T16:49:37.666 INFO:teuthology.orchestra.run.smithi142.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T16:49:37.666 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:37.666 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:37.890 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T16:49:37.890 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:37.890 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:37.890 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T16:49:37.891 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:37.891 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:37.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install ceph-fuse 2024-08-17T16:49:38.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados-devel 2024-08-17T16:49:38.456 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:22 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:38.584 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:38.584 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repository Size 2024-08-17T16:49:38.584 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:38.584 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 813 k 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout:Installing dependencies: 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout:Install 2 Packages 2024-08-17T16:49:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:38.586 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 893 k 2024-08-17T16:49:38.586 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 2.6 M 2024-08-17T16:49:38.586 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:38.734 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:23 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:38.867 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:38.868 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:38.868 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-08-17T16:49:38.868 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:38.868 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:38.868 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 126 k 2024-08-17T16:49:38.868 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 126 k 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 448 k 2024-08-17T16:49:38.869 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:38.956 INFO:teuthology.orchestra.run.smithi142.stdout:(1/2): ceph-fuse-17.2.7-1404.gf2c39d4d.el9.x86_ 2.6 MB/s | 813 kB 00:00 2024-08-17T16:49:39.073 INFO:teuthology.orchestra.run.smithi142.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 188 kB/s | 80 kB 00:00 2024-08-17T16:49:39.074 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:39.074 INFO:teuthology.orchestra.run.smithi142.stdout:Total 1.8 MB/s | 893 kB 00:00 2024-08-17T16:49:39.086 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:39.109 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:39.109 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:39.117 INFO:teuthology.orchestra.run.smithi152.stdout:librados-devel-17.2.7-1404.gf2c39d4d.el9.x86_64 509 kB/s | 126 kB 00:00 2024-08-17T16:49:39.118 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:39.119 INFO:teuthology.orchestra.run.smithi152.stdout:Total 505 kB/s | 126 kB 00:00 2024-08-17T16:49:39.119 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:39.124 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:39.124 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:39.188 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:39.189 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:39.189 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:39.189 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:39.337 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:39.371 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:39.432 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:39.503 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T16:49:39.565 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T16:49:39.846 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:40.095 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:40.096 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:40.096 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:40.096 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:40.096 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:40.096 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:40.291 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T16:49:40.291 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/2 2024-08-17T16:49:40.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs2 2024-08-17T16:49:40.548 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T16:49:40.548 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:40.548 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:40.548 INFO:teuthology.orchestra.run.smithi142.stdout: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T16:49:40.549 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:40.549 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:40.830 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install librados-devel 2024-08-17T16:49:40.938 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:25 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:41.027 INFO:teuthology.orchestra.run.smithi152.stdout:Package libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:41.071 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:41.072 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:41.072 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:41.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs-devel 2024-08-17T16:49:41.390 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:25 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:41.523 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repo Size 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout: librados-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 126 k 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:41.524 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:41.525 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 126 k 2024-08-17T16:49:41.525 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 448 k 2024-08-17T16:49:41.525 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:41.704 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:26 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:41.756 INFO:teuthology.orchestra.run.smithi142.stdout:librados-devel-17.2.7-1404.gf2c39d4d.el9.x86_64 547 kB/s | 126 kB 00:00 2024-08-17T16:49:41.756 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:41.757 INFO:teuthology.orchestra.run.smithi142.stdout:Total 543 kB/s | 126 kB 00:00 2024-08-17T16:49:41.757 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:41.762 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:41.762 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:41.826 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:41.826 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:41.834 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:41.834 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 25 k 2024-08-17T16:49:41.835 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 110 k 2024-08-17T16:49:41.836 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:41.977 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:42.000 INFO:teuthology.orchestra.run.smithi152.stdout:libcephfs-devel-17.2.7-1404.gf2c39d4d.el9.x86_6 151 kB/s | 25 kB 00:00 2024-08-17T16:49:42.001 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:42.001 INFO:teuthology.orchestra.run.smithi152.stdout:Total 150 kB/s | 25 kB 00:00 2024-08-17T16:49:42.001 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:42.004 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:42.005 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:42.006 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:42.006 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:42.094 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:42.531 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:42.786 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:42.786 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:42.786 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:42.786 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:42.786 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:42.786 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:42.798 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:42.798 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:42.798 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:42.798 INFO:teuthology.orchestra.run.smithi142.stdout: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:42.798 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:42.798 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:42.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados2 2024-08-17T16:49:43.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install libcephfs2 2024-08-17T16:49:43.516 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:43.602 INFO:teuthology.orchestra.run.smithi152.stdout:Package librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:43.612 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:27 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:43.644 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:43.645 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:43.645 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:43.700 INFO:teuthology.orchestra.run.smithi142.stdout:Package libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:43.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librbd1 2024-08-17T16:49:43.744 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:43.745 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:43.746 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:43.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install libcephfs-devel 2024-08-17T16:49:44.275 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:29 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:44.362 INFO:teuthology.orchestra.run.smithi152.stdout:Package librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:44.383 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:44.405 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:44.406 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:44.406 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:44.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rados 2024-08-17T16:49:44.512 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:44.512 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:44.512 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repo Size 2024-08-17T16:49:44.512 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout: libcephfs-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:44.513 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:44.514 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 25 k 2024-08-17T16:49:44.514 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 110 k 2024-08-17T16:49:44.514 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:44.682 INFO:teuthology.orchestra.run.smithi142.stdout:libcephfs-devel-17.2.7-1404.gf2c39d4d.el9.x86_6 148 kB/s | 25 kB 00:00 2024-08-17T16:49:44.683 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:44.683 INFO:teuthology.orchestra.run.smithi142.stdout:Total 147 kB/s | 25 kB 00:00 2024-08-17T16:49:44.683 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:44.686 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:44.686 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:44.688 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:44.688 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:44.766 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:44.871 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:45.033 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:45.119 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:45.161 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:45.162 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:45.162 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:45.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rgw 2024-08-17T16:49:45.230 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:45.446 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:45.446 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:45.446 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:45.446 INFO:teuthology.orchestra.run.smithi142.stdout: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:45.446 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:45.446 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:45.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install librados2 2024-08-17T16:49:45.781 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:45.867 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:45.909 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:45.910 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:45.910 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:45.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-cephfs 2024-08-17T16:49:46.205 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:46.291 INFO:teuthology.orchestra.run.smithi142.stdout:Package librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:46.333 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:46.334 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:46.334 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:46.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install librbd1 2024-08-17T16:49:46.529 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:46.615 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:46.657 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:46.658 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:46.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rbd 2024-08-17T16:49:46.956 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:47.042 INFO:teuthology.orchestra.run.smithi142.stdout:Package librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:47.084 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:47.085 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:47.085 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:47.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install python3-rados 2024-08-17T16:49:47.277 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:47.362 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:47.404 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:47.405 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-08-17T16:49:47.405 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:47.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-fuse 2024-08-17T16:49:47.701 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:47.787 INFO:teuthology.orchestra.run.smithi142.stdout:Package python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:47.829 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:47.830 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:47.830 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:47.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install python3-rgw 2024-08-17T16:49:48.020 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:48.148 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:48.149 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:48.149 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-08-17T16:49:48.149 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 82 k 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 222 k 2024-08-17T16:49:48.150 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:48.373 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-fuse-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 367 kB/s | 82 kB 00:00 2024-08-17T16:49:48.374 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:48.374 INFO:teuthology.orchestra.run.smithi152.stdout:Total 365 kB/s | 82 kB 00:00 2024-08-17T16:49:48.374 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:48.382 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:48.382 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:48.444 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:48.444 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:48.449 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:48.535 INFO:teuthology.orchestra.run.smithi142.stdout:Package python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:48.577 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:48.578 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:48.578 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:48.588 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:48.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install python3-cephfs 2024-08-17T16:49:48.691 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:49.069 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:49.208 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:49.294 INFO:teuthology.orchestra.run.smithi142.stdout:Package python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:49.310 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:49.310 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:49.310 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:49.310 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:49.310 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:49.311 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:49.337 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:49.338 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:49.338 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:49.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install python3-rbd 2024-08-17T16:49:49.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-mirror 2024-08-17T16:49:49.955 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:50.041 INFO:teuthology.orchestra.run.smithi142.stdout:Package python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T16:49:50.083 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:50.084 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-08-17T16:49:50.084 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:50.084 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:35 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:50.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install rbd-fuse 2024-08-17T16:49:50.213 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:50.213 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:50.213 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-08-17T16:49:50.213 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:50.213 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:50.213 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T16:49:50.214 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:50.214 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:50.214 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:50.214 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:50.214 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:50.214 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.9 M 2024-08-17T16:49:50.215 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 12 M 2024-08-17T16:49:50.215 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:50.573 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-mirror-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 8.2 MB/s | 2.9 MB 00:00 2024-08-17T16:49:50.573 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:50.573 INFO:teuthology.orchestra.run.smithi152.stdout:Total 8.2 MB/s | 2.9 MB 00:00 2024-08-17T16:49:50.574 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:50.583 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:50.583 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:50.663 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:50.663 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:50.698 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:50.828 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:50.828 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:50.828 INFO:teuthology.orchestra.run.smithi142.stdout: Package Architecture Version Repository Size 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout: rbd-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 82 k 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:50.829 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:50.830 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 82 k 2024-08-17T16:49:50.830 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 222 k 2024-08-17T16:49:50.830 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:51.035 INFO:teuthology.orchestra.run.smithi142.stdout:rbd-fuse-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 398 kB/s | 82 kB 00:00 2024-08-17T16:49:51.036 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:51.036 INFO:teuthology.orchestra.run.smithi142.stdout:Total 396 kB/s | 82 kB 00:00 2024-08-17T16:49:51.036 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:51.042 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:51.043 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:51.076 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:51.105 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:51.106 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:51.157 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:51.157 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T16:49:51.157 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:49:51.157 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:49:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:51.246 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:51.366 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:51.733 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:51.987 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:51.987 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:51.987 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:51.987 INFO:teuthology.orchestra.run.smithi142.stdout: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:51.987 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:51.988 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:52.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install rbd-mirror 2024-08-17T16:49:52.215 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:52.215 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:52.215 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:52.216 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:52.216 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:52.216 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:52.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-nbd 2024-08-17T16:49:52.771 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:36 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:52.901 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:52.901 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout: Package Arch Version Repo Size 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout: rbd-mirror x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:52.902 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 2.9 M 2024-08-17T16:49:52.903 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 12 M 2024-08-17T16:49:52.903 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:53.018 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:38 ago on Sat 17 Aug 2024 04:48:15 PM UTC. 2024-08-17T16:49:53.149 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-08-17T16:49:53.149 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 165 k 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 165 k 2024-08-17T16:49:53.151 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 477 k 2024-08-17T16:49:53.151 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-08-17T16:49:53.254 INFO:teuthology.orchestra.run.smithi142.stdout:rbd-mirror-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 8.4 MB/s | 2.9 MB 00:00 2024-08-17T16:49:53.255 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:53.255 INFO:teuthology.orchestra.run.smithi142.stdout:Total 8.3 MB/s | 2.9 MB 00:00 2024-08-17T16:49:53.255 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:53.264 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:53.264 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:53.342 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:53.343 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:53.370 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-nbd-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 751 kB/s | 165 kB 00:00 2024-08-17T16:49:53.370 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:53.371 INFO:teuthology.orchestra.run.smithi152.stdout:Total 747 kB/s | 165 kB 00:00 2024-08-17T16:49:53.371 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-08-17T16:49:53.378 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-08-17T16:49:53.378 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-08-17T16:49:53.442 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-08-17T16:49:53.442 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-08-17T16:49:53.598 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-08-17T16:49:53.717 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:53.766 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:53.863 INFO:teuthology.orchestra.run.smithi142.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:49:53.863 INFO:teuthology.orchestra.run.smithi142.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T16:49:53.863 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:49:53.863 INFO:teuthology.orchestra.run.smithi142.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:49:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:54.130 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:54.385 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:54.385 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:54.385 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-08-17T16:49:54.385 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:54.385 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:49:54.386 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-08-17T16:49:54.638 DEBUG:teuthology.parallel:result is None 2024-08-17T16:49:54.921 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:54.922 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:54.922 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:54.922 INFO:teuthology.orchestra.run.smithi142.stdout: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:54.922 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:54.922 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:55.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum -y install rbd-nbd 2024-08-17T16:49:55.731 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:39 ago on Sat 17 Aug 2024 04:48:16 PM UTC. 2024-08-17T16:49:55.861 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout: Package Architecture Version Repository Size 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout:Installing: 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout: rbd-nbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 165 k 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction Summary 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout:================================================================================ 2024-08-17T16:49:55.862 INFO:teuthology.orchestra.run.smithi142.stdout:Install 1 Package 2024-08-17T16:49:55.863 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:55.863 INFO:teuthology.orchestra.run.smithi142.stdout:Total download size: 165 k 2024-08-17T16:49:55.863 INFO:teuthology.orchestra.run.smithi142.stdout:Installed size: 477 k 2024-08-17T16:49:55.863 INFO:teuthology.orchestra.run.smithi142.stdout:Downloading Packages: 2024-08-17T16:49:56.085 INFO:teuthology.orchestra.run.smithi142.stdout:rbd-nbd-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 742 kB/s | 165 kB 00:00 2024-08-17T16:49:56.086 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:49:56.086 INFO:teuthology.orchestra.run.smithi142.stdout:Total 738 kB/s | 165 kB 00:00 2024-08-17T16:49:56.086 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction check 2024-08-17T16:49:56.093 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction check succeeded. 2024-08-17T16:49:56.094 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction test 2024-08-17T16:49:56.158 INFO:teuthology.orchestra.run.smithi142.stdout:Transaction test succeeded. 2024-08-17T16:49:56.158 INFO:teuthology.orchestra.run.smithi142.stdout:Running transaction 2024-08-17T16:49:56.314 INFO:teuthology.orchestra.run.smithi142.stdout: Preparing : 1/1 2024-08-17T16:49:56.416 INFO:teuthology.orchestra.run.smithi142.stdout: Installing : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:57.087 INFO:teuthology.orchestra.run.smithi142.stdout: Verifying : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T16:49:57.087 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:57.087 INFO:teuthology.orchestra.run.smithi142.stdout:Installed: 2024-08-17T16:49:57.087 INFO:teuthology.orchestra.run.smithi142.stdout: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T16:49:57.088 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:49:57.088 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-08-17T16:49:57.340 DEBUG:teuthology.parallel:result is None 2024-08-17T16:49:57.340 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:49:57.340 INFO:teuthology.packaging:ref: None 2024-08-17T16:49:57.340 INFO:teuthology.packaging:tag: None 2024-08-17T16:49:57.341 INFO:teuthology.packaging:branch: quincy 2024-08-17T16:49:57.341 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:49:57.341 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T16:49:57.568 DEBUG:teuthology.orchestra.run.smithi142:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T16:49:57.590 INFO:teuthology.orchestra.run.smithi142.stdout:17.2.7-1404.gf2c39d4d.el9 2024-08-17T16:49:57.594 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1404.gf2c39d4d.el9 2024-08-17T16:49:57.594 INFO:teuthology.task.install:The correct ceph version 17.2.7-1404.gf2c39d4d is installed. 2024-08-17T16:49:57.596 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:49:57.596 INFO:teuthology.packaging:ref: None 2024-08-17T16:49:57.596 INFO:teuthology.packaging:tag: None 2024-08-17T16:49:57.596 INFO:teuthology.packaging:branch: quincy 2024-08-17T16:49:57.596 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:49:57.596 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T16:49:57.816 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T16:49:57.842 INFO:teuthology.orchestra.run.smithi152.stdout:17.2.7-1404.gf2c39d4d.el9 2024-08-17T16:49:57.842 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1404.gf2c39d4d.el9 2024-08-17T16:49:57.842 INFO:teuthology.task.install:The correct ceph version 17.2.7-1404.gf2c39d4d is installed. 2024-08-17T16:49:57.844 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-17T16:49:57.844 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:49:57.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T16:49:57.872 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:49:57.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T16:49:57.911 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-17T16:49:57.911 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:49:57.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T16:49:57.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T16:49:58.010 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:49:58.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T16:49:58.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T16:49:58.104 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-17T16:49:58.105 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:49:58.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T16:49:58.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T16:49:58.195 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:49:58.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T16:49:58.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T16:49:58.289 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-17T16:49:58.289 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:49:58.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T16:49:58.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T16:49:58.379 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:49:58.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T16:49:58.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T16:49:58.471 INFO:teuthology.run_tasks:Running task cephadm... 2024-08-17T16:49:58.563 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T16:49:58.564 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-08-17T16:49:58.564 INFO:tasks.cephadm:Cluster fsid is bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:49:58.564 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-08-17T16:49:58.565 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.142', 'mon.c': '[v2:172.21.15.142:3301,v1:172.21.15.142:6790]', 'mon.b': '172.21.15.152'} 2024-08-17T16:49:58.565 INFO:tasks.cephadm:First mon is mon.a on smithi142 2024-08-17T16:49:58.565 INFO:tasks.cephadm:First mgr is y 2024-08-17T16:49:58.565 INFO:tasks.cephadm:Normalizing hostnames... 2024-08-17T16:49:58.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo hostname $(hostname -s) 2024-08-17T16:49:58.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo hostname $(hostname -s) 2024-08-17T16:49:58.617 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-08-17T16:49:58.617 DEBUG:teuthology.orchestra.run.smithi142:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T16:49:58.779 INFO:teuthology.orchestra.run.smithi142.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Aug 17 16:49 /home/ubuntu/cephtest/cephadm 2024-08-17T16:49:58.779 DEBUG:teuthology.orchestra.run.smithi152:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T16:49:58.930 INFO:teuthology.orchestra.run.smithi152.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Aug 17 16:49 /home/ubuntu/cephtest/cephadm 2024-08-17T16:49:58.930 DEBUG:teuthology.orchestra.run.smithi142:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T16:49:58.947 DEBUG:teuthology.orchestra.run.smithi152:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T16:49:58.973 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-08-17T16:49:58.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-08-17T16:49:58.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-08-17T16:49:59.876 INFO:teuthology.orchestra.run.smithi142.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T16:49:59.908 INFO:teuthology.orchestra.run.smithi152.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T16:50:38.481 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T16:50:38.481 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph_version": "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)", 2024-08-17T16:50:38.481 INFO:teuthology.orchestra.run.smithi142.stdout: "image_id": "f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce", 2024-08-17T16:50:38.482 INFO:teuthology.orchestra.run.smithi142.stdout: "repo_digests": [ 2024-08-17T16:50:38.482 INFO:teuthology.orchestra.run.smithi142.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251" 2024-08-17T16:50:38.482 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T16:50:38.482 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph_version": "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)", 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout: "image_id": "f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce", 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout: "repo_digests": [ 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251" 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T16:50:43.352 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T16:50:43.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo mkdir -p /etc/ceph 2024-08-17T16:50:43.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo mkdir -p /etc/ceph 2024-08-17T16:50:43.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo chmod 777 /etc/ceph 2024-08-17T16:50:43.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 777 /etc/ceph 2024-08-17T16:50:43.498 INFO:tasks.cephadm:Writing seed config... 2024-08-17T16:50:43.499 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-08-17T16:50:43.499 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-08-17T16:50:43.499 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-08-17T16:50:43.499 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-08-17T16:50:43.499 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-08-17T16:50:43.500 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-08-17T16:50:43.500 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-08-17T16:50:43.500 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-08-17T16:50:43.500 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-08-17T16:50:43.500 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-08-17T16:50:43.500 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-08-17T16:50:43.501 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:50:43.501 DEBUG:teuthology.orchestra.run.smithi142:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-08-17T16:50:43.529 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = bcf463ee-5cb8-11ef-bccf-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-08-17T16:50:43.530 DEBUG:teuthology.orchestra.run.smithi142:mon.a> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service 2024-08-17T16:50:43.571 DEBUG:teuthology.orchestra.run.smithi142:mgr.y> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service 2024-08-17T16:50:43.614 INFO:tasks.cephadm:Bootstrapping... 2024-08-17T16:50:43.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.142 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:50:43.818 INFO:teuthology.orchestra.run.smithi142.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:50:43.818 INFO:teuthology.orchestra.run.smithi142.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'bcf463ee-5cb8-11ef-bccf-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.142', '--skip-admin-label'] 2024-08-17T16:50:43.844 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:50:43.844 INFO:teuthology.orchestra.run.smithi142.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-08-17T16:50:43.845 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying podman|docker is present... 2024-08-17T16:50:43.868 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:50:43.869 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying lvm2 is present... 2024-08-17T16:50:43.869 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying time synchronization is in place... 2024-08-17T16:50:43.877 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T16:50:43.877 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T16:50:43.883 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T16:50:43.883 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout inactive 2024-08-17T16:50:43.891 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout enabled 2024-08-17T16:50:43.897 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout active 2024-08-17T16:50:43.898 INFO:teuthology.orchestra.run.smithi142.stdout:Unit chronyd.service is enabled and running 2024-08-17T16:50:43.898 INFO:teuthology.orchestra.run.smithi142.stdout:Repeating the final host check... 2024-08-17T16:50:43.922 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:50:43.922 INFO:teuthology.orchestra.run.smithi142.stdout:podman (/bin/podman) version 5.1.2 is present 2024-08-17T16:50:43.922 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl is present 2024-08-17T16:50:43.922 INFO:teuthology.orchestra.run.smithi142.stdout:lvcreate is present 2024-08-17T16:50:43.928 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T16:50:43.928 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T16:50:43.935 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T16:50:43.935 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout inactive 2024-08-17T16:50:43.942 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout enabled 2024-08-17T16:50:43.949 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout active 2024-08-17T16:50:43.949 INFO:teuthology.orchestra.run.smithi142.stdout:Unit chronyd.service is enabled and running 2024-08-17T16:50:43.949 INFO:teuthology.orchestra.run.smithi142.stdout:Host looks OK 2024-08-17T16:50:43.949 INFO:teuthology.orchestra.run.smithi142.stdout:Cluster fsid: bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:43.949 INFO:teuthology.orchestra.run.smithi142.stdout:Acquiring lock 139691855659888 on /run/cephadm/bcf463ee-5cb8-11ef-bccf-c7b262605968.lock 2024-08-17T16:50:43.949 INFO:teuthology.orchestra.run.smithi142.stdout:Lock 139691855659888 acquired on /run/cephadm/bcf463ee-5cb8-11ef-bccf-c7b262605968.lock 2024-08-17T16:50:43.950 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying IP 172.21.15.142 port 3300 ... 2024-08-17T16:50:43.950 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying IP 172.21.15.142 port 6789 ... 2024-08-17T16:50:43.950 INFO:teuthology.orchestra.run.smithi142.stdout:Base mon IP(s) is [172.21.15.142:3300, 172.21.15.142:6789], mon addrv is [v2:172.21.15.142:3300,v1:172.21.15.142:6789] 2024-08-17T16:50:43.953 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.142 metric 100 2024-08-17T16:50:43.953 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.142 metric 100 2024-08-17T16:50:43.956 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-08-17T16:50:43.956 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-08-17T16:50:43.956 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1799sec hoplimit 64 pref medium 2024-08-17T16:50:43.959 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-08-17T16:50:43.959 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-08-17T16:50:43.959 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T16:50:43.959 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-08-17T16:50:43.959 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:6e0d/64 scope link 2024-08-17T16:50:43.959 INFO:teuthology.orchestra.run.smithi142.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T16:50:43.960 INFO:teuthology.orchestra.run.smithi142.stdout:Mon IP `172.21.15.142` is in CIDR network `172.21.0.0/20` 2024-08-17T16:50:43.960 INFO:teuthology.orchestra.run.smithi142.stdout:Mon IP `172.21.15.142` is in CIDR network `172.21.0.0/20` 2024-08-17T16:50:43.960 INFO:teuthology.orchestra.run.smithi142.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-08-17T16:50:43.960 INFO:teuthology.orchestra.run.smithi142.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-08-17T16:50:43.961 INFO:teuthology.orchestra.run.smithi142.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stdout f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stderr Getting image source signatures 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stderr Copying blob sha256:11a3dd1ce731c9342fb0d936b067311689a4bd18187bbc27c4f481cc3262ce2c 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stderr Copying config sha256:f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce 2024-08-17T16:50:44.431 INFO:teuthology.orchestra.run.smithi142.stdout:/bin/podman: stderr Writing manifest to image destination 2024-08-17T16:50:45.335 INFO:teuthology.orchestra.run.smithi142.stdout:ceph: stdout ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable) 2024-08-17T16:50:45.335 INFO:teuthology.orchestra.run.smithi142.stdout:Ceph version: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable) 2024-08-17T16:50:45.335 INFO:teuthology.orchestra.run.smithi142.stdout:Extracting ceph user uid/gid from container image... 2024-08-17T16:50:46.265 INFO:teuthology.orchestra.run.smithi142.stdout:stat: stdout 167 167 2024-08-17T16:50:46.265 INFO:teuthology.orchestra.run.smithi142.stdout:Creating initial keys... 2024-08-17T16:50:47.128 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph-authtool: stdout AQDm1MBmmxwNJRAAD20oqYgwZzRhRfQ58Nw+GA== 2024-08-17T16:50:47.959 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph-authtool: stdout AQDn1MBmenzkGhAAynwvwm85RD8GWJb7QdyEDw== 2024-08-17T16:50:48.897 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph-authtool: stdout AQDo1MBmX9k+EBAAGw3I7PjVpGTOSGy+HzIZAA== 2024-08-17T16:50:48.897 INFO:teuthology.orchestra.run.smithi142.stdout:Creating initial monmap... 2024-08-17T16:50:49.853 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T16:50:49.853 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:monmaptool for a [v2:172.21.15.142:3300,v1:172.21.15.142:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:setting min_mon_release = octopus 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/monmaptool: set fsid to bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:50:49.854 INFO:teuthology.orchestra.run.smithi142.stdout:Creating mon... 2024-08-17T16:50:50.784 INFO:teuthology.orchestra.run.smithi142.stdout:create mon.a on 2024-08-17T16:50:50.986 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-08-17T16:50:51.128 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-08-17T16:50:51.276 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968.target. 2024-08-17T16:50:51.277 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968.target. 2024-08-17T16:50:51.448 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a 2024-08-17T16:50:51.448 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Failed to reset failed state of unit ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service: Unit ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service not loaded. 2024-08-17T16:50:51.608 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968.target.wants/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service → /etc/systemd/system/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@.service. 2024-08-17T16:50:52.102 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:52 smithi142 ceph-mon[30711]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:50:52.116 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:50:52.116 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout disabled 2024-08-17T16:50:52.124 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:50:52.124 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout inactive 2024-08-17T16:50:52.124 INFO:teuthology.orchestra.run.smithi142.stdout:firewalld.service is not enabled 2024-08-17T16:50:52.124 INFO:teuthology.orchestra.run.smithi142.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T16:50:52.124 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for mon to start... 2024-08-17T16:50:52.125 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for mon... 2024-08-17T16:50:52.354 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:52 smithi142 bash[30672]: 5d343f076dfbad97637b67d74c4f8259a9cdb266670887f2bc69215040b95983 2024-08-17T16:50:52.354 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:52 smithi142 systemd[1]: Started Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:50:53.205 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:53 smithi142 ceph-mon[30711]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:50:53.206 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:53 smithi142 ceph-mon[30711]: monmap e1: 1 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:50:53.206 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:53 smithi142 ceph-mon[30711]: fsmap 2024-08-17T16:50:53.206 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:53 smithi142 ceph-mon[30711]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T16:50:53.206 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:53 smithi142 ceph-mon[30711]: mgrmap e1: no daemons active 2024-08-17T16:50:53.206 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:53 smithi142 ceph-mon[30711]: from='client.? 172.21.15.142:0/3596049325' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-17T16:50:53.209 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout cluster: 2024-08-17T16:50:53.209 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout id: bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout services: 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.574147s) 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout data: 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-08-17T16:50:53.210 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-08-17T16:50:53.211 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout pgs: 2024-08-17T16:50:53.211 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:53.211 INFO:teuthology.orchestra.run.smithi142.stdout:mon is available 2024-08-17T16:50:53.211 INFO:teuthology.orchestra.run.smithi142.stdout:Assimilating anything we can from ceph.conf... 2024-08-17T16:50:54.143 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:54.143 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T16:50:54.143 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout fsid = bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:54.143 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.142:3300,v1:172.21.15.142:6789] 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T16:50:54.144 INFO:teuthology.orchestra.run.smithi142.stdout:Generating new minimal ceph.conf... 2024-08-17T16:50:54.265 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:54 smithi142 ceph-mon[30711]: from='client.? 172.21.15.142:0/502669938' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T16:50:54.265 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:54 smithi142 ceph-mon[30711]: from='client.? 172.21.15.142:0/502669938' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-17T16:50:55.064 INFO:teuthology.orchestra.run.smithi142.stdout:Restarting the monitor... 2024-08-17T16:50:55.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 ceph-mon[30711]: from='client.? 172.21.15.142:0/606173018' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:55.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 systemd[1]: Stopping Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:50:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[30688]: 2024-08-17T16:50:55.323+0000 7f58ccdcf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T16:50:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[30688]: 2024-08-17T16:50:55.323+0000 7f58ccdcf640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-17T16:50:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 podman[31027]: 2024-08-17 16:50:55.477927493 +0000 UTC m=+0.248742852 container died 5d343f076dfbad97637b67d74c4f8259a9cdb266670887f2bc69215040b95983 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T16:50:56.098 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 podman[31027]: 2024-08-17 16:50:55.821624274 +0000 UTC m=+0.592439632 container cleanup 5d343f076dfbad97637b67d74c4f8259a9cdb266670887f2bc69215040b95983 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T16:50:56.099 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:55 smithi142 bash[31027]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a 2024-08-17T16:50:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 podman[31044]: 2024-08-17 16:50:56.097855422 +0000 UTC m=+0.621026891 container remove 5d343f076dfbad97637b67d74c4f8259a9cdb266670887f2bc69215040b95983 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-08-17T16:50:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T16:50:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 systemd[1]: Stopped Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:50:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 systemd[1]: Starting Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:50:56.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 podman[31144]: 2024-08-17 16:50:56.336346051 +0000 UTC m=+0.016849076 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph:quincy 2024-08-17T16:50:56.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 podman[31144]: 2024-08-17 16:50:56.457209442 +0000 UTC m=+0.137712453 container create 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T16:50:56.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 podman[31144]: 2024-08-17 16:50:56.659440621 +0000 UTC m=+0.339943639 container init 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T16:50:56.686 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 podman[31144]: 2024-08-17 16:50:56.663874434 +0000 UTC m=+0.344377448 container start 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4) 2024-08-17T16:50:56.686 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:50:56.686 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable), process ceph-mon, pid 7 2024-08-17T16:50:56.686 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: pidfile_write: ignore empty --pid-file 2024-08-17T16:50:56.686 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: load: jerasure load: lrc 2024-08-17T16:50:56.714 INFO:teuthology.orchestra.run.smithi142.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-08-17T16:50:56.943 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: RocksDB version: 6.15.5 2024-08-17T16:50:56.943 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-08-17T16:50:56.943 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Compile date Aug 17 2024 2024-08-17T16:50:56.943 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: DB SUMMARY 2024-08-17T16:50:56.943 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: DB Session ID: 2F5ZKKSFZYEEJZW6QMMX 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83706 ; 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:50:56.944 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.env: 0x560cd664c480 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.fs: Posix File System 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.info_log: 0x560cd7a5af80 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.statistics: (nil) 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.use_fsync: 0 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:50:56.945 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.db_log_dir: 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-08-17T16:50:56.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:50:56.947 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:50:56.947 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:50:56.947 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:50:56.947 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:50:56.947 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:50:56.948 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:50:56.948 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:50:56.948 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.write_buffer_manager: 0x560cd7b18720 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:50:56.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.unordered_write: 0 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.row_cache: None 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.wal_filter: None 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.preserve_deletes: 0 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:50:56.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:50:56.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_open_files: -1 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:50:56.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Compression algorithms supported: 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kZSTD supported: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:50:56.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.merge_operator: 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_filter: None 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:50:56.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560cd7a01a78) 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: index_type: 0 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: data_block_index_type: 0 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: index_shortening: 1 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:50:56.955 INFO:journalctl@ceph.mon.a.smithi142.stdout: hash_index_allow_collision: 1 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: checksum: 1 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: no_block_cache: 0 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache: 0x560cd7a371f0 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache_options: 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: capacity : 536870912 2024-08-17T16:50:56.956 INFO:journalctl@ceph.mon.a.smithi142.stdout: num_shard_bits : 4 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: strict_capacity_limit : 0 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache_compressed: (nil) 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: persistent_cache: (nil) 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_size: 4096 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_size_deviation: 10 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_restart_interval: 16 2024-08-17T16:50:56.957 INFO:journalctl@ceph.mon.a.smithi142.stdout: index_block_restart_interval: 1 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: metadata_block_size: 4096 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: partition_filters: 0 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: use_delta_encoding: 1 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: whole_key_filtering: 1 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: verify_compression: 0 2024-08-17T16:50:56.958 INFO:journalctl@ceph.mon.a.smithi142.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout: format_version: 4 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout: enable_index_compression: 1 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_align: 0 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression: NoCompression 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:50:56.959 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.num_levels: 7 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:50:56.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:50:56.961 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:50:56.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:50:56.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:50:56.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:50:56.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:50:56.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:50:56.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:50:56.963 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:50:56.963 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:50:56.963 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:50:56.963 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:50:56.963 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:50:56.963 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:50:56.964 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:50:56.965 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:50:56.965 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:50:56.965 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:50:56.965 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.arena_block_size: 4194304 2024-08-17T16:50:56.965 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:50:56.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T16:50:56.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:50:56.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.ttl: 2592000 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:50:56.969 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913456686940, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-08-17T16:50:56.970 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913456687656, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80279, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78521, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72734, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1723913456, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f63ca15d-6837-4794-bb7d-ef94a8156fe8", "db_session_id": "2F5ZKKSFZYEEJZW6QMMX"}} 2024-08-17T16:50:56.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-08-17T16:50:56.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913456687994, "job": 1, "event": "recovery_finished"} 2024-08-17T16:50:56.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:50:56.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560cd7a6a700 2024-08-17T16:50:56.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: DB pointer 0x560cd7a3a000 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** DB Stats ** 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:50:56.972 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: L0 2/0 80.18 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: Sum 2/0 80.18 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:50:56.973 INFO:journalctl@ceph.mon.a.smithi142.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative compaction: 0.00 GB write, 25.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval compaction: 0.00 GB write, 25.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:50:56.974 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: L0 2/0 80.18 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: Sum 2/0 80.18 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:50:56.975 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative compaction: 0.00 GB write, 25.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: starting mon.a rank 0 at public addrs [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] at bind addrs [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???) e1 preinit fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).mds e1 new map 2024-08-17T16:50:56.976 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).mds e1 print_map 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout: e1 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout: legacy client fscid: -1 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout: No filesystems configured 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:50:56.977 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 bash[31144]: 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 systemd[1]: Started Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: monmap e1: 1 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:50:56.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: fsmap 2024-08-17T16:50:56.979 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T16:50:56.979 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:56 smithi142 ceph-mon[31182]: mgrmap e1: no daemons active 2024-08-17T16:50:57.728 INFO:teuthology.orchestra.run.smithi142.stdout:Wrote config to /etc/ceph/ceph.conf 2024-08-17T16:50:57.728 INFO:teuthology.orchestra.run.smithi142.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:50:57.728 INFO:teuthology.orchestra.run.smithi142.stdout:Creating mgr... 2024-08-17T16:50:57.728 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying port 0.0.0.0:9283 ... 2024-08-17T16:50:57.729 INFO:teuthology.orchestra.run.smithi142.stdout:Verifying port 0.0.0.0:8765 ... 2024-08-17T16:50:57.938 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y 2024-08-17T16:50:57.938 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Failed to reset failed state of unit ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Unit ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service not loaded. 2024-08-17T16:50:58.084 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968.target.wants/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@.service. 2024-08-17T16:50:58.317 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:58 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1016276715' entity='client.admin' 2024-08-17T16:50:58.619 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:58 smithi142 podman[31418]: 2024-08-17 16:50:58.523488674 +0000 UTC m=+0.317195015 container start c1b9a1fd281fe5799d3c49f73ecec258e418fea7942142fd721ba5fbcc15a58e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731) 2024-08-17T16:50:58.620 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:58 smithi142 bash[31418]: c1b9a1fd281fe5799d3c49f73ecec258e418fea7942142fd721ba5fbcc15a58e 2024-08-17T16:50:58.633 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:50:58.633 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout disabled 2024-08-17T16:50:58.641 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:50:58.641 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout inactive 2024-08-17T16:50:58.641 INFO:teuthology.orchestra.run.smithi142.stdout:firewalld.service is not enabled 2024-08-17T16:50:58.641 INFO:teuthology.orchestra.run.smithi142.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T16:50:58.652 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:50:58.652 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout disabled 2024-08-17T16:50:58.658 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:50:58.659 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout inactive 2024-08-17T16:50:58.659 INFO:teuthology.orchestra.run.smithi142.stdout:firewalld.service is not enabled 2024-08-17T16:50:58.659 INFO:teuthology.orchestra.run.smithi142.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-08-17T16:50:58.659 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for mgr to start... 2024-08-17T16:50:58.659 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for mgr... 2024-08-17T16:50:58.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:58 smithi142 systemd[1]: Started Ceph mgr.y for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:50:58.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:50:58.639+0000 7f21632f6ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:50:58.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:50:58.766+0000 7f21632f6ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:50:58.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:50:58.832+0000 7f21632f6ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:50:59.183 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:59 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:50:59.081+0000 7f21632f6ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:50:59.184 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:50:59 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:50:59.142+0000 7f21632f6ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:50:59.483 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:50:59 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3450959516' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:50:59.756 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:50:59.756 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:50:59.756 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "fsid": "bcf463ee-5cb8-11ef-bccf-c7b262605968", 2024-08-17T16:50:59.756 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:50:59.756 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:50:59.756 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-08-17T16:50:59.757 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:50:59.758 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:50:59.759 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:50:59.759 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:50:59.759 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:50:59.759 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:50:59.759 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:50:59.762 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.762 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:50:59.762 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:50:59.762 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:50:59.762 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:50:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:50:52.039208+0000", 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:50:59.764 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:50:59.765 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:50:59.765 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:50:59.765 INFO:teuthology.orchestra.run.smithi142.stdout:mgr not available, waiting (1/15)... 2024-08-17T16:51:00.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.053+0000 7f21632f6ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:51:00.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.162+0000 7f21632f6ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:51:00.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.216+0000 7f21632f6ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:51:00.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.398+0000 7f21632f6ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:51:00.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.457+0000 7f21632f6ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:51:00.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.520+0000 7f21632f6ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:51:00.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:00 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:00.639+0000 7f21632f6ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:51:01.317 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.037+0000 7f21632f6ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:51:01.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.315+0000 7f21632f6ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:51:01.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.380+0000 7f21632f6ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:51:01.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.503+0000 7f21632f6ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:51:02.012 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.751+0000 7f21632f6ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:51:02.012 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.814+0000 7f21632f6ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:51:02.012 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:01.998+0000 7f21632f6ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:51:02.295 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:02 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:02.062+0000 7f21632f6ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:51:02.295 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:02 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:02.143+0000 7f21632f6ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:51:02.295 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:02 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:02.202+0000 7f21632f6ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:51:02.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:02 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2890952045' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:51:02.865 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:02 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:02.731+0000 7f21632f6ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:51:02.873 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:51:02.873 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:51:02.873 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "fsid": "bcf463ee-5cb8-11ef-bccf-c7b262605968", 2024-08-17T16:51:02.873 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:51:02.873 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:51:02.873 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:51:02.874 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:51:02.875 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:51:02.876 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.877 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:50:52.039208+0000", 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:51:02.878 INFO:teuthology.orchestra.run.smithi142.stdout:mgr not available, waiting (2/15)... 2024-08-17T16:51:03.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:02 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:02.866+0000 7f21632f6ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: Activating manager daemon y 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: mgrmap e2: y(active, starting, since 0.0028516s) 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: Manager daemon y is now available 2024-08-17T16:51:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:51:03.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:51:03.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' 2024-08-17T16:51:03.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' 2024-08-17T16:51:03.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:03 smithi142 ceph-mon[31182]: from='mgr.14100 172.21.15.142:0/1774945030' entity='mgr.y' 2024-08-17T16:51:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:04 smithi142 ceph-mon[31182]: mgrmap e3: y(active, since 1.00734s) 2024-08-17T16:51:06.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:05 smithi142 ceph-mon[31182]: mgrmap e4: y(active, since 2s) 2024-08-17T16:51:08.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:08 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3541986041' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:51:14.102 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:51:14.102 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "fsid": "bcf463ee-5cb8-11ef-bccf-c7b262605968", 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:51:14.103 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:51:14.104 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:51:14.105 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:51:14.106 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:50:52.039208+0000", 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:51:14.107 INFO:teuthology.orchestra.run.smithi142.stdout:mgr is available 2024-08-17T16:51:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:14 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2592386785' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T16:51:15.144 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout fsid = bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T16:51:15.145 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T16:51:15.146 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T16:51:15.146 INFO:teuthology.orchestra.run.smithi142.stdout:Enabling cephadm module... 2024-08-17T16:51:16.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:15 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3042980753' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-08-17T16:51:16.013 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:15 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setuser ceph since I am not root 2024-08-17T16:51:16.013 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:15 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setgroup ceph since I am not root 2024-08-17T16:51:16.013 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:15 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:15.890+0000 7fcd4efdeec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:51:16.014 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:16 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:16.011+0000 7fcd4efdeec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:51:16.318 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:16 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:16.077+0000 7fcd4efdeec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:51:16.319 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:16 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:16.317+0000 7fcd4efdeec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:51:16.579 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:16 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:16.379+0000 7fcd4efdeec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:51:16.948 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:16 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3042980753' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-08-17T16:51:16.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:16 smithi142 ceph-mon[31182]: mgrmap e5: y(active, since 12s) 2024-08-17T16:51:17.553 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.297+0000 7fcd4efdeec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:51:17.553 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.406+0000 7fcd4efdeec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:51:17.553 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.465+0000 7fcd4efdeec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for the mgr to restart... 2024-08-17T16:51:17.556 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for mgr epoch 5... 2024-08-17T16:51:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1974215522' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T16:51:17.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.658+0000 7fcd4efdeec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:51:17.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.719+0000 7fcd4efdeec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:51:17.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.789+0000 7fcd4efdeec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:51:18.332 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:17.915+0000 7fcd4efdeec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:51:18.617 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:18.331+0000 7fcd4efdeec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:51:18.617 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:18.615+0000 7fcd4efdeec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:51:18.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:18.679+0000 7fcd4efdeec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:51:18.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:18.804+0000 7fcd4efdeec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:51:19.354 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:19.054+0000 7fcd4efdeec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:51:19.354 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:19.112+0000 7fcd4efdeec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:51:19.355 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:19.291+0000 7fcd4efdeec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:51:19.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:19.352+0000 7fcd4efdeec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:51:19.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:19.433+0000 7fcd4efdeec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:51:19.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:19.490+0000 7fcd4efdeec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:51:20.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:20.001+0000 7fcd4efdeec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:51:20.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:20.131+0000 7fcd4efdeec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: Active manager daemon y restarted 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: Activating manager daemon y 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: osdmap e2: 0 total, 0 up, 0 in 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: mgrmap e6: y(active, starting, since 0.00328338s) 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:51:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: Manager daemon y is now available 2024-08-17T16:51:20.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:20.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:20.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:20.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:20.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:51:20.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:20 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:51:21.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-mon[31182]: Found migration_current of "None". Setting to last migration. 2024-08-17T16:51:21.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-mon[31182]: mgrmap e7: y(active, since 1.00483s) 2024-08-17T16:51:21.611 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:51:21.611 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-08-17T16:51:21.612 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T16:51:21.612 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:51:21.612 INFO:teuthology.orchestra.run.smithi142.stdout:mgr epoch 5 is available 2024-08-17T16:51:21.612 INFO:teuthology.orchestra.run.smithi142.stdout:Setting orchestrator backend to cephadm... 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:21.854+0000 7fcd01365640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:51:21] ENGINE Error in HTTPServer.serve 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: Traceback (most recent call last): 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._connections.run(self.expiration_interval) 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._run(expiration_interval) 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:51:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: s = self.context.wrap_socket( 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: return self.sslsocket_class._create( 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self.do_handshake() 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._sslobj.do_handshake() 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:51:21.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:22.257 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:22 smithi142 ceph-mon[31182]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-17T16:51:22.257 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:22 smithi142 ceph-mon[31182]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-17T16:51:22.257 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:22 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:21] ENGINE Bus STARTING 2024-08-17T16:51:22.257 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:22 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:22.257 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:22 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:21] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:21] ENGINE Bus STARTED 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:21] ENGINE Error in HTTPServer.serve 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout: Traceback (most recent call last): 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:51:23.381 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._run(expiration_interval) 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:51:23.382 INFO:journalctl@ceph.mon.a.smithi142.stdout: self.do_handshake() 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:23.383 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:23 smithi142 ceph-mon[31182]: mgrmap e8: y(active, since 2s) 2024-08-17T16:51:23.882 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout value unchanged 2024-08-17T16:51:23.882 INFO:teuthology.orchestra.run.smithi142.stdout:Generating ssh key... 2024-08-17T16:51:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-mon[31182]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:24.985 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: Generating public/private rsa key pair. 2024-08-17T16:51:24.985 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: Your identification has been saved in /tmp/tmpwo_erf91/key 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: Your public key has been saved in /tmp/tmpwo_erf91/key.pub 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: The key fingerprint is: 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: SHA256:vycJlXRh0zhMvewAzsznsOC2Vw+IMDZufDVQu6LRHTU ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: The key's randomart image is: 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: +---[RSA 3072]----+ 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | ..oE= | 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | . o+=oo | 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | Bo+ o . | 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | =...@oo o | 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | +.=S=oB o | 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | +o*oo + . | 2024-08-17T16:51:24.986 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | ..o o.o o | 2024-08-17T16:51:24.987 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | . +.. . | 2024-08-17T16:51:24.987 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: | ..o | 2024-08-17T16:51:24.987 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: +----[SHA256]-----+ 2024-08-17T16:51:25.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:25 smithi142 ceph-mon[31182]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:25.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:25 smithi142 ceph-mon[31182]: Generating ssh key... 2024-08-17T16:51:25.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:25 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:25.849 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:25 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:26.386 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYwRB5Y8wE2H8hKU7nDi3t9jUtvEG4lZTraiB8v5mK03VkvNz/bIdQUsi8c6QxiWCeDksw4lXdpUfkBR/zaaHRwK+bBt7JCAtzutTcaq50OK4Sbq0V8KYBfj/I0d8SmExEM1oo0I+JZWr/E67lAypcfyj715YjZ9tiApzeFKHUiAZQMWjN6obX6/QrDsx3fftLdqwopybn6eKxrCaWkhTbOxi+qwADQYbfS6jOKKqQ75aS7zTj/LhblZyX2wgusglBKyX3O85EDwt/A9puDmKkvixhbqvTQN1CWUElhcSwu1AQx1gR0BTQgkwCGiAZ2In6pDMBXh9U27IYoP3qydSLU8yZJyFwyEqcrPxmHHLuRnoId8fXb2qbHOgNGaKUoe1fa2mcLGTaTpdHiaGdCsrA/qlr7qGrZGZVjW43NR6wpa0o7iQQDbq4nMjpZIfzNPJGc5i4bllrTm3rhxLDChe0/FFhPMuVHzFTzKzsY8lfI6SlaSbHU+mNqCuYAsqzcPM= ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:51:26.391 INFO:teuthology.orchestra.run.smithi142.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-08-17T16:51:26.391 INFO:teuthology.orchestra.run.smithi142.stdout:Adding key to root@localhost authorized_keys... 2024-08-17T16:51:26.392 INFO:teuthology.orchestra.run.smithi142.stdout:Adding host smithi142... 2024-08-17T16:51:27.095 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:26 smithi142 ceph-mon[31182]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:28.036 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:27 smithi142 ceph-mon[31182]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi142", "addr": "172.21.15.142", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:28 smithi142 ceph-mon[31182]: Deploying cephadm binary to smithi142 2024-08-17T16:51:30.319 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout Added host 'smithi142' with addr '172.21.15.142' 2024-08-17T16:51:30.319 INFO:teuthology.orchestra.run.smithi142.stdout:Deploying unmanaged mon service... 2024-08-17T16:51:30.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:30 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:30.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:30 smithi142 ceph-mon[31182]: Added host smithi142 2024-08-17T16:51:30.949 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:30 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:31.636 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-08-17T16:51:31.636 INFO:teuthology.orchestra.run.smithi142.stdout:Deploying unmanaged mgr service... 2024-08-17T16:51:32.171 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:31 smithi142 ceph-mon[31182]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:32.171 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:31 smithi142 ceph-mon[31182]: Saving service mon spec with placement count:5 2024-08-17T16:51:32.171 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:31 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:33.116 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-08-17T16:51:33.765 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:33 smithi142 ceph-mon[31182]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:33.765 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:33 smithi142 ceph-mon[31182]: Saving service mgr spec with placement count:2 2024-08-17T16:51:33.765 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:33 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:34.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:34 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:34.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:34 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:34.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:34 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/475408523' entity='client.admin' 2024-08-17T16:51:35.871 INFO:teuthology.orchestra.run.smithi142.stdout:Enabling the dashboard module... 2024-08-17T16:51:36.287 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:36 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3458449835' entity='client.admin' 2024-08-17T16:51:37.502 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2984928789' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-08-17T16:51:37.502 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-mon[31182]: from='mgr.14118 172.21.15.142:0/1798095917' entity='mgr.y' 2024-08-17T16:51:37.503 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setuser ceph since I am not root 2024-08-17T16:51:37.503 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setgroup ceph since I am not root 2024-08-17T16:51:37.503 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:37.377+0000 7fb0327e7ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:51:37.503 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:37.500+0000 7fb0327e7ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:51:37.787 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:37.566+0000 7fb0327e7ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:51:38.050 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:37.807+0000 7fb0327e7ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:51:38.051 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:37.864+0000 7fb0327e7ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:51:38.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:38 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2984928789' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-08-17T16:51:38.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:38 smithi142 ceph-mon[31182]: mgrmap e9: y(active, since 17s) 2024-08-17T16:51:39.033 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:38.814+0000 7fb0327e7ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:51:39.034 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:38.929+0000 7fb0327e7ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:51:39.034 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:38.986+0000 7fb0327e7ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:51:39.036 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:51:39.037 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-08-17T16:51:39.037 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:51:39.037 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T16:51:39.037 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T16:51:39.037 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:51:39.038 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for the mgr to restart... 2024-08-17T16:51:39.038 INFO:teuthology.orchestra.run.smithi142.stdout:Waiting for mgr epoch 9... 2024-08-17T16:51:39.288 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:39 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3664032925' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T16:51:39.288 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:39.184+0000 7fb0327e7ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:51:39.288 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:39.247+0000 7fb0327e7ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:51:39.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:39.318+0000 7fb0327e7ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:51:39.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:39.447+0000 7fb0327e7ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:51:40.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:39.852+0000 7fb0327e7ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:51:40.570 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.133+0000 7fb0327e7ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:51:40.571 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.197+0000 7fb0327e7ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:51:40.571 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.319+0000 7fb0327e7ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:51:40.571 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.569+0000 7fb0327e7ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:51:40.865 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.626+0000 7fb0327e7ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:51:40.865 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.803+0000 7fb0327e7ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:51:41.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.864+0000 7fb0327e7ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:51:41.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:40.944+0000 7fb0327e7ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:51:41.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:41.000+0000 7fb0327e7ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:51:41.792 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:41.511+0000 7fb0327e7ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:41.642+0000 7fb0327e7ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: Active manager daemon y restarted 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: Activating manager daemon y 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: osdmap e3: 0 total, 0 up, 0 in 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: mgrmap e10: y(active, starting, since 0.00308071s) 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:51:41.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:51:41.794 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:51:41.794 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: Manager daemon y is now available 2024-08-17T16:51:41.794 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:41.794 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:51:41.794 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:51:43.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:43.051+0000 7fafe331d640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:51:43] ENGINE Error in HTTPServer.serve 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: Traceback (most recent call last): 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._connections.run(self.expiration_interval) 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._run(expiration_interval) 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:51:43.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: s = self.context.wrap_socket( 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: return self.sslsocket_class._create( 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self.do_handshake() 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._sslobj.do_handshake() 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:51:43.375 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:51:43.412 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:51:43.412 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-08-17T16:51:43.412 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T16:51:43.412 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:51:43.412 INFO:teuthology.orchestra.run.smithi142.stdout:mgr epoch 9 is available 2024-08-17T16:51:43.412 INFO:teuthology.orchestra.run.smithi142.stdout:Generating a dashboard self-signed certificate... 2024-08-17T16:51:43.666 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: mgrmap e11: y(active, since 1.00545s) 2024-08-17T16:51:43.666 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:42] ENGINE Bus STARTING 2024-08-17T16:51:43.666 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:43] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:43] ENGINE Bus STARTED 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: [17/Aug/2024:16:51:43] ENGINE Error in HTTPServer.serve 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: Traceback (most recent call last): 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._run(expiration_interval) 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:51:43.667 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: self.do_handshake() 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:43.668 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:43 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:44.685 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-08-17T16:51:44.685 INFO:teuthology.orchestra.run.smithi142.stdout:Creating initial admin user... 2024-08-17T16:51:45.345 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:45 smithi142 ceph-mon[31182]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:45.346 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:45 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:45.346 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:45 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:45.346 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:45 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:45.346 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:45 smithi142 ceph-mon[31182]: mgrmap e12: y(active, since 2s) 2024-08-17T16:51:46.847 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$8PCk8qwfN4OVJtTPrDQjpuj/IugZ/OAqf4WsizWgOmACYCVt6VxAK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1723913505, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-08-17T16:51:46.847 INFO:teuthology.orchestra.run.smithi142.stdout:Fetching dashboard port number... 2024-08-17T16:51:47.089 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:46 smithi142 ceph-mon[31182]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:47.089 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:46 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:47.937 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stdout 8443 2024-08-17T16:51:47.947 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:51:47.947 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout disabled 2024-08-17T16:51:47.954 INFO:teuthology.orchestra.run.smithi142.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:51:47.955 INFO:teuthology.orchestra.run.smithi142.stdout:systemctl: stdout inactive 2024-08-17T16:51:47.955 INFO:teuthology.orchestra.run.smithi142.stdout:firewalld.service is not enabled 2024-08-17T16:51:47.955 INFO:teuthology.orchestra.run.smithi142.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-08-17T16:51:47.956 INFO:teuthology.orchestra.run.smithi142.stdout:Ceph Dashboard is now available at: 2024-08-17T16:51:47.956 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:47.957 INFO:teuthology.orchestra.run.smithi142.stdout: URL: https://smithi142.front.sepia.ceph.com:8443/ 2024-08-17T16:51:47.957 INFO:teuthology.orchestra.run.smithi142.stdout: User: admin 2024-08-17T16:51:47.957 INFO:teuthology.orchestra.run.smithi142.stdout: Password: nhtmbgspok 2024-08-17T16:51:47.957 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:47.957 INFO:teuthology.orchestra.run.smithi142.stdout:Saving cluster configuration to /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config directory 2024-08-17T16:51:47.957 INFO:teuthology.orchestra.run.smithi142.stdout:Enabling autotune for osd_memory_target 2024-08-17T16:51:48.080 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:47 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/581792048' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-08-17T16:51:49.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:48 smithi142 ceph-mon[31182]: mgrmap e13: y(active, since 6s) 2024-08-17T16:51:50.182 INFO:teuthology.orchestra.run.smithi142.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-08-17T16:51:50.182 INFO:teuthology.orchestra.run.smithi142.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-08-17T16:51:50.182 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout:Or, if you are only running a single cluster on this host: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout:Please consider enabling telemetry to help improve Ceph: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: ceph telemetry on 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout:For more information see: 2024-08-17T16:51:50.183 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.184 INFO:teuthology.orchestra.run.smithi142.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-08-17T16:51:50.184 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:51:50.184 INFO:teuthology.orchestra.run.smithi142.stdout:Bootstrap complete. 2024-08-17T16:51:50.218 INFO:tasks.cephadm:Fetching config... 2024-08-17T16:51:50.218 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:51:50.218 DEBUG:teuthology.orchestra.run.smithi142:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-08-17T16:51:50.235 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-08-17T16:51:50.235 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:51:50.235 DEBUG:teuthology.orchestra.run.smithi142:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-08-17T16:51:50.291 INFO:tasks.cephadm:Fetching mon keyring... 2024-08-17T16:51:50.291 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:51:50.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd if=/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/keyring of=/dev/stdout 2024-08-17T16:51:50.356 INFO:tasks.cephadm:Fetching pub ssh key... 2024-08-17T16:51:50.357 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:51:50.357 DEBUG:teuthology.orchestra.run.smithi142:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-08-17T16:51:50.412 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-08-17T16:51:50.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYwRB5Y8wE2H8hKU7nDi3t9jUtvEG4lZTraiB8v5mK03VkvNz/bIdQUsi8c6QxiWCeDksw4lXdpUfkBR/zaaHRwK+bBt7JCAtzutTcaq50OK4Sbq0V8KYBfj/I0d8SmExEM1oo0I+JZWr/E67lAypcfyj715YjZ9tiApzeFKHUiAZQMWjN6obX6/QrDsx3fftLdqwopybn6eKxrCaWkhTbOxi+qwADQYbfS6jOKKqQ75aS7zTj/LhblZyX2wgusglBKyX3O85EDwt/A9puDmKkvixhbqvTQN1CWUElhcSwu1AQx1gR0BTQgkwCGiAZ2In6pDMBXh9U27IYoP3qydSLU8yZJyFwyEqcrPxmHHLuRnoId8fXb2qbHOgNGaKUoe1fa2mcLGTaTpdHiaGdCsrA/qlr7qGrZGZVjW43NR6wpa0o7iQQDbq4nMjpZIfzNPJGc5i4bllrTm3rhxLDChe0/FFhPMuVHzFTzKzsY8lfI6SlaSbHU+mNqCuYAsqzcPM= ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T16:51:50.487 INFO:teuthology.orchestra.run.smithi142.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYwRB5Y8wE2H8hKU7nDi3t9jUtvEG4lZTraiB8v5mK03VkvNz/bIdQUsi8c6QxiWCeDksw4lXdpUfkBR/zaaHRwK+bBt7JCAtzutTcaq50OK4Sbq0V8KYBfj/I0d8SmExEM1oo0I+JZWr/E67lAypcfyj715YjZ9tiApzeFKHUiAZQMWjN6obX6/QrDsx3fftLdqwopybn6eKxrCaWkhTbOxi+qwADQYbfS6jOKKqQ75aS7zTj/LhblZyX2wgusglBKyX3O85EDwt/A9puDmKkvixhbqvTQN1CWUElhcSwu1AQx1gR0BTQgkwCGiAZ2In6pDMBXh9U27IYoP3qydSLU8yZJyFwyEqcrPxmHHLuRnoId8fXb2qbHOgNGaKUoe1fa2mcLGTaTpdHiaGdCsrA/qlr7qGrZGZVjW43NR6wpa0o7iQQDbq4nMjpZIfzNPJGc5i4bllrTm3rhxLDChe0/FFhPMuVHzFTzKzsY8lfI6SlaSbHU+mNqCuYAsqzcPM= ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:51:50.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYwRB5Y8wE2H8hKU7nDi3t9jUtvEG4lZTraiB8v5mK03VkvNz/bIdQUsi8c6QxiWCeDksw4lXdpUfkBR/zaaHRwK+bBt7JCAtzutTcaq50OK4Sbq0V8KYBfj/I0d8SmExEM1oo0I+JZWr/E67lAypcfyj715YjZ9tiApzeFKHUiAZQMWjN6obX6/QrDsx3fftLdqwopybn6eKxrCaWkhTbOxi+qwADQYbfS6jOKKqQ75aS7zTj/LhblZyX2wgusglBKyX3O85EDwt/A9puDmKkvixhbqvTQN1CWUElhcSwu1AQx1gR0BTQgkwCGiAZ2In6pDMBXh9U27IYoP3qydSLU8yZJyFwyEqcrPxmHHLuRnoId8fXb2qbHOgNGaKUoe1fa2mcLGTaTpdHiaGdCsrA/qlr7qGrZGZVjW43NR6wpa0o7iQQDbq4nMjpZIfzNPJGc5i4bllrTm3rhxLDChe0/FFhPMuVHzFTzKzsY8lfI6SlaSbHU+mNqCuYAsqzcPM= ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T16:51:50.552 INFO:teuthology.orchestra.run.smithi152.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYwRB5Y8wE2H8hKU7nDi3t9jUtvEG4lZTraiB8v5mK03VkvNz/bIdQUsi8c6QxiWCeDksw4lXdpUfkBR/zaaHRwK+bBt7JCAtzutTcaq50OK4Sbq0V8KYBfj/I0d8SmExEM1oo0I+JZWr/E67lAypcfyj715YjZ9tiApzeFKHUiAZQMWjN6obX6/QrDsx3fftLdqwopybn6eKxrCaWkhTbOxi+qwADQYbfS6jOKKqQ75aS7zTj/LhblZyX2wgusglBKyX3O85EDwt/A9puDmKkvixhbqvTQN1CWUElhcSwu1AQx1gR0BTQgkwCGiAZ2In6pDMBXh9U27IYoP3qydSLU8yZJyFwyEqcrPxmHHLuRnoId8fXb2qbHOgNGaKUoe1fa2mcLGTaTpdHiaGdCsrA/qlr7qGrZGZVjW43NR6wpa0o7iQQDbq4nMjpZIfzNPJGc5i4bllrTm3rhxLDChe0/FFhPMuVHzFTzKzsY8lfI6SlaSbHU+mNqCuYAsqzcPM= ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:51:50.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-08-17T16:51:50.824 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:50 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1349698283' entity='client.admin' 2024-08-17T16:51:53.482 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-08-17T16:51:53.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-08-17T16:51:53.729 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2136080076' entity='client.admin' 2024-08-17T16:51:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi142", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:56.131 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi152 2024-08-17T16:51:56.131 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:51:56.131 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/etc/ceph/ceph.conf 2024-08-17T16:51:56.152 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:51:56.152 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:51:56.210 INFO:tasks.cephadm:Adding host smithi152 to orchestrator... 2024-08-17T16:51:56.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch host add smithi152 2024-08-17T16:51:56.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:51:56.456 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:58 smithi142 ceph-mon[31182]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi152", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:51:59 smithi142 ceph-mon[31182]: Deploying cephadm binary to smithi152 2024-08-17T16:52:00.555 INFO:teuthology.orchestra.run.smithi142.stdout:Added host 'smithi152' with addr '172.21.15.152' 2024-08-17T16:52:01.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch host ls --format=json 2024-08-17T16:52:01.304 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:01.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:01 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:01.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:01 smithi142 ceph-mon[31182]: Added host smithi152 2024-08-17T16:52:01.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:01 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:01.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:01 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:01.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:01 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:02.885 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:52:02.886 INFO:teuthology.orchestra.run.smithi142.stdout:[{"addr": "172.21.15.142", "hostname": "smithi142", "labels": [], "status": ""}, {"addr": "172.21.15.152", "hostname": "smithi152", "labels": [], "status": ""}] 2024-08-17T16:52:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:02 smithi142 ceph-mon[31182]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:03.502 INFO:tasks.cephadm:Setting crush tunables to default 2024-08-17T16:52:03.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd crush tunables default 2024-08-17T16:52:03.750 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:04.136 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:03 smithi142 ceph-mon[31182]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:05.918 INFO:teuthology.orchestra.run.smithi142.stderr:adjusted tunables profile to default 2024-08-17T16:52:06.263 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:52:06.263 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:06.263 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:06.263 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:06.263 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/478512479' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-17T16:52:06.263 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:06.264 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:06.264 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:05 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:06.572 INFO:tasks.cephadm:Adding mon.a on smithi142 2024-08-17T16:52:06.572 INFO:tasks.cephadm:Adding mon.c on smithi142 2024-08-17T16:52:06.572 INFO:tasks.cephadm:Adding mon.b on smithi152 2024-08-17T16:52:06.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply mon '3;smithi142:172.21.15.142=a;smithi142:[v2:172.21.15.142:3301,v1:172.21.15.142:6790]=c;smithi152:172.21.15.152=b' 2024-08-17T16:52:06.827 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:07.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:06 smithi142 ceph-mon[31182]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:07.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:06 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/478512479' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T16:52:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:06 smithi142 ceph-mon[31182]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:52:08.372 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled mon update... 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:09.186 DEBUG:teuthology.orchestra.run.smithi142:mon.c> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.c.service 2024-08-17T16:52:09.189 DEBUG:teuthology.orchestra.run.smithi152:mon.b> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.b.service 2024-08-17T16:52:09.192 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T16:52:09.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T16:52:10.143 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:10.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:09 smithi142 ceph-mon[31182]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi142:172.21.15.142=a;smithi142:[v2:172.21.15.142:3301,v1:172.21.15.142:6790]=c;smithi152:172.21.15.152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:10.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:09 smithi142 ceph-mon[31182]: Saving service mon spec with placement smithi142:172.21.15.142=a;smithi142:[v2:172.21.15.142:3301,v1:172.21.15.142:6790]=c;smithi152:172.21.15.152=b;count:3 2024-08-17T16:52:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:09 smithi142 ceph-mon[31182]: Deploying daemon mon.b on smithi152 2024-08-17T16:52:11.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:10 smithi142 ceph-mon[31182]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:12.868 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:52:12.868 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":1,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","modified":"2024-08-17T16:50:49.283901Z","created":"2024-08-17T16:50:49.283901Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:3300","nonce":0},{"type":"v1","addr":"172.21.15.142:6789","nonce":0}]},"addr":"172.21.15.142:6789/0","public_addr":"172.21.15.142:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-08-17T16:52:12.868 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 1 2024-08-17T16:52:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:12 smithi142 ceph-mon[31182]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:13.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:12 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/1127359729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:14.788 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T16:52:14.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T16:52:15.047 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:18.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: mon.a calling monitor election 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: mon.b calling monitor election 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: monmap e2: 2 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: fsmap 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: mgrmap e13: y(active, since 36s) 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: overall HEALTH_OK 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:18.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:18.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:18 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:19 smithi142 ceph-mon[31182]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:19 smithi142 ceph-mon[31182]: Deploying daemon mon.c on smithi142 2024-08-17T16:52:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:19 smithi152 ceph-mon[34775]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:19 smithi152 ceph-mon[34775]: Deploying daemon mon.c on smithi142 2024-08-17T16:52:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:52:19.791 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":2,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","modified":"2024-08-17T16:52:13.013973Z","created":"2024-08-17T16:50:49.283901Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:3300","nonce":0},{"type":"v1","addr":"172.21.15.142:6789","nonce":0}]},"addr":"172.21.15.142:6789/0","public_addr":"172.21.15.142:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-17T16:52:19.792 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 2 2024-08-17T16:52:20.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:20 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/2745405625' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:20.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:20 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/2745405625' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.101 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 podman[39981]: 2024-08-17 16:52:20.941370469 +0000 UTC m=+0.016816705 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T16:52:21.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[31182]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 podman[39981]: 2024-08-17 16:52:21.194125578 +0000 UTC m=+0.269571807 container init 24ff3a4330748f88b794dad982f1e769cb8e98ca76b572f74f86ff5b497d7f0c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 podman[39981]: 2024-08-17 16:52:21.199562749 +0000 UTC m=+0.275008976 container start 24ff3a4330748f88b794dad982f1e769cb8e98ca76b572f74f86ff5b497d7f0c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable), process ceph-mon, pid 7 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pidfile_write: ignore empty --pid-file 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: load: jerasure load: lrc 2024-08-17T16:52:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: RocksDB version: 6.15.5 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Compile date Aug 17 2024 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: DB SUMMARY 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: DB Session ID: SBMOAEOX4ND7U4ISKIBY 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:52:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.env: 0x55e37f77c480 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.fs: Posix File System 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.info_log: 0x55e3819f2f80 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.statistics: (nil) 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.use_fsync: 0 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:52:21.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.db_log_dir: 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:52:21.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.write_buffer_manager: 0x55e381ab0750 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:52:21.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.unordered_write: 0 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.row_cache: None 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.wal_filter: None 2024-08-17T16:52:21.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.preserve_deletes: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:52:21.380 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:52:21.381 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_open_files: -1 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:52:21.382 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Compression algorithms supported: 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kZSTD supported: 0 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:52:21.383 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.merge_operator: 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_filter: None 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e381999a78) 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:52:21.384 INFO:journalctl@ceph.mon.c.smithi142.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: index_type: 0 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: data_block_index_type: 0 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: index_shortening: 1 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: hash_index_allow_collision: 1 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: checksum: 1 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: no_block_cache: 0 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache: 0x55e3819cf1f0 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache_options: 2024-08-17T16:52:21.385 INFO:journalctl@ceph.mon.c.smithi142.stdout: capacity : 536870912 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: num_shard_bits : 4 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: strict_capacity_limit : 0 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache_compressed: (nil) 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: persistent_cache: (nil) 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_size: 4096 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_size_deviation: 10 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_restart_interval: 16 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: index_block_restart_interval: 1 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: metadata_block_size: 4096 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: partition_filters: 0 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: use_delta_encoding: 1 2024-08-17T16:52:21.386 INFO:journalctl@ceph.mon.c.smithi142.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout: whole_key_filtering: 1 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout: verify_compression: 0 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout: format_version: 4 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout: enable_index_compression: 1 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_align: 0 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression: NoCompression 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:52:21.387 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.num_levels: 7 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:52:21.388 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:52:21.389 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:52:21.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.arena_block_size: 4194304 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:52:21.391 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:52:21.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.ttl: 2592000 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913541222772, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T16:52:21.393 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913541223123, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1723913541, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5bf64931-1daf-4630-987d-62e7637cb82d", "db_session_id": "SBMOAEOX4ND7U4ISKIBY"}} 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913541223469, "job": 1, "event": "recovery_finished"} 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55e381a02700 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: DB pointer 0x55e3819d2000 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** DB Stats ** 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:52:21.394 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.395 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:52:21.396 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T16:52:21.397 INFO:journalctl@ceph.mon.c.smithi142.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:52:21.398 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: using public_addrv [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: starting mon.c rank -1 at public addrs [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] at bind addrs [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(???) e0 preinit fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).mds e1 new map 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).mds e1 print_map 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: e1 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: legacy client fscid: -1 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T16:52:21.399 INFO:journalctl@ceph.mon.c.smithi142.stdout: No filesystems configured 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi142:172.21.15.142=a;smithi142:[v2:172.21.15.142:3301,v1:172.21.15.142:6790]=c;smithi152:172.21.15.152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: Saving service mon spec with placement smithi142:172.21.15.142=a;smithi142:[v2:172.21.15.142:3301,v1:172.21.15.142:6790]=c;smithi152:172.21.15.152=b;count:3 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: Deploying daemon mon.b on smithi152 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/1127359729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.401 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.a calling monitor election 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.b calling monitor election 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T16:52:21.402 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: monmap e2: 2 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: fsmap 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mgrmap e13: y(active, since 36s) 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: overall HEALTH_OK 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: Deploying daemon mon.c on smithi142 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.403 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/2745405625' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.404 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.404 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 ceph-mon[40020]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T16:52:21.404 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 bash[39981]: 24ff3a4330748f88b794dad982f1e769cb8e98ca76b572f74f86ff5b497d7f0c 2024-08-17T16:52:21.404 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:21 smithi142 systemd[1]: Started Ceph mon.c for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:52:21.404 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T16:52:21.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T16:52:21.405 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:21 smithi152 ceph-mon[34775]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:21.677 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: mon.b calling monitor election 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: mon.c calling monitor election 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: monmap e3: 3 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: fsmap 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: mgrmap e13: y(active, since 44s) 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: mon.a calling monitor election 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: mon.b calling monitor election 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: mon.c calling monitor election 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:52:26.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: monmap e3: 3 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:52:26.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: fsmap 2024-08-17T16:52:26.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:52:26.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: mgrmap e13: y(active, since 44s) 2024-08-17T16:52:26.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: overall HEALTH_OK 2024-08-17T16:52:26.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.626 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: mon.a calling monitor election 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: mon.b calling monitor election 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: mon.c calling monitor election 2024-08-17T16:52:26.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:26.629 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:26.629 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:52:26.629 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: monmap e3: 3 mons at {a=[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:52:26.629 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: fsmap 2024-08-17T16:52:26.629 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:52:26.629 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: mgrmap e13: y(active, since 44s) 2024-08-17T16:52:26.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: overall HEALTH_OK 2024-08-17T16:52:26.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.631 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:26.639 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:52:26.639 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":3,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","modified":"2024-08-17T16:52:21.246059Z","created":"2024-08-17T16:50:49.283901Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:3300","nonce":0},{"type":"v1","addr":"172.21.15.142:6789","nonce":0}]},"addr":"172.21.15.142:6789/0","public_addr":"172.21.15.142:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:3301","nonce":0},{"type":"v1","addr":"172.21.15.142:6790","nonce":0}]},"addr":"172.21.15.142:6790/0","public_addr":"172.21.15.142:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-08-17T16:52:26.640 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 3 2024-08-17T16:52:27.248 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-08-17T16:52:27.249 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph config generate-minimal-conf 2024-08-17T16:52:27.508 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:27.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:27 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:27.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:27 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/1039011126' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:27.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:27 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:27.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:27 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:27.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/1039011126' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/1039011126' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:27.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:27 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:28.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:28.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:28.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:28.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:28 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:28.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:28.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:28 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:29.260 INFO:teuthology.orchestra.run.smithi142.stdout:# minimal ceph.conf for bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:52:29.260 INFO:teuthology.orchestra.run.smithi142.stdout:[global] 2024-08-17T16:52:29.260 INFO:teuthology.orchestra.run.smithi142.stdout: fsid = bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:52:29.260 INFO:teuthology.orchestra.run.smithi142.stdout: mon_host = [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] 2024-08-17T16:52:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:29 smithi152 ceph-mon[34775]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T16:52:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:29 smithi152 ceph-mon[34775]: Reconfiguring daemon mon.a on smithi142 2024-08-17T16:52:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:29 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2407782387' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:29.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:29 smithi142 ceph-mon[31182]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T16:52:29.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:29 smithi142 ceph-mon[31182]: Reconfiguring daemon mon.a on smithi142 2024-08-17T16:52:29.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:29 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2407782387' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:29.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:29 smithi142 ceph-mon[40020]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T16:52:29.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:29 smithi142 ceph-mon[40020]: Reconfiguring daemon mon.a on smithi142 2024-08-17T16:52:29.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:29 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2407782387' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:29.869 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-08-17T16:52:29.870 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:52:29.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T16:52:29.896 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:52:29.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:29.962 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:52:29.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T16:52:29.995 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:52:29.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:30.064 INFO:tasks.cephadm:Adding mgr.y on smithi142 2024-08-17T16:52:30.064 INFO:tasks.cephadm:Adding mgr.x on smithi152 2024-08-17T16:52:30.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply mgr '2;smithi142=y;smithi152=x' 2024-08-17T16:52:30.350 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:30.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:30 smithi152 ceph-mon[34775]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:30 smithi142 ceph-mon[40020]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:30 smithi142 ceph-mon[31182]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:31.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:31.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:31.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: Reconfiguring mon.c (monmap changed)... 2024-08-17T16:52:31.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:31.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:31.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:31.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:31 smithi152 ceph-mon[34775]: Reconfiguring daemon mon.c on smithi142 2024-08-17T16:52:32.058 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled mgr update... 2024-08-17T16:52:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: Reconfiguring mon.c (monmap changed)... 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[40020]: Reconfiguring daemon mon.c on smithi142 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: Reconfiguring mon.c (monmap changed)... 2024-08-17T16:52:32.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:32.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:32.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:32.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:31 smithi142 ceph-mon[31182]: Reconfiguring daemon mon.c on smithi142 2024-08-17T16:52:32.635 DEBUG:teuthology.orchestra.run.smithi152:mgr.x> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service 2024-08-17T16:52:32.637 DEBUG:tasks.cephadm:set 0 configs 2024-08-17T16:52:32.637 INFO:tasks.cephadm:Deploying OSDs... 2024-08-17T16:52:32.637 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:52:32.637 DEBUG:teuthology.orchestra.run.smithi142:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T16:52:32.655 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T16:52:32.655 DEBUG:teuthology.orchestra.run.smithi142:> stat /dev/vg_nvme/lv_1 2024-08-17T16:52:32.711 INFO:teuthology.orchestra.run.smithi142.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T16:52:32.711 INFO:teuthology.orchestra.run.smithi142.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:32.711 INFO:teuthology.orchestra.run.smithi142.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-08-17T16:52:32.711 INFO:teuthology.orchestra.run.smithi142.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:32.712 INFO:teuthology.orchestra.run.smithi142.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:32.712 INFO:teuthology.orchestra.run.smithi142.stdout:Access: 2024-08-17 16:52:20.765890184 +0000 2024-08-17T16:52:32.712 INFO:teuthology.orchestra.run.smithi142.stdout:Modify: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:32.712 INFO:teuthology.orchestra.run.smithi142.stdout:Change: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:32.712 INFO:teuthology.orchestra.run.smithi142.stdout: Birth: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:32.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T16:52:32.776 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records in 2024-08-17T16:52:32.777 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records out 2024-08-17T16:52:32.777 INFO:teuthology.orchestra.run.smithi142.stderr:512 bytes copied, 0.000219679 s, 2.3 MB/s 2024-08-17T16:52:32.778 DEBUG:teuthology.orchestra.run.smithi142:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T16:52:32.836 DEBUG:teuthology.orchestra.run.smithi142:> stat /dev/vg_nvme/lv_2 2024-08-17T16:52:32.892 INFO:teuthology.orchestra.run.smithi142.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T16:52:32.908 INFO:teuthology.orchestra.run.smithi142.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:32.908 INFO:teuthology.orchestra.run.smithi142.stdout:Device: 5h/5d Inode: 988 Links: 1 2024-08-17T16:52:32.908 INFO:teuthology.orchestra.run.smithi142.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:32.908 INFO:teuthology.orchestra.run.smithi142.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:32.908 INFO:teuthology.orchestra.run.smithi142.stdout:Access: 2024-08-17 16:52:20.765890184 +0000 2024-08-17T16:52:32.908 INFO:teuthology.orchestra.run.smithi142.stdout:Modify: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:32.909 INFO:teuthology.orchestra.run.smithi142.stdout:Change: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:32.909 INFO:teuthology.orchestra.run.smithi142.stdout: Birth: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:32.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T16:52:32.956 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records in 2024-08-17T16:52:32.956 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records out 2024-08-17T16:52:32.956 INFO:teuthology.orchestra.run.smithi142.stderr:512 bytes copied, 0.000249579 s, 2.1 MB/s 2024-08-17T16:52:32.957 DEBUG:teuthology.orchestra.run.smithi142:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T16:52:33.015 DEBUG:teuthology.orchestra.run.smithi142:> stat /dev/vg_nvme/lv_3 2024-08-17T16:52:33.070 INFO:teuthology.orchestra.run.smithi142.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T16:52:33.070 INFO:teuthology.orchestra.run.smithi142.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:33.070 INFO:teuthology.orchestra.run.smithi142.stdout:Device: 5h/5d Inode: 986 Links: 1 2024-08-17T16:52:33.070 INFO:teuthology.orchestra.run.smithi142.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:33.071 INFO:teuthology.orchestra.run.smithi142.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:33.071 INFO:teuthology.orchestra.run.smithi142.stdout:Access: 2024-08-17 16:52:20.766890166 +0000 2024-08-17T16:52:33.071 INFO:teuthology.orchestra.run.smithi142.stdout:Modify: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:33.071 INFO:teuthology.orchestra.run.smithi142.stdout:Change: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:33.071 INFO:teuthology.orchestra.run.smithi142.stdout: Birth: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:33.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T16:52:33.133 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records in 2024-08-17T16:52:33.133 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records out 2024-08-17T16:52:33.133 INFO:teuthology.orchestra.run.smithi142.stderr:512 bytes copied, 0.000178376 s, 2.9 MB/s 2024-08-17T16:52:33.135 DEBUG:teuthology.orchestra.run.smithi142:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T16:52:33.191 DEBUG:teuthology.orchestra.run.smithi142:> stat /dev/vg_nvme/lv_4 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout:Access: 2024-08-17 16:52:20.766890166 +0000 2024-08-17T16:52:33.246 INFO:teuthology.orchestra.run.smithi142.stdout:Modify: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:33.247 INFO:teuthology.orchestra.run.smithi142.stdout:Change: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:33.247 INFO:teuthology.orchestra.run.smithi142.stdout: Birth: 2024-08-17 16:47:56.096779434 +0000 2024-08-17T16:52:33.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T16:52:33.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:33.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi142=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:33.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: Saving service mgr spec with placement smithi142=y;smithi152=x;count:2 2024-08-17T16:52:33.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: Reconfiguring mon.b (monmap changed)... 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[40020]: Reconfiguring daemon mon.b on smithi152 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi142=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: Saving service mgr spec with placement smithi142=y;smithi152=x;count:2 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: Reconfiguring mon.b (monmap changed)... 2024-08-17T16:52:33.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:33.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:33.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:33.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:33 smithi142 ceph-mon[31182]: Reconfiguring daemon mon.b on smithi152 2024-08-17T16:52:33.309 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records in 2024-08-17T16:52:33.309 INFO:teuthology.orchestra.run.smithi142.stderr:1+0 records out 2024-08-17T16:52:33.309 INFO:teuthology.orchestra.run.smithi142.stderr:512 bytes copied, 0.000200578 s, 2.6 MB/s 2024-08-17T16:52:33.311 DEBUG:teuthology.orchestra.run.smithi142:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi142=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: Saving service mgr spec with placement smithi142=y;smithi152=x;count:2 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: Reconfiguring mon.b (monmap changed)... 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:33 smithi152 ceph-mon[34775]: Reconfiguring daemon mon.b on smithi152 2024-08-17T16:52:33.367 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:52:33.367 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T16:52:33.384 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T16:52:33.384 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_1 2024-08-17T16:52:33.440 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T16:52:33.440 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:33.440 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 971 Links: 1 2024-08-17T16:52:33.440 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:33.440 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:33.441 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-08-17 16:52:12.096931799 +0000 2024-08-17T16:52:33.441 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.441 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.441 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T16:52:33.505 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-08-17T16:52:33.505 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-08-17T16:52:33.505 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000201778 s, 2.5 MB/s 2024-08-17T16:52:33.506 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T16:52:33.562 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_2 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-08-17 16:52:12.096931799 +0000 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.617 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T16:52:33.680 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-08-17T16:52:33.680 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-08-17T16:52:33.680 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000210521 s, 2.4 MB/s 2024-08-17T16:52:33.681 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T16:52:33.738 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_3 2024-08-17T16:52:33.794 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T16:52:33.794 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:33.794 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 979 Links: 1 2024-08-17T16:52:33.794 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:33.795 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:33.795 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-08-17 16:52:12.097931788 +0000 2024-08-17T16:52:33.795 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.795 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.795 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-08-17 16:47:57.028877902 +0000 2024-08-17T16:52:33.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T16:52:33.858 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-08-17T16:52:33.859 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-08-17T16:52:33.859 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000228649 s, 2.2 MB/s 2024-08-17T16:52:33.860 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T16:52:33.915 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_4 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 963 Links: 1 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-08-17 16:52:12.097931788 +0000 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-08-17 16:47:57.027877913 +0000 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-08-17 16:47:57.027877913 +0000 2024-08-17T16:52:33.972 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-08-17 16:47:57.027877913 +0000 2024-08-17T16:52:33.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T16:52:34.035 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-08-17T16:52:34.035 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-08-17T16:52:34.035 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000172629 s, 3.0 MB/s 2024-08-17T16:52:34.037 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T16:52:34.093 INFO:tasks.cephadm:Deploying osd.0 on smithi142 with /dev/vg_nvme/lv_4... 2024-08-17T16:52:34.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T16:52:34.350 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:34 smithi152 ceph-mon[34775]: Deploying daemon mgr.x on smithi152 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:35.121 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[40020]: Deploying daemon mgr.x on smithi152 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T16:52:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:52:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:34 smithi142 ceph-mon[31182]: Deploying daemon mgr.x on smithi152 2024-08-17T16:52:35.800 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:35 smithi152 podman[36827]: 2024-08-17 16:52:35.698469347 +0000 UTC m=+0.116228657 container create 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T16:52:36.139 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:35 smithi152 podman[36827]: 2024-08-17 16:52:35.875826086 +0000 UTC m=+0.293585396 container init 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-08-17T16:52:36.139 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:35 smithi152 podman[36827]: 2024-08-17 16:52:35.88044903 +0000 UTC m=+0.298208347 container start 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4) 2024-08-17T16:52:36.139 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:35 smithi152 ceph-mgr[36866]: -- 172.21.15.152:0/835396391 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x565493a489c0 con 0x565493a55000 2024-08-17T16:52:36.139 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:35 smithi152 bash[36827]: 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 2024-08-17T16:52:36.139 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:35 smithi152 systemd[1]: Started Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:52:36.139 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:36.003+0000 7fde774d8ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:52:36.140 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:36.137+0000 7fde774d8ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:52:36.475 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:36.206+0000 7fde774d8ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:52:36.475 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:36.473+0000 7fde774d8ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:52:36.742 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:52:36.745 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:36.540+0000 7fde774d8ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:52:36.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi142:vg_nvme/lv_4 2024-08-17T16:52:37.015 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:37.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:37.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:37 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:37.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:37.493+0000 7fde774d8ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:52:37.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:37.603+0000 7fde774d8ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:52:37.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:37.657+0000 7fde774d8ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:52:38.235 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:37.843+0000 7fde774d8ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:52:38.235 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:37.903+0000 7fde774d8ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:52:38.235 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:37.967+0000 7fde774d8ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:52:38.236 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:38.092+0000 7fde774d8ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:52:38.507 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:38.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:38.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:38.513 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:52:38.513 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:52:38.513 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:38.790 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:38.505+0000 7fde774d8ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:52:38.790 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:38.789+0000 7fde774d8ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:52:39.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:38.853+0000 7fde774d8ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:52:39.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:38.977+0000 7fde774d8ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:52:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-mon[34775]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T16:52:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-mon[34775]: Reconfiguring daemon mgr.y on smithi142 2024-08-17T16:52:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-mon[34775]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:39.537 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:39.232+0000 7fde774d8ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:52:39.538 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:39.293+0000 7fde774d8ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:52:39.538 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:39.473+0000 7fde774d8ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:52:39.617 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:39 smithi142 ceph-mon[40020]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T16:52:39.617 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:39 smithi142 ceph-mon[40020]: Reconfiguring daemon mgr.y on smithi142 2024-08-17T16:52:39.617 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:39 smithi142 ceph-mon[40020]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:39.618 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:39 smithi142 ceph-mon[31182]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T16:52:39.618 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:39 smithi142 ceph-mon[31182]: Reconfiguring daemon mgr.y on smithi142 2024-08-17T16:52:39.618 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:39 smithi142 ceph-mon[31182]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:39.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:39.536+0000 7fde774d8ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:52:39.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:39.617+0000 7fde774d8ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:52:39.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:39.674+0000 7fde774d8ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:52:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:52:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:52:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:40.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:40.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:40.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:40.201+0000 7fde774d8ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:52:40.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:52:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:52:40.337+0000 7fde774d8ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:52:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:40.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T16:52:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:52:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:52:41.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:52:41.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: Standby manager daemon x started 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:52:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: Standby manager daemon x started 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:52:41.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/106287502' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:52:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: mgrmap e14: y(active, since 59s), standbys: x 2024-08-17T16:52:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:52:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3072432703' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]: dispatch 2024-08-17T16:52:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]: dispatch 2024-08-17T16:52:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]': finished 2024-08-17T16:52:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T16:52:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:42 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1909076284' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:52:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: mgrmap e14: y(active, since 59s), standbys: x 2024-08-17T16:52:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3072432703' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]': finished 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1909076284' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: mgrmap e14: y(active, since 59s), standbys: x 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3072432703' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]: dispatch 2024-08-17T16:52:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]: dispatch 2024-08-17T16:52:42.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4a1ac03b-9520-4665-ba56-380cd889d3f0"}]': finished 2024-08-17T16:52:42.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T16:52:42.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:42.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:42 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1909076284' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:52:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:43 smithi152 ceph-mon[34775]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:43 smithi142 ceph-mon[31182]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:43.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:43 smithi142 ceph-mon[40020]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:45 smithi142 ceph-mon[31182]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:45 smithi142 ceph-mon[40020]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:45 smithi152 ceph-mon[34775]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:47 smithi152 ceph-mon[34775]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:47.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:47 smithi142 ceph-mon[31182]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:47.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:47 smithi142 ceph-mon[40020]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:48.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T16:52:48.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:48.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T16:52:48.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:48.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T16:52:48.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:49.550 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:49 smithi142 ceph-mon[40020]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:49.550 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:49 smithi142 ceph-mon[40020]: Deploying daemon osd.0 on smithi142 2024-08-17T16:52:49.551 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:49 smithi142 ceph-mon[31182]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:49.551 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:49 smithi142 ceph-mon[31182]: Deploying daemon osd.0 on smithi142 2024-08-17T16:52:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:49 smithi152 ceph-mon[34775]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:49 smithi152 ceph-mon[34775]: Deploying daemon osd.0 on smithi142 2024-08-17T16:52:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:51 smithi142 ceph-mon[31182]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:51.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:51 smithi142 ceph-mon[40020]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:51 smithi152 ceph-mon[34775]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:53.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:53 smithi152 ceph-mon[34775]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:53.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:53 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:53.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:53 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:53.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:53 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[40020]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[31182]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:53.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:55.555 INFO:teuthology.orchestra.run.smithi142.stdout:Created osd(s) 0 on host 'smithi142' 2024-08-17T16:52:55.555 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:55 smithi142 ceph-mon[31182]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:55.555 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:55 smithi142 ceph-mon[40020]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:55 smithi152 ceph-mon[34775]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:56.152 DEBUG:teuthology.orchestra.run.smithi142:osd.0> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.0.service 2024-08-17T16:52:56.154 INFO:tasks.cephadm:Deploying osd.1 on smithi142 with /dev/vg_nvme/lv_3... 2024-08-17T16:52:56.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T16:52:56.454 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:56 smithi152 ceph-mon[34775]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:52:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:56 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:56 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:56 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:56 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.580 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[40020]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:52:56.580 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.580 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[31182]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:56.581 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:56 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:52:57.562 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[40020]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:57.562 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[40020]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T16:52:57.562 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[40020]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T16:52:57.562 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:57.562 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[40020]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:52:57.563 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[31182]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:57.563 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[31182]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T16:52:57.563 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[31182]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T16:52:57.563 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:57.563 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-mon[31182]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:52:57.563 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[44296]: 2024-08-17T16:52:57.294+0000 7faa43299640 -1 osd.0 0 waiting for initial osdmap 2024-08-17T16:52:57.563 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:52:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[44296]: 2024-08-17T16:52:57.297+0000 7faa3f8d3640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:52:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:57 smithi152 ceph-mon[34775]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:57 smithi152 ceph-mon[34775]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T16:52:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:57 smithi152 ceph-mon[34775]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T16:52:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:57 smithi152 ceph-mon[34775]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[31182]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[31182]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[40020]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[40020]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:58.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:58 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:58 smithi152 ceph-mon[34775]: from='osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:52:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:58 smithi152 ceph-mon[34775]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T16:52:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:58 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:58 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:59.170 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:52:59.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi142:vg_nvme/lv_3 2024-08-17T16:52:59.567 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[31182]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[31182]: osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577] boot 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[31182]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[40020]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[40020]: osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577] boot 2024-08-17T16:52:59.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[40020]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T16:52:59.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:52:59 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:59 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:52:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:59 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:52:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:59 smithi152 ceph-mon[34775]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:52:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:59 smithi152 ceph-mon[34775]: osd.0 [v2:172.21.15.142:6802/3785349577,v1:172.21.15.142:6803/3785349577] boot 2024-08-17T16:52:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:59 smithi152 ceph-mon[34775]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T16:52:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:52:59 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:59.637 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.389 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:00 smithi142 ceph-mon[31182]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T16:53:00.389 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:00 smithi142 ceph-mon[31182]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:00.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:00 smithi142 ceph-mon[40020]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T16:53:00.390 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:00 smithi142 ceph-mon[40020]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:00 smithi152 ceph-mon[34775]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T16:53:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:00 smithi152 ceph-mon[34775]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi142 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi142 to 16064M 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:02.987 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:02.988 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi142 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi142 to 16064M 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:02.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:02.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:02.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:02.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi142 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi142 to 16064M 2024-08-17T16:53:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:04.060 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:03 smithi142 ceph-mon[40020]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:04.060 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:03 smithi142 ceph-mon[31182]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:03 smithi152 ceph-mon[34775]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:04 smithi152 ceph-mon[34775]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:04 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3015174127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa145811-9dfa-468b-a005-75eb2042368d"}]: dispatch 2024-08-17T16:53:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:04 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3015174127' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa145811-9dfa-468b-a005-75eb2042368d"}]': finished 2024-08-17T16:53:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:04 smithi152 ceph-mon[34775]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T16:53:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:04 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[31182]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3015174127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa145811-9dfa-468b-a005-75eb2042368d"}]: dispatch 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3015174127' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa145811-9dfa-468b-a005-75eb2042368d"}]': finished 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[31182]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[40020]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3015174127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa145811-9dfa-468b-a005-75eb2042368d"}]: dispatch 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3015174127' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa145811-9dfa-468b-a005-75eb2042368d"}]': finished 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[40020]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T16:53:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:04 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:05 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2445131000' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:06.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:05 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2445131000' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:06.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:05 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2445131000' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:06 smithi152 ceph-mon[34775]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:06 smithi142 ceph-mon[31182]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:07.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:06 smithi142 ceph-mon[40020]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:08 smithi152 ceph-mon[34775]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:08 smithi142 ceph-mon[31182]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:08 smithi142 ceph-mon[40020]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:10 smithi142 ceph-mon[31182]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:10 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T16:53:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:10 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:10 smithi142 ceph-mon[40020]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:10 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T16:53:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:10 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:10 smithi152 ceph-mon[34775]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:10 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T16:53:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:10 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:11 smithi152 ceph-mon[34775]: Deploying daemon osd.1 on smithi142 2024-08-17T16:53:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:11 smithi142 ceph-mon[31182]: Deploying daemon osd.1 on smithi142 2024-08-17T16:53:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:11 smithi142 ceph-mon[40020]: Deploying daemon osd.1 on smithi142 2024-08-17T16:53:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:12 smithi152 ceph-mon[34775]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:13.106 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:12 smithi142 ceph-mon[31182]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:13.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:12 smithi142 ceph-mon[40020]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:14 smithi152 ceph-mon[34775]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:15.092 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:14 smithi142 ceph-mon[31182]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:15.093 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:14 smithi142 ceph-mon[40020]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:16.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:15 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:16.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:15 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:16.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:15 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:15 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:15 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:15 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:15 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:15 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:15 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:16 smithi142 ceph-mon[31182]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:16 smithi142 ceph-mon[40020]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:16 smithi152 ceph-mon[34775]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:18.784 INFO:teuthology.orchestra.run.smithi142.stdout:Created osd(s) 1 on host 'smithi142' 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:19.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.433 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:53:19.433 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:53:19.433 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:19.433 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.433 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.455 DEBUG:teuthology.orchestra.run.smithi142:osd.1> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.1.service 2024-08-17T16:53:19.457 INFO:tasks.cephadm:Deploying osd.2 on smithi142 with /dev/vg_nvme/lv_2... 2024-08-17T16:53:19.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T16:53:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:19.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:53:19.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:53:19.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:19.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:19.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:20.254 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:20 smithi152 ceph-mon[34775]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T16:53:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:20 smithi152 ceph-mon[34775]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T16:53:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:20 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:20 smithi152 ceph-mon[34775]: from='osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:20 smithi152 ceph-mon[34775]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:20.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[31182]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T16:53:20.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[31182]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T16:53:20.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:20.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[31182]: from='osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:20.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[31182]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:20.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[40020]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T16:53:20.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[40020]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T16:53:20.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:20.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[40020]: from='osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:20.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-mon[40020]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:20.605 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[50321]: 2024-08-17T16:53:20.149+0000 7f4853ad4640 -1 osd.1 0 waiting for initial osdmap 2024-08-17T16:53:20.605 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[50321]: 2024-08-17T16:53:20.151+0000 7f485090f640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:53:21.353 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[31182]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:21.353 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[31182]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:53:21.353 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[31182]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[40020]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[40020]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[40020]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:21.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:21.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:21 smithi152 ceph-mon[34775]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:21.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:21 smithi152 ceph-mon[34775]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:53:21.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:21 smithi152 ceph-mon[34775]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T16:53:21.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:21 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:21.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:21 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[31182]: osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993] boot 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[31182]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:53:22.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:53:22.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[40020]: osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993] boot 2024-08-17T16:53:22.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[40020]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T16:53:22.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:22 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:53:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:22 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:53:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:22 smithi152 ceph-mon[34775]: osd.1 [v2:172.21.15.142:6810/2041524993,v1:172.21.15.142:6811/2041524993] boot 2024-08-17T16:53:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:22 smithi152 ceph-mon[34775]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T16:53:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:53:23.447 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:23 smithi142 ceph-mon[40020]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:23.447 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:23 smithi142 ceph-mon[40020]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T16:53:23.447 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:23 smithi142 ceph-mon[31182]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:23.447 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:23 smithi142 ceph-mon[31182]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T16:53:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:23 smithi152 ceph-mon[34775]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:53:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:23 smithi152 ceph-mon[34775]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T16:53:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi142 2024-08-17T16:53:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi142 to 8032M 2024-08-17T16:53:24.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:24.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:24.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:24 smithi152 ceph-mon[34775]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi142 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi142 to 8032M 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:24.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[31182]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi142 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi142 to 8032M 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:24.863 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:24.863 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:24.863 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:24 smithi142 ceph-mon[40020]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:24.865 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:53:24.887 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi142:vg_nvme/lv_2 2024-08-17T16:53:25.138 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:26 smithi152 ceph-mon[34775]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:27.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:26 smithi142 ceph-mon[31182]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:26 smithi142 ceph-mon[40020]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[31182]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[40020]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:27 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:27 smithi152 ceph-mon[34775]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:27 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:28.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:27 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:28.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:27 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:28.841 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:28 smithi142 ceph-mon[40020]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:28.841 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:28 smithi142 ceph-mon[31182]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:28 smithi152 ceph-mon[34775]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:29 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4169878585' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]: dispatch 2024-08-17T16:53:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:29 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]: dispatch 2024-08-17T16:53:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:29 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]': finished 2024-08-17T16:53:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:29 smithi152 ceph-mon[34775]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T16:53:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:29 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:29 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/363474199' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/4169878585' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]: dispatch 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]: dispatch 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]': finished 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[31182]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/363474199' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4169878585' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]: dispatch 2024-08-17T16:53:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]: dispatch 2024-08-17T16:53:30.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63"}]': finished 2024-08-17T16:53:30.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[40020]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T16:53:30.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:30.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:29 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/363474199' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:31 smithi152 ceph-mon[34775]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:31 smithi142 ceph-mon[31182]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:31 smithi142 ceph-mon[40020]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:33.155 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:33 smithi142 ceph-mon[31182]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:33.155 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:33 smithi142 ceph-mon[40020]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:33 smithi152 ceph-mon[34775]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:35.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:35 smithi142 ceph-mon[31182]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:35.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T16:53:35.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:35.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:35 smithi142 ceph-mon[40020]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:35.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T16:53:35.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:35 smithi152 ceph-mon[34775]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T16:53:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:36.514 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:36 smithi142 ceph-mon[31182]: Deploying daemon osd.2 on smithi142 2024-08-17T16:53:36.515 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:36 smithi142 ceph-mon[40020]: Deploying daemon osd.2 on smithi142 2024-08-17T16:53:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:36 smithi152 ceph-mon[34775]: Deploying daemon osd.2 on smithi142 2024-08-17T16:53:37.362 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:37 smithi142 ceph-mon[40020]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:37.363 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:37 smithi142 ceph-mon[31182]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:37 smithi152 ceph-mon[34775]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:39.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:39 smithi142 ceph-mon[40020]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:39.538 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:39 smithi142 ceph-mon[31182]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:39.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:39 smithi152 ceph-mon[34775]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:40.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:40.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:40.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:40.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:41.457 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:41 smithi142 ceph-mon[40020]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:41.457 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:41 smithi142 ceph-mon[31182]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:41 smithi152 ceph-mon[34775]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:43.434 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:43 smithi142 ceph-mon[31182]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:43.434 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:43 smithi142 ceph-mon[40020]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:43.474 INFO:teuthology.orchestra.run.smithi142.stdout:Created osd(s) 2 on host 'smithi142' 2024-08-17T16:53:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:43 smithi152 ceph-mon[34775]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:44.154 DEBUG:teuthology.orchestra.run.smithi142:osd.2> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.2.service 2024-08-17T16:53:44.157 INFO:tasks.cephadm:Deploying osd.3 on smithi142 with /dev/vg_nvme/lv_1... 2024-08-17T16:53:44.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T16:53:44.448 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[40020]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:53:44.448 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.448 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.448 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.449 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[31182]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:53:44.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:44 smithi152 ceph-mon[34775]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:53:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:44.904 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[31182]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[31182]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[31182]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[31182]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[40020]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[40020]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T16:53:45.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[40020]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T16:53:45.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:45.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-mon[40020]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:45.374 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[56556]: 2024-08-17T16:53:45.180+0000 7f5ee3d15640 -1 osd.2 0 waiting for initial osdmap 2024-08-17T16:53:45.374 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:45 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[56556]: 2024-08-17T16:53:45.182+0000 7f5edeb4c640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:53:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:45 smithi152 ceph-mon[34775]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:45 smithi152 ceph-mon[34775]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T16:53:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:45 smithi152 ceph-mon[34775]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T16:53:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:45 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:45 smithi152 ceph-mon[34775]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:53:46.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:53:46.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:53:46.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[40020]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:53:46.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[40020]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T16:53:46.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:46.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:46.292 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:53:46.292 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:53:46.292 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[31182]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:53:46.292 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[31182]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T16:53:46.292 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:46.292 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:46 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:46 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:53:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:46 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:53:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:46 smithi152 ceph-mon[34775]: from='osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:53:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:46 smithi152 ceph-mon[34775]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T16:53:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:46 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:46 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:47 smithi152 ceph-mon[34775]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:47 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:47 smithi152 ceph-mon[34775]: osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488] boot 2024-08-17T16:53:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:47 smithi152 ceph-mon[34775]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T16:53:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:47 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:47.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[40020]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:47.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:47.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[40020]: osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488] boot 2024-08-17T16:53:47.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[40020]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T16:53:47.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:47.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[31182]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T16:53:47.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:47.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[31182]: osd.2 [v2:172.21.15.142:6818/3610394488,v1:172.21.15.142:6819/3610394488] boot 2024-08-17T16:53:47.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[31182]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T16:53:47.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:47 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:48.439 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.440 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:48.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:48 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:48.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:48.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:48.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:48 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[31182]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi142 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi142 to 5354M 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[31182]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[40020]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:49.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi142 2024-08-17T16:53:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi142 to 5354M 2024-08-17T16:53:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-17T16:53:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[40020]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T16:53:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:49 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:53:49.385 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:53:49.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi142:vg_nvme/lv_1 2024-08-17T16:53:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:49 smithi152 ceph-mon[34775]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:49 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi142 2024-08-17T16:53:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:49 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi142 to 5354M 2024-08-17T16:53:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:49 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-17T16:53:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:49 smithi152 ceph-mon[34775]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T16:53:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:49 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:53:49.683 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:50.065 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:49 smithi142 sudo[60863]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:53:50.065 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:49 smithi142 sudo[60863]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:50.066 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:49 smithi142 sudo[60863]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:50.066 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:49 smithi142 sudo[60863]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:50.345 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:50 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:53:50.345 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:50 smithi142 ceph-mon[31182]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T16:53:50.345 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:50 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:50.345 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:50 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:53:50.346 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:50 smithi142 ceph-mon[40020]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T16:53:50.346 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:50 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:50.346 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60892]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:53:50.346 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60892]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:50.346 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60892]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:50.346 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60892]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:50 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:53:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:50 smithi152 ceph-mon[34775]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T16:53:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:50 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:50.602 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60907]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:53:50.602 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60907]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:50.603 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60907]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:50.603 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60907]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:50.858 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60918]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:53:50.858 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60918]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:50.858 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60918]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:50.859 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60918]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:51.110 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60929]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:53:51.110 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60929]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:51.110 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60929]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:51.110 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:50 smithi142 sudo[60929]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:51.371 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60959]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:53:51.372 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60959]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:51.372 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60959]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:51.372 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60959]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:51.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60992]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:53:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60992]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60992]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60992]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:52.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[40020]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:52.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[40020]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T16:53:52.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[40020]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T16:53:52.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[40020]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60996]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60996]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60996]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 sudo[60996]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[31182]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[31182]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[31182]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T16:53:52.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:51 smithi142 ceph-mon[31182]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:52.130 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:51 smithi152 sudo[37533]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:53:52.130 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:51 smithi152 sudo[37533]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:52.130 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:51 smithi152 sudo[37533]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:52.130 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:51 smithi152 sudo[37533]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 sudo[37536]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 sudo[37536]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 sudo[37536]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 sudo[37536]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T16:53:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T16:53:52.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:52.666 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61160]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:53:52.666 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61160]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:52.667 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61160]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:52.667 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61160]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.032 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61171]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61171]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61171]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:53:53.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 sudo[61171]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:53.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:53.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:53.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:53.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi142:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:53:53.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:53:53.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:53:53.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:53:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:53 smithi142 ceph-mon[31182]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:53 smithi142 ceph-mon[31182]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:53 smithi142 ceph-mon[40020]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:53 smithi142 ceph-mon[40020]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:53 smithi152 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:53:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:53 smithi152 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:53:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3277336462' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]: dispatch 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]: dispatch 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]': finished 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1992235346' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3277336462' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]: dispatch 2024-08-17T16:53:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]: dispatch 2024-08-17T16:53:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]': finished 2024-08-17T16:53:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T16:53:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:53:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:54 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1992235346' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3277336462' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]: dispatch 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]: dispatch 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9b26ffd-2955-4af8-90c8-5473d3c78fa4"}]': finished 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:53:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:54 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1992235346' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:53:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:56 smithi142 ceph-mon[31182]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:56 smithi142 ceph-mon[40020]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:56 smithi152 ceph-mon[34775]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:53:58.972 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:53:58 smithi142 ceph-mon[31182]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 66 KiB/s wr, 7 op/s 2024-08-17T16:53:58.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:53:58 smithi142 ceph-mon[40020]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 66 KiB/s wr, 7 op/s 2024-08-17T16:53:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:53:58 smithi152 ceph-mon[34775]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 66 KiB/s wr, 7 op/s 2024-08-17T16:54:00.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[40020]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-08-17T16:54:00.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T16:54:00.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:00.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[40020]: Deploying daemon osd.3 on smithi142 2024-08-17T16:54:00.992 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[31182]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-08-17T16:54:00.993 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T16:54:00.993 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:00.993 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:00 smithi142 ceph-mon[31182]: Deploying daemon osd.3 on smithi142 2024-08-17T16:54:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:00 smithi152 ceph-mon[34775]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-08-17T16:54:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:00 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T16:54:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:00 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:00 smithi152 ceph-mon[34775]: Deploying daemon osd.3 on smithi142 2024-08-17T16:54:03.059 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:02 smithi142 ceph-mon[31182]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.4 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-08-17T16:54:03.060 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:02 smithi142 ceph-mon[40020]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.4 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-08-17T16:54:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:02 smithi152 ceph-mon[34775]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.4 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-08-17T16:54:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:04 smithi152 ceph-mon[34775]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.1 KiB/s rd, 26 KiB/s wr, 2 op/s 2024-08-17T16:54:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:04 smithi142 ceph-mon[31182]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.1 KiB/s rd, 26 KiB/s wr, 2 op/s 2024-08-17T16:54:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:04 smithi142 ceph-mon[40020]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.1 KiB/s rd, 26 KiB/s wr, 2 op/s 2024-08-17T16:54:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:05 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:05 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:05 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:05 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:05 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:05 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:05 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:05 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:05 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:07.048 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:06 smithi142 ceph-mon[40020]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 957 B/s rd, 22 KiB/s wr, 2 op/s 2024-08-17T16:54:07.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:06 smithi142 ceph-mon[31182]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 957 B/s rd, 22 KiB/s wr, 2 op/s 2024-08-17T16:54:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:06 smithi152 ceph-mon[34775]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 957 B/s rd, 22 KiB/s wr, 2 op/s 2024-08-17T16:54:08.129 INFO:teuthology.orchestra.run.smithi142.stdout:Created osd(s) 3 on host 'smithi142' 2024-08-17T16:54:08.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 938 B/s rd, 22 KiB/s wr, 2 op/s 2024-08-17T16:54:08.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:08.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:08.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 938 B/s rd, 22 KiB/s wr, 2 op/s 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:54:08.725 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:08.725 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.725 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.748 DEBUG:teuthology.orchestra.run.smithi142:osd.3> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.3.service 2024-08-17T16:54:08.751 INFO:tasks.cephadm:Deploying osd.4 on smithi152 with /dev/vg_nvme/lv_4... 2024-08-17T16:54:08.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T16:54:08.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 938 B/s rd, 22 KiB/s wr, 2 op/s 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:09.010 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[40020]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[40020]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[40020]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[31182]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[31182]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T16:54:09.988 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:09.989 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-mon[31182]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:54:09.989 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[63049]: 2024-08-17T16:54:09.807+0000 7f0b646e4640 -1 osd.3 0 waiting for initial osdmap 2024-08-17T16:54:09.989 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 16:54:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[63049]: 2024-08-17T16:54:09.812+0000 7f0b5fd1c640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:54:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:09 smithi152 ceph-mon[34775]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T16:54:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:09 smithi152 ceph-mon[34775]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T16:54:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:09 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:09 smithi152 ceph-mon[34775]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]: dispatch 2024-08-17T16:54:11.035 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:10 smithi152 ceph-mon[34775]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:54:11.035 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:10 smithi152 ceph-mon[34775]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:54:11.035 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:10 smithi152 ceph-mon[34775]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T16:54:11.035 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:10 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:11.035 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:10 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[31182]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[31182]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[31182]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[40020]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[40020]: from='osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]}]': finished 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[40020]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:10 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:11.325 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:54:11.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_4 2024-08-17T16:54:11.634 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:54:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:54:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:54:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971] boot 2024-08-17T16:54:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T16:54:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:12.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:11 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971] boot 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: osd.3 [v2:172.21.15.142:6826/2365382971,v1:172.21.15.142:6827/2365382971] boot 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:12.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:11 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:12 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi142 2024-08-17T16:54:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:12 smithi152 ceph-mon[34775]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:12 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi142 to 4016M 2024-08-17T16:54:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:12 smithi152 ceph-mon[34775]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T16:54:13.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi142 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[31182]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi142 to 4016M 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[31182]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi142 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[40020]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi142 to 4016M 2024-08-17T16:54:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:12 smithi142 ceph-mon[40020]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T16:54:14.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:13 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:54:14.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:13 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:54:14.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:13 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:13 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:54:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:13 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:54:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:13 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:13 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:54:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:13 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:54:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:13 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:14 smithi142 ceph-mon[31182]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:14 smithi142 ceph-mon[31182]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:14 smithi142 ceph-mon[40020]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:14 smithi142 ceph-mon[40020]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:15.162 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:14 smithi152 ceph-mon[34775]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:15.163 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:14 smithi152 ceph-mon[34775]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:15 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/4108532962' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]: dispatch 2024-08-17T16:54:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:15 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]: dispatch 2024-08-17T16:54:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:15 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]': finished 2024-08-17T16:54:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:15 smithi152 ceph-mon[34775]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T16:54:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:15 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/4108532962' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]: dispatch 2024-08-17T16:54:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]: dispatch 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]': finished 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[31182]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/4108532962' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]: dispatch 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]: dispatch 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "76eecfff-f5c2-4180-a0c5-26b83e8359e8"}]': finished 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[40020]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T16:54:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:15 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:16 smithi142 ceph-mon[31182]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:16 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/3385116121' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:54:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:16 smithi142 ceph-mon[40020]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:16 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/3385116121' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:54:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:16 smithi152 ceph-mon[34775]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:16 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/3385116121' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:54:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:18 smithi152 ceph-mon[34775]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:19.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:18 smithi142 ceph-mon[40020]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:18 smithi142 ceph-mon[31182]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:20 smithi142 ceph-mon[40020]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:20 smithi142 ceph-mon[31182]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:21.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:20 smithi152 ceph-mon[34775]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:21 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T16:54:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:21 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:22.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:21 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T16:54:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:21 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T16:54:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:21 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:22 smithi142 ceph-mon[31182]: Deploying daemon osd.4 on smithi152 2024-08-17T16:54:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:22 smithi142 ceph-mon[31182]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:23.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:22 smithi142 ceph-mon[40020]: Deploying daemon osd.4 on smithi152 2024-08-17T16:54:23.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:22 smithi142 ceph-mon[40020]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:23.141 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:22 smithi152 ceph-mon[34775]: Deploying daemon osd.4 on smithi152 2024-08-17T16:54:23.141 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:22 smithi152 ceph-mon[34775]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:24.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:24 smithi152 ceph-mon[34775]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:24 smithi142 ceph-mon[31182]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:24 smithi142 ceph-mon[40020]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:26 smithi152 ceph-mon[34775]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:26 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[40020]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[31182]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:27.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:26 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:29.066 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:28 smithi152 ceph-mon[34775]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:29.098 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 4 on host 'smithi152' 2024-08-17T16:54:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:28 smithi142 ceph-mon[31182]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:28 smithi142 ceph-mon[40020]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:29.675 DEBUG:teuthology.orchestra.run.smithi152:osd.4> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.4.service 2024-08-17T16:54:29.677 INFO:tasks.cephadm:Deploying osd.5 on smithi152 with /dev/vg_nvme/lv_3... 2024-08-17T16:54:29.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T16:54:29.975 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:54:30.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:30 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:30 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:30 smithi152 ceph-mon[34775]: from='osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:54:30.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:30 smithi152 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:54:30.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:30 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:30 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[31182]: from='osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[31182]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[40020]: from='osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[40020]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:30 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[31182]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[31182]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[31182]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[31182]: from='osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[31182]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[40020]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[40020]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T16:54:31.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[40020]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T16:54:31.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:31.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[40020]: from='osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:31.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:31 smithi142 ceph-mon[40020]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:31.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-mon[34775]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:31.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T16:54:31.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-mon[34775]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T16:54:31.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:31.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-mon[34775]: from='osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:31.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:31.438 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T16:54:31.113+0000 7f6240ada640 -1 osd.4 0 waiting for initial osdmap 2024-08-17T16:54:31.438 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 16:54:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T16:54:31.118+0000 7f623d915640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:54:32.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:54:32.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T16:54:32.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822] boot 2024-08-17T16:54:32.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T16:54:32.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:54:32.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T16:54:32.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822] boot 2024-08-17T16:54:32.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T16:54:32.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:32 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.469 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:54:32.469 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T16:54:32.470 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.470 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:32.470 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: osd.4 [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822] boot 2024-08-17T16:54:32.470 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T16:54:32.470 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:32 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:54:33.362 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:54:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:54:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:54:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[31182]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[31182]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T16:54:33.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:54:33.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:54:33.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[40020]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:33.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:33 smithi142 ceph-mon[40020]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T16:54:33.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:33 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:54:33.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:33 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:54:33.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:33 smithi152 ceph-mon[34775]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T16:54:33.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:33 smithi152 ceph-mon[34775]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T16:54:33.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_3 2024-08-17T16:54:33.917 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:54:34.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:34 smithi152 ceph-mon[34775]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:34.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:34 smithi152 ceph-mon[34775]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T16:54:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:34 smithi142 ceph-mon[31182]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:34 smithi142 ceph-mon[31182]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T16:54:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:34 smithi142 ceph-mon[40020]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:34 smithi142 ceph-mon[40020]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi152 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi152 to 17088M 2024-08-17T16:54:35.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:35.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:35 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi152 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi152 to 17088M 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:36.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi152 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi152 to 17088M 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:36.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:35 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:36 smithi152 ceph-mon[34775]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:36 smithi152 ceph-mon[34775]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T16:54:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:36 smithi152 ceph-mon[34775]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:36 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:54:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:36 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:54:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:36 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[40020]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[40020]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[40020]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[31182]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[31182]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[31182]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:54:37.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:54:37.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:36 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:38 smithi152 ceph-mon[34775]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-08-17T16:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:38 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/1972534380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]: dispatch 2024-08-17T16:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:38 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]: dispatch 2024-08-17T16:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:38 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]': finished 2024-08-17T16:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:38 smithi152 ceph-mon[34775]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T16:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:38 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[31182]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/1972534380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]: dispatch 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]: dispatch 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]': finished 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[31182]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[40020]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/1972534380' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]: dispatch 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]: dispatch 2024-08-17T16:54:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f36620dd-2b34-4322-92c9-59ff43a6799e"}]': finished 2024-08-17T16:54:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[40020]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T16:54:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:38 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:39 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/1461854383' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:54:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:39 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/1461854383' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:54:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:39 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/1461854383' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:54:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:40 smithi152 ceph-mon[34775]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-08-17T16:54:41.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:40 smithi142 ceph-mon[31182]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-08-17T16:54:41.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:40 smithi142 ceph-mon[40020]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-08-17T16:54:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:42 smithi152 ceph-mon[34775]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-08-17T16:54:43.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:42 smithi142 ceph-mon[31182]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-08-17T16:54:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:42 smithi142 ceph-mon[40020]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-08-17T16:54:44.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:44 smithi152 ceph-mon[34775]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:54:44.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T16:54:44.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:44.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:44 smithi152 ceph-mon[34775]: Deploying daemon osd.5 on smithi152 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[31182]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[31182]: Deploying daemon osd.5 on smithi152 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[40020]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:44 smithi142 ceph-mon[40020]: Deploying daemon osd.5 on smithi152 2024-08-17T16:54:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:46 smithi152 ceph-mon[34775]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:54:47.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:46 smithi142 ceph-mon[31182]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:54:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:46 smithi142 ceph-mon[40020]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:54:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:48 smithi152 ceph-mon[34775]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:49.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:48 smithi142 ceph-mon[40020]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:48 smithi142 ceph-mon[31182]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:50.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:49 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:50.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:49 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:50.029 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:49 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:50.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:49 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:50.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:49 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:50.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:49 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:50.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:49 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:50.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:49 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:50.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:49 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:51.337 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:51 smithi152 ceph-mon[34775]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:51 smithi142 ceph-mon[40020]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:51 smithi142 ceph-mon[31182]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:52.385 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 5 on host 'smithi152' 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:54:52.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:54:52.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:52 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:52.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:54:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:52 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:53.067 DEBUG:teuthology.orchestra.run.smithi152:osd.5> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5.service 2024-08-17T16:54:53.069 INFO:tasks.cephadm:Deploying osd.6 on smithi152 with /dev/vg_nvme/lv_2... 2024-08-17T16:54:53.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T16:54:53.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:53.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:53.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T16:54:53.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T16:54:53.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:53.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: from='osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:53.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:53.552 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[46487]: 2024-08-17T16:54:53.401+0000 7f6741368640 -1 osd.5 0 waiting for initial osdmap 2024-08-17T16:54:53.552 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 16:54:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[46487]: 2024-08-17T16:54:53.408+0000 7f673a19b640 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:54:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: from='osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[31182]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T16:54:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T16:54:53.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:53.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: from='osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:53.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:53 smithi142 ceph-mon[40020]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:54:54.271 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:54:54.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:54 smithi152 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:54:54.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:54 smithi152 ceph-mon[34775]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T16:54:54.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:54 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:54.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:54 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:54.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:54 smithi152 ceph-mon[34775]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[40020]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:54:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[40020]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T16:54:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[40020]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[31182]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:54:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[31182]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T16:54:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:54 smithi142 ceph-mon[31182]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T16:54:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:55 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:54:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:55 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:54:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:55 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:55 smithi152 ceph-mon[34775]: osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029] boot 2024-08-17T16:54:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:55 smithi152 ceph-mon[34775]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T16:54:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:55 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[40020]: osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029] boot 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[40020]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:54:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[31182]: osd.5 [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029] boot 2024-08-17T16:54:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[31182]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T16:54:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:55 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:54:56.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:56 smithi152 ceph-mon[34775]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T16:54:56.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:56 smithi152 ceph-mon[34775]: pgmap v127: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:54:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:56 smithi142 ceph-mon[40020]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T16:54:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:56 smithi142 ceph-mon[40020]: pgmap v127: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:54:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:56 smithi142 ceph-mon[31182]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T16:54:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:56 smithi142 ceph-mon[31182]: pgmap v127: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:54:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T16:54:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T16:54:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi152 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi152 to 8544M 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:57.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:57 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.844 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:54:57.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_2 2024-08-17T16:54:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T16:54:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi152 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi152 to 8544M 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi152 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi152 to 8544M 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:57.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:57 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:54:58.119 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:54:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:58 smithi152 ceph-mon[34775]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T16:54:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:58 smithi152 ceph-mon[34775]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:54:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:58 smithi142 ceph-mon[40020]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T16:54:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:58 smithi142 ceph-mon[40020]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:54:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:58 smithi142 ceph-mon[31182]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T16:54:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:58 smithi142 ceph-mon[31182]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:54:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:54:59 smithi152 ceph-mon[34775]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T16:54:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:54:59 smithi142 ceph-mon[40020]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T16:54:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:54:59 smithi142 ceph-mon[31182]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T16:55:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:00 smithi152 ceph-mon[34775]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:00 smithi152 ceph-mon[34775]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:00 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:55:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:00 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:55:00.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:00 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:00.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[31182]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[31182]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[40020]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[40020]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:55:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:00 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:01 smithi152 ceph-mon[34775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T16:55:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:01 smithi152 ceph-mon[34775]: Cluster is now healthy 2024-08-17T16:55:02.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:01 smithi142 ceph-mon[40020]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T16:55:02.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:01 smithi142 ceph-mon[40020]: Cluster is now healthy 2024-08-17T16:55:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:01 smithi142 ceph-mon[31182]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T16:55:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:01 smithi142 ceph-mon[31182]: Cluster is now healthy 2024-08-17T16:55:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/2876151990' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]: dispatch 2024-08-17T16:55:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]: dispatch 2024-08-17T16:55:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]': finished 2024-08-17T16:55:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T16:55:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:03.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:02 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/1337632425' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:55:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/2876151990' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]: dispatch 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]: dispatch 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]': finished 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/1337632425' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/2876151990' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]: dispatch 2024-08-17T16:55:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]: dispatch 2024-08-17T16:55:03.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4dc7a5d8-e2a2-4687-a4d8-53799e862d2b"}]': finished 2024-08-17T16:55:03.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T16:55:03.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:03.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:02 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/1337632425' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:55:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:04 smithi152 ceph-mon[34775]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-08-17T16:55:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:04 smithi142 ceph-mon[40020]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-08-17T16:55:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:04 smithi142 ceph-mon[31182]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-08-17T16:55:06.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:06 smithi152 ceph-mon[34775]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:55:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:06 smithi142 ceph-mon[31182]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:55:07.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:06 smithi142 ceph-mon[40020]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:55:08.882 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:08 smithi152 ceph-mon[34775]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-08-17T16:55:08.882 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T16:55:08.882 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:08 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:08.882 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:08 smithi152 ceph-mon[34775]: Deploying daemon osd.6 on smithi152 2024-08-17T16:55:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[40020]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[40020]: Deploying daemon osd.6 on smithi152 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[31182]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:08 smithi142 ceph-mon[31182]: Deploying daemon osd.6 on smithi152 2024-08-17T16:55:10.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:10 smithi152 ceph-mon[34775]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:55:11.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:10 smithi142 ceph-mon[40020]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:55:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:10 smithi142 ceph-mon[31182]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:55:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:12 smithi152 ceph-mon[34775]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:13.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:12 smithi142 ceph-mon[31182]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:12 smithi142 ceph-mon[40020]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:13 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:13 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:13 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:13 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:13 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:13 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:13 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:13 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:13 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:15.305 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:15 smithi152 ceph-mon[34775]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:15.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:15 smithi142 ceph-mon[31182]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:15.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:15 smithi142 ceph-mon[40020]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:16.068 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 6 on host 'smithi152' 2024-08-17T16:55:16.609 DEBUG:teuthology.orchestra.run.smithi152:osd.6> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T16:55:16.611 INFO:tasks.cephadm:Deploying osd.7 on smithi152 with /dev/vg_nvme/lv_1... 2024-08-17T16:55:16.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T16:55:16.913 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:55:17.297 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:17.297 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: from='osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:55:17.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:55:17.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:17 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:17.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: from='osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: from='osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:55:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:17.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:17 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:18.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T16:55:18.299 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-mon[34775]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T16:55:18.299 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:18.299 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-mon[34775]: from='osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:18.299 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:18.299 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[52686]: 2024-08-17T16:55:18.078+0000 7f9ccbf5e640 -1 osd.6 0 waiting for initial osdmap 2024-08-17T16:55:18.299 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 16:55:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[52686]: 2024-08-17T16:55:18.086+0000 7f9cc8d99640 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:55:18.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[31182]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T16:55:18.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[31182]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[31182]: from='osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[31182]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[40020]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[40020]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[40020]: from='osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:18.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:18 smithi142 ceph-mon[40020]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:19.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:19.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:55:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T16:55:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576] boot 2024-08-17T16:55:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T16:55:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:19 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576] boot 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:55:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T16:55:19.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:19.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: osd.6 [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576] boot 2024-08-17T16:55:19.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T16:55:19.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:19 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:55:20.267 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-08-17T16:55:20.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi152:vg_nvme/lv_1 2024-08-17T16:55:20.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:20 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:55:20.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:20 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:55:20.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:20 smithi142 ceph-mon[40020]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T16:55:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:20 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:55:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:20 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:55:20.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:20 smithi142 ceph-mon[31182]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T16:55:20.439 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:20 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:55:20.439 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:20 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:55:20.439 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:20 smithi152 ceph-mon[34775]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T16:55:20.807 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:55:21.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:21 smithi152 ceph-mon[34775]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:21.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:21 smithi152 ceph-mon[34775]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T16:55:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:21 smithi142 ceph-mon[40020]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:21 smithi142 ceph-mon[40020]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T16:55:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:21 smithi142 ceph-mon[31182]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:21 smithi142 ceph-mon[31182]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T16:55:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi152 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi152 to 5696M 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:23.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:22 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi152 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi152 to 5696M 2024-08-17T16:55:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T16:55:23.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi152 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi152 to 5696M 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:23.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:23.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:22 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:23.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:23 smithi152 ceph-mon[34775]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T16:55:23.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:23 smithi152 ceph-mon[34775]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:23.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:23 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:55:23.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:23 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:55:23.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:23 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:24.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[40020]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[40020]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[31182]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[31182]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:55:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:23 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:25.030 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:24 smithi152 ceph-mon[34775]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:24 smithi142 ceph-mon[31182]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:24 smithi142 ceph-mon[40020]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:25.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:25 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/247825251' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]: dispatch 2024-08-17T16:55:25.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:25 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]: dispatch 2024-08-17T16:55:25.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:25 smithi152 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]': finished 2024-08-17T16:55:25.910 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:25 smithi152 ceph-mon[34775]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T16:55:25.910 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:25 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:26.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/247825251' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]: dispatch 2024-08-17T16:55:26.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]: dispatch 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[31182]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]': finished 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[31182]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/247825251' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]: dispatch 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]: dispatch 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[40020]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3a1c6335-e51f-45be-afe7-fd487ca70a0a"}]': finished 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[40020]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T16:55:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:25 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:26 smithi152 ceph-mon[34775]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-08-17T16:55:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:26 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/2421999174' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:55:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:26 smithi142 ceph-mon[31182]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-08-17T16:55:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:26 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/2421999174' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:55:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:26 smithi142 ceph-mon[40020]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 125 KiB/s, 0 objects/s recovering 2024-08-17T16:55:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:26 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/2421999174' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:55:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:28 smithi152 ceph-mon[34775]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:28 smithi142 ceph-mon[40020]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:28 smithi142 ceph-mon[31182]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:30 smithi152 ceph-mon[34775]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:55:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:30 smithi142 ceph-mon[40020]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:55:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:30 smithi142 ceph-mon[31182]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:55:31.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T16:55:31.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:31 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:31.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:31 smithi152 ceph-mon[34775]: Deploying daemon osd.7 on smithi152 2024-08-17T16:55:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T16:55:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:31 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:31 smithi142 ceph-mon[40020]: Deploying daemon osd.7 on smithi152 2024-08-17T16:55:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T16:55:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:31 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:31 smithi142 ceph-mon[31182]: Deploying daemon osd.7 on smithi152 2024-08-17T16:55:33.035 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:32 smithi152 ceph-mon[34775]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-17T16:55:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:32 smithi142 ceph-mon[40020]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-17T16:55:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:32 smithi142 ceph-mon[31182]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-17T16:55:34.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:34 smithi152 ceph-mon[34775]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:55:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:34 smithi142 ceph-mon[40020]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:55:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:34 smithi142 ceph-mon[31182]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:55:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:36 smithi152 ceph-mon[34775]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:36 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:36 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:36 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[40020]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[31182]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:36 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:39.002 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:38 smithi152 ceph-mon[34775]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:38 smithi142 ceph-mon[40020]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:38 smithi142 ceph-mon[31182]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:39.306 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 7 on host 'smithi152' 2024-08-17T16:55:39.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:39 smithi152 ceph-mon[34775]: from='osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:55:39.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:39 smithi152 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:55:39.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:39 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:39.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:39 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:39.914 DEBUG:teuthology.orchestra.run.smithi152:osd.7> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T16:55:39.917 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-08-17T16:55:39.917 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd stat -f json 2024-08-17T16:55:40.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[31182]: from='osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:55:40.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[31182]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:55:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[40020]: from='osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:55:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[40020]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:55:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:39 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:40.198 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:55:41.012 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:41.012 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: from='osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T16:55:41.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.014 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: from='osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:41.014 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:41.014 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:41.014 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:40 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T16:55:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T16:55:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: from='osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:41.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-08-17T16:55:41.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:41.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:41.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[59204]: 2024-08-17T16:55:40.781+0000 7fabfc7a3640 -1 osd.7 0 waiting for initial osdmap 2024-08-17T16:55:41.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 16:55:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[59204]: 2024-08-17T16:55:40.788+0000 7fabf55d6640 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:55:41.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638] boot 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638] boot 2024-08-17T16:55:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T16:55:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:41 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:42.009 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:55:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-08-17T16:55:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T16:55:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: osd.7 [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638] boot 2024-08-17T16:55:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T16:55:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:41 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:55:42.653 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1723913741,"num_in_osds":8,"osd_in_since":1723913725,"num_remapped_pgs":1} 2024-08-17T16:55:42.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:55:42.910 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:55:43.032 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[40020]: purged_snaps scrub starts 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[40020]: purged_snaps scrub ok 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[40020]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1929022276' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[40020]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[31182]: purged_snaps scrub starts 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[31182]: purged_snaps scrub ok 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[31182]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1929022276' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:55:43.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:42 smithi142 ceph-mon[31182]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T16:55:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:42 smithi152 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T16:55:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:42 smithi152 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T16:55:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:42 smithi152 ceph-mon[34775]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T16:55:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:42 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1929022276' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:55:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:42 smithi152 ceph-mon[34775]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: Detected new or changed devices on smithi152 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi152 to 4272M 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:44.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[31182]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: Detected new or changed devices on smithi152 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi152 to 4272M 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.376 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:44 smithi142 ceph-mon[40020]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T16:55:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: Detected new or changed devices on smithi152 2024-08-17T16:55:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:55:44.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi152 to 4272M 2024-08-17T16:55:44.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:44.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:44.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:44.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:44.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:44.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:44.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:44 smithi152 ceph-mon[34775]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T16:55:44.817 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:55:44.817 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":58,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T16:55:44.738273+0000","last_up_change":"2024-08-17T16:55:41.730067+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6803","nonce":3785349577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6805","nonce":3785349577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6809","nonce":3785349577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6807","nonce":3785349577}]},"public_addr":"172.21.15.142:6803/3785349577","cluster_addr":"172.21.15.142:6805/3785349577","heartbeat_back_addr":"172.21.15.142:6809/3785349577","heartbeat_front_addr":"172.21.15.142:6807/3785349577","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2041524993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2041524993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2041524993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2041524993}]},"public_addr":"172.21.15.142:6811/2041524993","cluster_addr":"172.21.15.142:6813/2041524993","heartbeat_back_addr":"172.21.15.142:6817/2041524993","heartbeat_front_addr":"172.21.15.142:6815/2041524993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6819","nonce":3610394488}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6821","nonce":3610394488}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6825","nonce":3610394488}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6823","nonce":3610394488}]},"public_addr":"172.21.15.142:6819/3610394488","cluster_addr":"172.21.15.142:6821/3610394488","heartbeat_back_addr":"172.21.15.142:6825/3610394488","heartbeat_front_addr":"172.21.15.142:6823/3610394488","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6827","nonce":2365382971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6829","nonce":2365382971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6833","nonce":2365382971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6831","nonce":2365382971}]},"public_addr":"172.21.15.142:6827/2365382971","cluster_addr":"172.21.15.142:6829/2365382971","heartbeat_back_addr":"172.21.15.142:6833/2365382971","heartbeat_front_addr":"172.21.15.142:6831/2365382971","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:55:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:45 smithi142 ceph-mon[31182]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:45 smithi142 ceph-mon[31182]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T16:55:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:45 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1273882933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:55:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:45 smithi142 ceph-mon[40020]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:45 smithi142 ceph-mon[40020]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T16:55:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:45 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1273882933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:55:45.513 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-08-17T16:53:47.946475+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-08-17T16:55:45.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-08-17T16:55:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:45 smithi152 ceph-mon[34775]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:45 smithi152 ceph-mon[34775]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T16:55:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:45 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1273882933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:55:45.769 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:55:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:46 smithi142 ceph-mon[40020]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T16:55:46.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:46 smithi142 ceph-mon[40020]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T16:55:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:46 smithi142 ceph-mon[31182]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T16:55:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:46 smithi142 ceph-mon[31182]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T16:55:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:46 smithi152 ceph-mon[34775]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T16:55:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:46 smithi152 ceph-mon[34775]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T16:55:47.540 INFO:teuthology.orchestra.run.smithi142.stdout:pg_num: 1 2024-08-17T16:55:47.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:47 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1951220263' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T16:55:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:47 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1951220263' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T16:55:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:47 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1951220263' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T16:55:48.294 INFO:tasks.cephadm:Adding prometheus.a on smithi152 2024-08-17T16:55:48.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply prometheus '1;smithi152=a' 2024-08-17T16:55:48.563 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:55:48.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:48 smithi152 ceph-mon[34775]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-08-17T16:55:49.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:48 smithi142 ceph-mon[31182]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-08-17T16:55:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:48 smithi142 ceph-mon[40020]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-08-17T16:55:50.231 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled prometheus update... 2024-08-17T16:55:50.809 DEBUG:teuthology.orchestra.run.smithi152:prometheus.a> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service 2024-08-17T16:55:50.811 INFO:tasks.cephadm:Adding node-exporter.a on smithi142 2024-08-17T16:55:50.811 INFO:tasks.cephadm:Adding node-exporter.b on smithi152 2024-08-17T16:55:50.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply node-exporter '2;smithi142=a;smithi152=b' 2024-08-17T16:55:51.122 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: Saving service prometheus spec with placement smithi152=a;count:1 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:51.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:51 smithi152 ceph-mon[34775]: Deploying daemon prometheus.a on smithi152 2024-08-17T16:55:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: Saving service prometheus spec with placement smithi152=a;count:1 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[40020]: Deploying daemon prometheus.a on smithi152 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: Saving service prometheus spec with placement smithi152=a;count:1 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:51.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:51.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:51.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:51 smithi142 ceph-mon[31182]: Deploying daemon prometheus.a on smithi152 2024-08-17T16:55:53.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:53 smithi152 ceph-mon[34775]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T16:55:53.616 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled node-exporter update... 2024-08-17T16:55:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:53 smithi142 ceph-mon[31182]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T16:55:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:53 smithi142 ceph-mon[40020]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T16:55:54.605 DEBUG:teuthology.orchestra.run.smithi142:node-exporter.a> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service 2024-08-17T16:55:54.609 DEBUG:teuthology.orchestra.run.smithi152:node-exporter.b> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service 2024-08-17T16:55:54.611 INFO:tasks.cephadm:Adding alertmanager.a on smithi142 2024-08-17T16:55:54.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply alertmanager '1;smithi142=a' 2024-08-17T16:55:54.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:54 smithi152 ceph-mon[34775]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi142=a;smithi152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:54.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:54 smithi152 ceph-mon[34775]: Saving service node-exporter spec with placement smithi142=a;smithi152=b;count:2 2024-08-17T16:55:54.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:54 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:54.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:54 smithi152 ceph-mon[34775]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[31182]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi142=a;smithi152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[31182]: Saving service node-exporter spec with placement smithi142=a;smithi152=b;count:2 2024-08-17T16:55:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[31182]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[40020]: from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi142=a;smithi152=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[40020]: Saving service node-exporter spec with placement smithi142=a;smithi152=b;count:2 2024-08-17T16:55:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:54 smithi142 ceph-mon[40020]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:54.925 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:55:56.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:56 smithi152 ceph-mon[34775]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:57.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:56 smithi142 ceph-mon[31182]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:56 smithi142 ceph-mon[40020]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:57.256 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled alertmanager update... 2024-08-17T16:55:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:58 smithi152 ceph-mon[34775]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi142=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:58 smithi152 ceph-mon[34775]: Saving service alertmanager spec with placement smithi142=a;count:1 2024-08-17T16:55:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:58 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:58 smithi142 ceph-mon[31182]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi142=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:58 smithi142 ceph-mon[31182]: Saving service alertmanager spec with placement smithi142=a;count:1 2024-08-17T16:55:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:58 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:58.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:58 smithi142 ceph-mon[40020]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi142=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:58.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:58 smithi142 ceph-mon[40020]: Saving service alertmanager spec with placement smithi142=a;count:1 2024-08-17T16:55:58.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:58 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:55:58.950 DEBUG:teuthology.orchestra.run.smithi142:alertmanager.a> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service 2024-08-17T16:55:58.953 INFO:tasks.cephadm:Adding grafana.a on smithi152 2024-08-17T16:55:58.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply grafana '1;smithi152=a' 2024-08-17T16:55:59.513 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:55:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:55:59 smithi152 ceph-mon[34775]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:55:59 smithi142 ceph-mon[40020]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:55:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:55:59 smithi142 ceph-mon[31182]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:01 smithi152 ceph-mon[34775]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:01.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:01 smithi142 ceph-mon[31182]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:01 smithi142 ceph-mon[40020]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:01.982 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled grafana update... 2024-08-17T16:56:02.791 DEBUG:teuthology.orchestra.run.smithi152:grafana.a> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service 2024-08-17T16:56:02.794 INFO:tasks.cephadm:Setting up client nodes... 2024-08-17T16:56:02.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T16:56:03.056 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:03.090 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:02 smithi152 ceph-mon[34775]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:03.090 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:02 smithi152 ceph-mon[34775]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:03.090 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:02 smithi152 ceph-mon[34775]: Saving service grafana spec with placement smithi152=a;count:1 2024-08-17T16:56:03.090 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:02 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[40020]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[40020]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[40020]: Saving service grafana spec with placement smithi152=a;count:1 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[31182]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[31182]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[31182]: Saving service grafana spec with placement smithi152=a;count:1 2024-08-17T16:56:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:02 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:04.804 INFO:teuthology.orchestra.run.smithi142.stdout:[client.0] 2024-08-17T16:56:04.805 INFO:teuthology.orchestra.run.smithi142.stdout: key = AQAk1sBmqAmxLxAAnL7gFa8vws8MOonLaavOJw== 2024-08-17T16:56:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:04 smithi152 ceph-mon[34775]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:04 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4207347153' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:56:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:04 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4207347153' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:56:05.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:04 smithi152 systemd[1]: Starting Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:04 smithi142 ceph-mon[31182]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:04 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/4207347153' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:56:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:04 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/4207347153' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:56:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:04 smithi142 ceph-mon[40020]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:04 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4207347153' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:56:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:04 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4207347153' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:56:05.539 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T16:56:05.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-08-17T16:56:05.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-17T16:56:05.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:05 smithi152 podman[64524]: 2024-08-17 16:56:05.109743367 +0000 UTC m=+0.017987708 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T16:56:05.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T16:56:06.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 podman[64524]: 2024-08-17 16:56:06.213233389 +0000 UTC m=+1.121477718 container create a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:06.965 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:56:07.091 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-mon[34775]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 podman[64524]: 2024-08-17 16:56:06.832464738 +0000 UTC m=+1.740709081 container init a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 podman[64524]: 2024-08-17 16:56:06.838520792 +0000 UTC m=+1.746765122 container start a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.864Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.864Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.864Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi152 (none))" 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.864Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.864Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.865Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:56:07.092 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.866Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.867Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.867Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.869Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.869Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.6µs 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.869Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.870Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.870Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.189µs wal_replay_duration=161.529µs wbl_replay_duration=121ns total_replay_duration=210.477µs 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.870Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.870Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.870Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.885Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.082799ms db_storage=848ns remote_storage=1.11µs web_handler=359ns query_engine=952ns scrape=657.695µs scrape_sd=18.721µs notify=1.154µs notify_sd=1.464µs rules=14.177443ms tracing=5.781µs 2024-08-17T16:56:07.093 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.885Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T16:56:07.094 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:56:06.885Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:56:07.094 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 bash[64524]: a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 2024-08-17T16:56:07.094 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:56:06 smithi152 systemd[1]: Started Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:06 smithi142 ceph-mon[40020]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:06 smithi142 ceph-mon[31182]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:08.150 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:07 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.150 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:07 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.151 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:07 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.151 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:07 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T16:56:08.151 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ignoring --setuser ceph since I am not root 2024-08-17T16:56:08.151 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ignoring --setgroup ceph since I am not root 2024-08-17T16:56:08.151 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-mgr[36866]: -- 172.21.15.152:0/2049365821 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555ea82729c0 con 0x555ea827f000 2024-08-17T16:56:08.151 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:08.148+0000 7f923343aec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:56:08.277 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.277 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setuser ceph since I am not root 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setgroup ceph since I am not root 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-mgr[31456]: -- 172.21.15.142:0/51289420 <== mon.2 v2:172.21.15.142:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55abf22909c0 con 0x55abf229cc00 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:08.150+0000 7f0351a38ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:56:08.278 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:08.275+0000 7f0351a38ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:56:08.279 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.279 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.279 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' 2024-08-17T16:56:08.279 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:07 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T16:56:08.459 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:08.284+0000 7f923343aec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:56:08.459 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:08.350+0000 7f923343aec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:56:08.589 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:08.342+0000 7f0351a38ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:56:08.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:08.602+0000 7f923343aec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:56:08.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:08.662+0000 7f923343aec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:56:08.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:08.587+0000 7f0351a38ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:56:08.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:08.646+0000 7f0351a38ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:56:08.902 INFO:teuthology.orchestra.run.smithi152.stdout:[client.1] 2024-08-17T16:56:08.902 INFO:teuthology.orchestra.run.smithi152.stdout: key = AQAo1sBmSceGNRAA6uCm3WiJhlS+8RbsemxkiA== 2024-08-17T16:56:09.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-mon[34775]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T16:56:09.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-mon[34775]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T16:56:09.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/2048834304' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:56:09.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-mon[34775]: from='client.? 172.21.15.152:0/2048834304' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:56:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[40020]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[40020]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/2048834304' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[40020]: from='client.? 172.21.15.152:0/2048834304' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[31182]: from='mgr.14150 172.21.15.142:0/4100769668' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[31182]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/2048834304' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:56:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-mon[31182]: from='client.? 172.21.15.152:0/2048834304' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:56:09.513 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-08-17T16:56:09.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-08-17T16:56:09.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-17T16:56:09.553 INFO:tasks.cephadm:Creating RBD pool 2024-08-17T16:56:09.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create rbd 8 2024-08-17T16:56:09.770 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:09.600+0000 7f923343aec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:56:09.770 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:09.713+0000 7f923343aec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:56:09.770 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:09.768+0000 7f923343aec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:56:09.817 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:09.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:09.535+0000 7f0351a38ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:56:09.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:09.653+0000 7f0351a38ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:56:09.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:09.711+0000 7f0351a38ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:56:10.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:09.957+0000 7f923343aec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:56:10.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:10.017+0000 7f923343aec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:56:10.170 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:09.913+0000 7f0351a38ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:56:10.170 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:09.974+0000 7f0351a38ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:56:10.170 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:10.039+0000 7f0351a38ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:56:10.170 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:10.168+0000 7f0351a38ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:56:10.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:10.083+0000 7f923343aec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:56:10.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:10.207+0000 7f923343aec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:56:10.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:10.585+0000 7f0351a38ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:56:10.909 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:10.624+0000 7f923343aec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:56:11.223 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:10.874+0000 7f0351a38ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:56:11.224 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:10.938+0000 7f0351a38ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:56:11.224 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.065+0000 7f0351a38ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:56:11.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:10.908+0000 7f923343aec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:56:11.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:10.972+0000 7f923343aec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:56:11.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.097+0000 7f923343aec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:56:11.481 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.324+0000 7f0351a38ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:56:11.481 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.390+0000 7f0351a38ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:56:11.659 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.351+0000 7f923343aec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:56:11.659 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.412+0000 7f923343aec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:56:11.659 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.594+0000 7f923343aec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:56:11.786 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.580+0000 7f0351a38ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:56:11.786 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.643+0000 7f0351a38ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:56:11.786 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.724+0000 7f0351a38ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:56:11.786 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:11.783+0000 7f0351a38ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:56:12.043 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'rbd' created 2024-08-17T16:56:12.045 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.658+0000 7f923343aec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:56:12.045 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.738+0000 7f923343aec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:56:12.045 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:11.794+0000 7f923343aec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:56:12.317 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4010358629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:56:12.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/4010358629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:56:12.319 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4010358629' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:56:12.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:12.318+0000 7f923343aec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:56:12.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:12.452+0000 7f923343aec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: [17/Aug/2024:16:56:12] ENGINE Bus STARTING 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: CherryPy Checker: 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: The Application mounted at '' has an empty config. 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:56:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: [17/Aug/2024:16:56:12] ENGINE Serving on http://:::9283 2024-08-17T16:56:12.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: [17/Aug/2024:16:56:12] ENGINE Bus STARTED 2024-08-17T16:56:12.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:12.316+0000 7f0351a38ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:56:12.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:12.452+0000 7f0351a38ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:56:12.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:12] ENGINE Bus STARTING 2024-08-17T16:56:12.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: CherryPy Checker: 2024-08-17T16:56:12.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: The Application mounted at '' has an empty config. 2024-08-17T16:56:12.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:12.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- rbd pool init rbd 2024-08-17T16:56:12.978 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:13.046 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:12] ENGINE Serving on http://:::9283 2024-08-17T16:56:13.046 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:12] ENGINE Bus STARTED 2024-08-17T16:56:13.047 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/4010358629' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T16:56:13.047 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T16:56:13.047 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: Active manager daemon y restarted 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: Activating manager daemon y 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: Standby manager daemon x restarted 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: Standby manager daemon x started 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: mgrmap e17: y(active, starting, since 0.00968642s), standbys: x 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:56:13.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:56:13.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: Manager daemon y is now available 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4010358629' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: Active manager daemon y restarted 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: Activating manager daemon y 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:56:13.298 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: Standby manager daemon x restarted 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: Standby manager daemon x started 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: mgrmap e17: y(active, starting, since 0.00968642s), standbys: x 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:56:13.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: Manager daemon y is now available 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:13.300 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:13.301 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:13.301 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4010358629' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T16:56:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T16:56:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: Active manager daemon y restarted 2024-08-17T16:56:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: Activating manager daemon y 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: mgrmap e17: y(active, starting, since 0.00968642s), standbys: x 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1967396903' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:56:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:56:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: Manager daemon y is now available 2024-08-17T16:56:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:14.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[40020]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T16:56:14.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[40020]: mgrmap e18: y(active, since 1.0154s), standbys: x 2024-08-17T16:56:14.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[40020]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:14.630 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[31182]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[31182]: mgrmap e18: y(active, since 1.0154s), standbys: x 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[31182]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:14.375+0000 7f02f4793640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:56:14] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: Traceback (most recent call last): 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:14.631 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._connections.run(self.expiration_interval) 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._run(expiration_interval) 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: s = self.context.wrap_socket( 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: return self.sslsocket_class._create( 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:14.632 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self.do_handshake() 2024-08-17T16:56:14.633 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:14.633 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: self._sslobj.do_handshake() 2024-08-17T16:56:14.633 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:14.633 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T16:56:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:14 smithi152 ceph-mon[34775]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T16:56:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:14 smithi152 ceph-mon[34775]: mgrmap e18: y(active, since 1.0154s), standbys: x 2024-08-17T16:56:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:14 smithi152 ceph-mon[34775]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:14 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: [17/Aug/2024:16:56:14] ENGINE Bus STARTING 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: [17/Aug/2024:16:56:14] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: [17/Aug/2024:16:56:14] ENGINE Bus STARTED 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: [17/Aug/2024:16:56:14] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: Traceback (most recent call last): 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._run(expiration_interval) 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:15.824 INFO:journalctl@ceph.mon.b.smithi152.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: s = self.context.wrap_socket( 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: return self.sslsocket_class._create( 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: self.do_handshake() 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._sslobj.do_handshake() 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:15.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:56:15.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:15.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1004367306' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:56:15.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:56:15.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:15 smithi152 ceph-mon[34775]: mgrmap e19: y(active, since 2s), standbys: x 2024-08-17T16:56:15.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: [17/Aug/2024:16:56:14] ENGINE Bus STARTING 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: [17/Aug/2024:16:56:14] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: [17/Aug/2024:16:56:14] ENGINE Bus STARTED 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: [17/Aug/2024:16:56:14] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: Traceback (most recent call last): 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._run(expiration_interval) 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:15.874 INFO:journalctl@ceph.mon.c.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: self.do_handshake() 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:15.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1004367306' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[40020]: mgrmap e19: y(active, since 2s), standbys: x 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: [17/Aug/2024:16:56:14] ENGINE Bus STARTING 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: [17/Aug/2024:16:56:14] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T16:56:15.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: [17/Aug/2024:16:56:14] ENGINE Bus STARTED 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: [17/Aug/2024:16:56:14] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: Traceback (most recent call last): 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._run(expiration_interval) 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:15.877 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: self.do_handshake() 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1004367306' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:56:15.878 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:56:15.879 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.879 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:15.879 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:15 smithi142 ceph-mon[31182]: mgrmap e19: y(active, since 2s), standbys: x 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi152 to 4016M 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:16.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:16 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:16.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi152 to 4016M 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:16.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi152 to 4016M 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:16.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:16 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:17.792 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:56:17.792 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:56:17.792 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:17.792 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: mgrmap e20: y(active, since 4s), standbys: x 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:56:17.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[31182]: Cluster is now healthy 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:17.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: mgrmap e20: y(active, since 4s), standbys: x 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:56:17.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:17 smithi142 ceph-mon[40020]: Cluster is now healthy 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T16:56:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: mgrmap e20: y(active, since 4s), standbys: x 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:56:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:17 smithi152 ceph-mon[34775]: Cluster is now healthy 2024-08-17T16:56:18.282 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph config log 1 --format=json 2024-08-17T16:56:18.574 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:18 smithi152 ceph-mon[34775]: Deploying daemon alertmanager.a on smithi142 2024-08-17T16:56:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:18 smithi152 ceph-mon[34775]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T16:56:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:18 smithi142 ceph-mon[31182]: Deploying daemon alertmanager.a on smithi142 2024-08-17T16:56:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:18 smithi142 ceph-mon[31182]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T16:56:18.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:18 smithi142 ceph-mon[40020]: Deploying daemon alertmanager.a on smithi142 2024-08-17T16:56:18.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:18 smithi142 ceph-mon[40020]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T16:56:19.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:19 smithi142 ceph-mon[31182]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:19.789 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:19 smithi142 ceph-mon[40020]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:19 smithi152 ceph-mon[34775]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:20.949 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:56:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:21 smithi152 ceph-mon[34775]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 341 B/s wr, 19 op/s 2024-08-17T16:56:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:21 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2189339545' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T16:56:21.848 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:21 smithi142 ceph-mon[40020]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 341 B/s wr, 19 op/s 2024-08-17T16:56:21.848 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:21 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2189339545' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T16:56:21.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:21 smithi142 ceph-mon[31182]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 341 B/s wr, 19 op/s 2024-08-17T16:56:21.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:21 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2189339545' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T16:56:22.645 INFO:teuthology.orchestra.run.smithi142.stdout:[{"version":18,"timestamp":"2024-08-17T16:56:15.477358+0000","name":"","changes":[{"name":"osd/host:smithi152/osd_memory_target","previous_value":"4479564800","new_value":"4211129344"}]}] 2024-08-17T16:56:22.645 INFO:tasks.ceph_manager:config epoch is 18 2024-08-17T16:56:22.645 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T16:56:22.645 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T16:56:22.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T16:56:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:22 smithi152 ceph-mon[34775]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T16:56:22.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:22 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:56:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:56:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:22 smithi142 ceph-mon[40020]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T16:56:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:22 smithi142 ceph-mon[31182]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T16:56:23.460 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:23 smithi142 ceph-mon[31182]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 341 B/s wr, 19 op/s 2024-08-17T16:56:23.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:23 smithi142 ceph-mon[40020]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 341 B/s wr, 19 op/s 2024-08-17T16:56:23.996 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:23 smithi152 ceph-mon[34775]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 341 B/s wr, 19 op/s 2024-08-17T16:56:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:25 smithi142 ceph-mon[31182]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 256 B/s wr, 14 op/s 2024-08-17T16:56:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:25 smithi142 ceph-mon[40020]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 256 B/s wr, 14 op/s 2024-08-17T16:56:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:25 smithi152 ceph-mon[34775]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 256 B/s wr, 14 op/s 2024-08-17T16:56:26.851 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:26 smithi142 systemd[1]: Starting Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:27.260 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:56:27.576 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 podman[71110]: 2024-08-17 16:56:27.146239235 +0000 UTC m=+0.020434074 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-17T16:56:27.576 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 podman[71110]: 2024-08-17 16:56:27.256811283 +0000 UTC m=+0.131006124 volume create 653ecd4409fe7c667e3bd0b511a2d4c550841280372ecc8de6bccf870f9bc581 2024-08-17T16:56:27.576 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 podman[71110]: 2024-08-17 16:56:27.3574914 +0000 UTC m=+0.231686243 container create 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-mon[40020]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 227 B/s wr, 13 op/s 2024-08-17T16:56:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2969472293' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:56:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-mon[31182]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 227 B/s wr, 13 op/s 2024-08-17T16:56:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2969472293' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:56:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:27.873 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 podman[71110]: 2024-08-17 16:56:27.710359709 +0000 UTC m=+0.584554541 container init 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:27.873 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 podman[71110]: 2024-08-17 16:56:27.715172639 +0000 UTC m=+0.589367471 container start 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.727Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.727Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.728Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.142 port=9094 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.729Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.753Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.753Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.756Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:27.756Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 bash[71110]: 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 2024-08-17T16:56:27.874 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:27 smithi142 systemd[1]: Started Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:27 smithi152 ceph-mon[34775]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 227 B/s wr, 13 op/s 2024-08-17T16:56:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:27 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2969472293' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:56:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:28.186 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:27] ENGINE Bus STOPPING 2024-08-17T16:56:28.482 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:56:28.482 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:28] ENGINE Bus STOPPED 2024-08-17T16:56:28.482 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:28] ENGINE Bus STARTING 2024-08-17T16:56:28.482 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:28] ENGINE Serving on http://:::9283 2024-08-17T16:56:28.482 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:56:28] ENGINE Bus STARTED 2024-08-17T16:56:28.506 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":20,"flags":0,"active_gid":14445,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6800","nonce":3356817710},{"type":"v1","addr":"172.21.15.142:6801","nonce":3356817710}]},"active_addr":"172.21.15.142:6801/3356817710","active_change":"2024-08-17T16:56:12.454867+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":61,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":2472242802}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":2627951813}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":1554300251}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":639679378}]}]} 2024-08-17T16:56:28.510 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T16:56:28.510 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T16:56:28.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:56:28.761 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:29.295 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.295 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[40020]: Deploying daemon grafana.a on smithi152 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.296 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:56:29.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:56:29.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:28 smithi142 ceph-mon[31182]: Deploying daemon grafana.a on smithi152 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:28 smithi152 ceph-mon[34775]: Deploying daemon grafana.a on smithi152 2024-08-17T16:56:30.091 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:29 smithi142 ceph-mon[40020]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 204 B/s wr, 11 op/s 2024-08-17T16:56:30.092 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:29 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:29.729Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000273087s 2024-08-17T16:56:30.092 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:29 smithi142 ceph-mon[31182]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 204 B/s wr, 11 op/s 2024-08-17T16:56:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:29 smithi152 ceph-mon[34775]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 204 B/s wr, 11 op/s 2024-08-17T16:56:30.734 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:56:30.734 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":66,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T16:56:21.736219+0000","last_up_change":"2024-08-17T16:55:41.730067+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6803","nonce":3785349577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6805","nonce":3785349577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6809","nonce":3785349577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6807","nonce":3785349577}]},"public_addr":"172.21.15.142:6803/3785349577","cluster_addr":"172.21.15.142:6805/3785349577","heartbeat_back_addr":"172.21.15.142:6809/3785349577","heartbeat_front_addr":"172.21.15.142:6807/3785349577","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2041524993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2041524993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2041524993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2041524993}]},"public_addr":"172.21.15.142:6811/2041524993","cluster_addr":"172.21.15.142:6813/2041524993","heartbeat_back_addr":"172.21.15.142:6817/2041524993","heartbeat_front_addr":"172.21.15.142:6815/2041524993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6819","nonce":3610394488}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6821","nonce":3610394488}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6825","nonce":3610394488}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6823","nonce":3610394488}]},"public_addr":"172.21.15.142:6819/3610394488","cluster_addr":"172.21.15.142:6821/3610394488","heartbeat_back_addr":"172.21.15.142:6825/3610394488","heartbeat_front_addr":"172.21.15.142:6823/3610394488","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6827","nonce":2365382971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6829","nonce":2365382971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6833","nonce":2365382971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6831","nonce":2365382971}]},"public_addr":"172.21.15.142:6827/2365382971","cluster_addr":"172.21.15.142:6829/2365382971","heartbeat_back_addr":"172.21.15.142:6833/2365382971","heartbeat_front_addr":"172.21.15.142:6831/2365382971","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:56:31.081 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:30 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/680566805' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:56:31.082 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:30 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/680566805' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:56:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:30 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/680566805' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:56:31.483 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T16:56:31.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:56:31.736 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:31 smithi142 ceph-mon[40020]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:31 smithi142 ceph-mon[31182]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:31 smithi152 ceph-mon[34775]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:32.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:32 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:56:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:56:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:33 smithi142 ceph-mon[40020]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:33.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:33 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:33 smithi142 ceph-mon[31182]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:33 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:33.718 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:56:33.719 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":66,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T16:56:21.736219+0000","last_up_change":"2024-08-17T16:55:41.730067+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6803","nonce":3785349577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6805","nonce":3785349577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6809","nonce":3785349577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6807","nonce":3785349577}]},"public_addr":"172.21.15.142:6803/3785349577","cluster_addr":"172.21.15.142:6805/3785349577","heartbeat_back_addr":"172.21.15.142:6809/3785349577","heartbeat_front_addr":"172.21.15.142:6807/3785349577","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2041524993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2041524993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2041524993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2041524993}]},"public_addr":"172.21.15.142:6811/2041524993","cluster_addr":"172.21.15.142:6813/2041524993","heartbeat_back_addr":"172.21.15.142:6817/2041524993","heartbeat_front_addr":"172.21.15.142:6815/2041524993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6819","nonce":3610394488}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6821","nonce":3610394488}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6825","nonce":3610394488}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6823","nonce":3610394488}]},"public_addr":"172.21.15.142:6819/3610394488","cluster_addr":"172.21.15.142:6821/3610394488","heartbeat_back_addr":"172.21.15.142:6825/3610394488","heartbeat_front_addr":"172.21.15.142:6823/3610394488","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6827","nonce":2365382971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6829","nonce":2365382971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6833","nonce":2365382971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6831","nonce":2365382971}]},"public_addr":"172.21.15.142:6827/2365382971","cluster_addr":"172.21.15.142:6829/2365382971","heartbeat_back_addr":"172.21.15.142:6833/2365382971","heartbeat_front_addr":"172.21.15.142:6831/2365382971","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:56:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:33 smithi152 ceph-mon[34775]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:33 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:34.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T16:56:34.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T16:56:34.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T16:56:34.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T16:56:34.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T16:56:34.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T16:56:34.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T16:56:34.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T16:56:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:34 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1035386123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:56:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:34 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1035386123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:56:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:34 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1035386123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:56:34.880 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.883 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.890 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.891 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.900 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.900 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.919 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:34.923 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:35.540 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:35 smithi142 ceph-mon[40020]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:35.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:35 smithi142 ceph-mon[31182]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:35 smithi152 ceph-mon[34775]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:37 smithi152 ceph-mon[34775]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:37.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:37 smithi142 ceph-mon[40020]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:37.867 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:56:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:56:37.731Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00177422s 2024-08-17T16:56:37.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:37 smithi142 ceph-mon[31182]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:39.741 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:39 smithi142 ceph-mon[31182]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:39.741 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:39 smithi142 ceph-mon[40020]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:39 smithi152 ceph-mon[34775]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:41.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:41 smithi152 ceph-mon[34775]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:41.744 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:41 smithi152 podman[66618]: 2024-08-17 16:56:41.558674765 +0000 UTC m=+0.017105368 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-08-17T16:56:41.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:41 smithi142 ceph-mon[40020]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:41.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:41 smithi142 ceph-mon[31182]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:42.093 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:41 smithi152 podman[66618]: 2024-08-17 16:56:41.841752321 +0000 UTC m=+0.300182917 container init e049e36329852db01f98c07d7d9b9b80c6288b917b7bd95e939be1bf10a03422 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, vcs-type=git, vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, version=8.5, io.openshift.expose-services=, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:41 smithi152 podman[66618]: 2024-08-17 16:56:41.846622331 +0000 UTC m=+0.305052927 container start e049e36329852db01f98c07d7d9b9b80c6288b917b7bd95e939be1bf10a03422 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, release=239.1651231664, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.expose-services=, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:41 smithi152 bash[66618]: e049e36329852db01f98c07d7d9b9b80c6288b917b7bd95e939be1bf10a03422 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:41 smithi152 systemd[1]: Started Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005460591Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005641366Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005660908Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005667958Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005673957Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T16:56:42.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005679815Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005685344Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005691123Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005696819Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005702588Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005708084Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005713377Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005718819Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=settings t=2024-08-17T16:56:42.005724888Z level=info msg="App mode production" 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=sqlstore t=2024-08-17T16:56:42.005774796Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=sqlstore t=2024-08-17T16:56:42.005790253Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.006666787Z level=info msg="Starting DB migrations" 2024-08-17T16:56:42.095 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.006738583Z level=info msg="Executing migration" id="create migration_log table" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.00801622Z level=info msg="Executing migration" id="create user table" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.008455631Z level=info msg="Executing migration" id="add unique index user.login" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.00894859Z level=info msg="Executing migration" id="add unique index user.email" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.011971615Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.012702486Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.015570206Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.018638908Z level=info msg="Executing migration" id="create user table v2" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.021318431Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.024512745Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.027468295Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-08-17T16:56:42.096 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.030278116Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.030672649Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.033686775Z level=info msg="Executing migration" id="Update user table charset" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.036499673Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.039381253Z level=info msg="Executing migration" id="Add missing user data" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.042039087Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.045025809Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.047837568Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.050427995Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.051663629Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.05441832Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.05722223Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-08-17T16:56:42.097 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.059945721Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.062747096Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.065405736Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.065801698Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.06854306Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.071496122Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.074082772Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-08-17T16:56:42.098 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.076648708Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.079521725Z level=info msg="Executing migration" id="create temp_user v2" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.082212545Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.085030427Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.08769613Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.090114085Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.091114308Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.091762368Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.092612132Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-08-17T16:56:42.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.093503415Z level=info msg="Executing migration" id="create star table" 2024-08-17T16:56:42.344 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.094305288Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-08-17T16:56:42.344 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.094741982Z level=info msg="Executing migration" id="create org table v1" 2024-08-17T16:56:42.344 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.095712248Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.096426331Z level=info msg="Executing migration" id="create org_user table v1" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.0968053Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.097653901Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.098501543Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.099333075Z level=info msg="Executing migration" id="Update org table charset" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.09963726Z level=info msg="Executing migration" id="Update org_user table charset" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.100540392Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.100967026Z level=info msg="Executing migration" id="create dashboard table" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.101358376Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.102236748Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-08-17T16:56:42.345 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.10265963Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-08-17T16:56:42.346 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.103546597Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-08-17T16:56:42.346 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.104400256Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-08-17T16:56:42.346 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.105354407Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-08-17T16:56:42.346 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.106427511Z level=info msg="Executing migration" id="create dashboard v2" 2024-08-17T16:56:42.346 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.109192745Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-08-17T16:56:42.347 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.11216909Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-08-17T16:56:42.347 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.114892014Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-08-17T16:56:42.347 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.115390658Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-08-17T16:56:42.347 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.115760926Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-08-17T16:56:42.347 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.116080633Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-08-17T16:56:42.347 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.118376894Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.121175781Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.121665848Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.122568705Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.1233865Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.124298827Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.124731068Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.125541691Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.125861364Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.126375877Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.126848573Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-08-17T16:56:42.348 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.127394573Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.127874948Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.128206209Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.128758326Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.129582414Z level=info msg="Executing migration" id="Update dashboard title length" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.129905456Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.130751144Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.131620726Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.132563999Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.132975623Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.133825193Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.134851707Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.135618396Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-08-17T16:56:42.349 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.1359589Z level=info msg="Executing migration" id="Add check_sum column" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.136526557Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.136930713Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.137262859Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.137799709Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.138281428Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.138815303Z level=info msg="Executing migration" id="create data_source table" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.139284016Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.139736902Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.140161261Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.141034747Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-08-17T16:56:42.350 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.141975822Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.143016759Z level=info msg="Executing migration" id="create data_source table v2" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.143906721Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.144834229Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.145611061Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.145969982Z level=info msg="Executing migration" id="Add column with_credentials" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.146516797Z level=info msg="Executing migration" id="Add secure json data column" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.149049143Z level=info msg="Executing migration" id="Update data_source table charset" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.151726026Z level=info msg="Executing migration" id="Update initial version to 1" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.154673676Z level=info msg="Executing migration" id="Add read_only data column" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.155361069Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.158198554Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-08-17T16:56:42.351 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.158739028Z level=info msg="Executing migration" id="Add uid column" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.161062461Z level=info msg="Executing migration" id="Update uid value" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.163813422Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.164321261Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.167418073Z level=info msg="Executing migration" id="create api_key table" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.169933745Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.172741775Z level=info msg="Executing migration" id="add index api_key.key" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.17567438Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.178671578Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.181636483Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.184388448Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.187212144Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-08-17T16:56:42.352 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.190180262Z level=info msg="Executing migration" id="create api_key table v2" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.192713611Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.195627159Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.198096364Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.200880903Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.203411939Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.203782873Z level=info msg="Executing migration" id="Update api_key table charset" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.206631625Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.209505569Z level=info msg="Executing migration" id="Add service account foreign key" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.212114912Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.214634873Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.217417388Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.220302789Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-08-17T16:56:42.353 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.22287459Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.225495284Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.228277445Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.230987575Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.23204387Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.232672408Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.233545404Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.234479981Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.235292883Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.236125468Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.236957065Z level=info msg="Executing migration" id="create quota table v1" 2024-08-17T16:56:42.354 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.23774211Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.238737419Z level=info msg="Executing migration" id="Update quota table charset" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.23921732Z level=info msg="Executing migration" id="create plugin_setting table" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.239625206Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.240525412Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.241382611Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.241684079Z level=info msg="Executing migration" id="create session table" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.242225343Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.242995207Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.243313958Z level=info msg="Executing migration" id="create playlist table v2" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.243717924Z level=info msg="Executing migration" id="create playlist item table v2" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.244652162Z level=info msg="Executing migration" id="Update playlist table charset" 2024-08-17T16:56:42.355 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.245366581Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.245677118Z level=info msg="Executing migration" id="drop preferences table v2" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.245961158Z level=info msg="Executing migration" id="drop preferences table v3" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.246275353Z level=info msg="Executing migration" id="create preferences table v3" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.246679416Z level=info msg="Executing migration" id="Update preferences table charset" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.246969362Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.247604796Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.247902147Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.24853194Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.24943322Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.250267312Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.250729127Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-08-17T16:56:42.356 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.251636437Z level=info msg="Executing migration" id="create alert table v1" 2024-08-17T16:56:42.357 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.252602918Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-08-17T16:56:42.357 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.253362711Z level=info msg="Executing migration" id="add index alert state" 2024-08-17T16:56:42.357 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.254293424Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-08-17T16:56:42.357 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.254919162Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.25530397Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.255752955Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.256218792Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.257651363Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.258050799Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.258490378Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.258789422Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.259219208Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.259614275Z level=info msg="Executing migration" id="Add column is_default" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.260276701Z level=info msg="Executing migration" id="Add column frequency" 2024-08-17T16:56:42.358 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.260905637Z level=info msg="Executing migration" id="Add column send_reminder" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.261565853Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.262250593Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.262717716Z level=info msg="Executing migration" id="Update alert table charset" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.263013335Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.263306238Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.263707227Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.264170518Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.264558268Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.26499508Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.265433856Z level=info msg="Executing migration" id="Add for to alert table" 2024-08-17T16:56:42.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.266104841Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.266959701Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.268016018Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.268711513Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.269705966Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.270425059Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.271394864Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.27214639Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.273001051Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.273994427Z level=info msg="Executing migration" id="create annotation table v5" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.274691942Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.2756776Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.276461629Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-08-17T16:56:42.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.277324896Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.280183407Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.283002816Z level=info msg="Executing migration" id="Update annotation table charset" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.285887892Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.288956836Z level=info msg="Executing migration" id="Drop category_id index" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.291727961Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.294618778Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.297302616Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.300042496Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.303224054Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.305993838Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.308975795Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-08-17T16:56:42.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.311682506Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.314666109Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.320120509Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.322987Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.325878375Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.328707558Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.331579623Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.332134559Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.334939128Z level=info msg="Executing migration" id="Add epoch_end column" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.337683224Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.340476696Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.340898929Z level=info msg="Executing migration" id="Move region to single row" 2024-08-17T16:56:42.362 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.343619181Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-08-17T16:56:42.363 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.346315943Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-08-17T16:56:42.363 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.348988951Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-08-17T16:56:42.597 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:56:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:56:42.597 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.349602694Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.352305812Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.354859659Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.357484851Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.357814563Z level=info msg="Executing migration" id="create test_data table" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.360543122Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.363230341Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.365825648Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.368376286Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.369259799Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-08-17T16:56:42.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.370112112Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.370942683Z level=info msg="Executing migration" id="create team table" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.371342801Z level=info msg="Executing migration" id="add index team.org_id" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.372175565Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.373027225Z level=info msg="Executing migration" id="create team member table" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.373879879Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.374315958Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.375167374Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.375991675Z level=info msg="Executing migration" id="Add column email to team table" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.376840647Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-08-17T16:56:42.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.377544458Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.378404833Z level=info msg="Executing migration" id="create dashboard acl table" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.37923397Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.380081855Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.380504017Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.381482556Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.382246672Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.383060922Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.383947116Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.384797672Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.385806808Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.38651107Z level=info msg="Executing migration" id="create tag table" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.387418849Z level=info msg="Executing migration" id="add index tag.key_value" 2024-08-17T16:56:42.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.388232007Z level=info msg="Executing migration" id="create login attempt table" 2024-08-17T16:56:42.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.389100782Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-08-17T16:56:42.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.390010662Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-08-17T16:56:42.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.390981086Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-08-17T16:56:42.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.392902374Z level=info msg="Executing migration" id="create login_attempt v2" 2024-08-17T16:56:42.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.393740447Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-08-17T16:56:42.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.394764644Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-08-17T16:56:42.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.395502378Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-08-17T16:56:42.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.396361555Z level=info msg="Executing migration" id="create user auth table" 2024-08-17T16:56:42.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.396827161Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-08-17T16:56:42.602 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.39727923Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.400075769Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.400829689Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.402999437Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.405887169Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.4086643Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.411561557Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.414307137Z level=info msg="Executing migration" id="create server_lock table" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.417152956Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.419999435Z level=info msg="Executing migration" id="create user auth token table" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.423088653Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.425945054Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.428720992Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-08-17T16:56:42.603 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.431694949Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.434462854Z level=info msg="Executing migration" id="create cache_data table" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.437384041Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.440228743Z level=info msg="Executing migration" id="create short_url table v1" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.443044894Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.4456878Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.448463331Z level=info msg="Executing migration" id="delete alert_definition table" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.451362031Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.454105691Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.456760435Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.459418086Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.461897114Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.464522583Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-08-17T16:56:42.604 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.467495462Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.470197321Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.472960513Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.47553402Z level=info msg="Executing migration" id="drop alert_definition table" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.478333983Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.481168883Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.48379563Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.486535249Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.489196218Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.491955476Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.49272272Z level=info msg="Executing migration" id="create alert_instance table" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.493664819Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.494410495Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-08-17T16:56:42.605 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.495253306Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.496089417Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.497073072Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.497796884Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.50135676Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.504771625Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.50575916Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.506481337Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.507368367Z level=info msg="Executing migration" id="create alert_rule table" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.507780199Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.508249413Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.508676068Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.509097356Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-08-17T16:56:42.606 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.509390762Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.510226963Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.511162192Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.511972236Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.512402845Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.512792765Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.516270067Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.517130193Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.517588509Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.51844027Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.519342955Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.519659245Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-08-17T16:56:42.607 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.520130088Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.520567806Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.521039549Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.521337658Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.52436454Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.527185129Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.529994799Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.533094016Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.535803944Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.538539355Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.541316132Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.544207969Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.544537144Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-08-17T16:56:42.608 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.547678318Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.550557041Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.553130474Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.553543233Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.556394572Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.559297109Z level=info msg="Executing migration" id="create provenance_type table" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.561959126Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.56243328Z level=info msg="Executing migration" id="create alert_image table" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.565265706Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.568063024Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.570701728Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.571150108Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.571538519Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T16:56:42.609 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.571610936Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.571866997Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.572231712Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.574155045Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.574301706Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.574411087Z level=info msg="alerts found to migrate" alerts=0 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.574816804Z level=info msg="Executing migration" id="create library_element table v1" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.576907836Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.579680115Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.582463197Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.583164257Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.583969199Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-08-17T16:56:42.610 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.584282238Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.58515441Z level=info msg="Executing migration" id="create data_keys table" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.585622293Z level=info msg="Executing migration" id="create secrets table" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.586070052Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.59039301Z level=info msg="Executing migration" id="add name column into data_keys" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.591346169Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.592185692Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-08-17T16:56:42.611 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.596534975Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-08-17T16:56:42.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.601082714Z level=info msg="Executing migration" id="create kv_store table v1" 2024-08-17T16:56:42.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.601523771Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-08-17T16:56:42.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.602360828Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-08-17T16:56:42.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.603200657Z level=info msg="Executing migration" id="create permission table" 2024-08-17T16:56:42.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.604045235Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-08-17T16:56:42.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.604517759Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.605418375Z level=info msg="Executing migration" id="create role table" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.606253489Z level=info msg="Executing migration" id="add column display_name" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.607205342Z level=info msg="Executing migration" id="add column group_name" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.608164687Z level=info msg="Executing migration" id="add index role.org_id" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.60907688Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.609950199Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.610813892Z level=info msg="Executing migration" id="create team role table" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.611268054Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.612208129Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-08-17T16:56:42.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.613203645Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-08-17T16:56:42.997 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.613976617Z level=info msg="Executing migration" id="create user role table" 2024-08-17T16:56:42.997 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.614413875Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-08-17T16:56:42.997 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.615349919Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.61622775Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.617195024Z level=info msg="Executing migration" id="create builtin role table" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.617664589Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.618537814Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.619450641Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.620455854Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-08-17T16:56:42.998 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.621018147Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.623933586Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.624369342Z level=info msg="Executing migration" id="add unique index role.uid" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.624783737Z level=info msg="Executing migration" id="create seed assignment table" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.62519436Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.625639596Z level=info msg="Executing migration" id="add column hidden to role table" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.626768466Z level=info msg="Executing migration" id="create query_history table v1" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.627188304Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.627623717Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.627892953Z level=info msg="Executing migration" id="teams permissions migration" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.628308072Z level=info msg="Executing migration" id="dashboard permissions" 2024-08-17T16:56:42.999 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.628727605Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.629105253Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.629413693Z level=info msg="Executing migration" id="alerting notification permissions" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.629729373Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.630130335Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.630571665Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.631536891Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.631837705Z level=info msg="Executing migration" id="create correlation table v1" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.632269485Z level=info msg="Executing migration" id="add index correlations.uid" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.632694529Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.633160903Z level=info msg="Executing migration" id="add correlation config column" 2024-08-17T16:56:43.000 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.634133689Z level=info msg="Executing migration" id="create entity_events table" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.634561958Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.634972Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.635052093Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.635340183Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.635422176Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.63566653Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.636067904Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.63647944Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.636910436Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.637338549Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-08-17T16:56:43.001 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.637713746Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.638132312Z level=info msg="Executing migration" id="Drop public config table" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.638527334Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.638917303Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.639347757Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.639768693Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.640208631Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.643322925Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.64432411Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.645349469Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.645667757Z level=info msg="Executing migration" id="add share column" 2024-08-17T16:56:43.002 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.646708756Z level=info msg="Executing migration" id="create default alerting folders" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.64718435Z level=info msg="Executing migration" id="create file table" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.647653441Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.650540297Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.651162632Z level=info msg="Executing migration" id="create file_meta table" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.654027263Z level=info msg="Executing migration" id="file table idx: path key" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.654533076Z level=info msg="Executing migration" id="set path collation in file table" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.657427456Z level=info msg="Executing migration" id="managed permissions migration" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.658004018Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.660323702Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.663259496Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-08-17T16:56:43.003 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.664436197Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.665299407Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.666119439Z level=info msg="Executing migration" id="update group index for alert rules" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.666979319Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.667308298Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.668136404Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.669205175Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.670220892Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.67112724Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.681102476Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.682006248Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-08-17T16:56:43.004 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.682834437Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.686272044Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.686594101Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.68754303Z level=info msg="Executing migration" id="create folder table" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.688157792Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.688611422Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.689592032Z level=info msg="Executing migration" id="Update folder title length" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.690132126Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=migrator t=2024-08-17T16:56:42.690936838Z level=info msg="migrations completed" performed=485 skipped=0 duration=684.216439ms 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=sqlstore t=2024-08-17T16:56:42.69134619Z level=info msg="Created default organization" 2024-08-17T16:56:43.005 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=plugin.loader t=2024-08-17T16:56:42.74680567Z level=info msg="Plugin registered" pluginID=input 2024-08-17T16:56:43.006 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=plugin.loader t=2024-08-17T16:56:42.773445509Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-17T16:56:43.006 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=plugin.loader t=2024-08-17T16:56:42.773464709Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-17T16:56:43.006 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=secrets t=2024-08-17T16:56:42.77353975Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T16:56:43.006 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=query_data t=2024-08-17T16:56:42.77516295Z level=info msg="Query Service initialization" 2024-08-17T16:56:43.006 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=live.push_http t=2024-08-17T16:56:42.777404424Z level=info msg="Live Push Gateway initialization" 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: Deploying daemon node-exporter.a on smithi142 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: Deploying daemon node-exporter.a on smithi142 2024-08-17T16:56:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: Deploying daemon node-exporter.a on smithi142 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:43.823 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=infra.usagestats.collector t=2024-08-17T16:56:43.510702891Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T16:56:43.823 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=provisioning.datasources t=2024-08-17T16:56:43.510991445Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-08-17T16:56:43.823 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=provisioning.datasources t=2024-08-17T16:56:43.543834737Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T16:56:43.823 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=provisioning.datasources t=2024-08-17T16:56:43.575779379Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=provisioning.alerting t=2024-08-17T16:56:43.597406268Z level=info msg="starting to provision alerting" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=provisioning.alerting t=2024-08-17T16:56:43.597425489Z level=info msg="finished to provision alerting" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=ngalert.state.manager t=2024-08-17T16:56:43.597495912Z level=info msg="Warming state cache for startup" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=grafanaStorageLogger t=2024-08-17T16:56:43.597506302Z level=info msg="storage starting" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=http.server t=2024-08-17T16:56:43.59931599Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=ngalert.state.manager t=2024-08-17T16:56:43.607651095Z level=info msg="State cache has been initialized" states=0 duration=10.154513ms 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=ticker t=2024-08-17T16:56:43.607712618Z level=info msg=starting first_tick=2024-08-17T16:56:50Z 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=ngalert.multiorg.alertmanager t=2024-08-17T16:56:43.607728845Z level=info msg="starting MultiOrg Alertmanager" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=sqlstore.transactions t=2024-08-17T16:56:43.608025629Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=sqlstore.transactions t=2024-08-17T16:56:43.659907333Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:56:43.824 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 16:56:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[66633]: logger=sqlstore.transactions t=2024-08-17T16:56:43.671251327Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-08-17T16:56:44.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:44 smithi142 ceph-mon[31182]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:44.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:44 smithi142 ceph-mon[40020]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:44 smithi152 ceph-mon[34775]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:45.771 INFO:teuthology.orchestra.run.smithi142.stdout:34359738416 2024-08-17T16:56:45.771 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T16:56:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:46 smithi152 ceph-mon[34775]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:46.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:46 smithi142 ceph-mon[31182]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:46.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:46 smithi142 ceph-mon[40020]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:46.863 INFO:teuthology.orchestra.run.smithi142.stdout:111669149728 2024-08-17T16:56:46.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T16:56:48.254 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:48 smithi142 ceph-mon[40020]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:48.254 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:48 smithi142 ceph-mon[31182]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:48.279 INFO:teuthology.orchestra.run.smithi142.stdout:55834574892 2024-08-17T16:56:48.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T16:56:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:48 smithi152 ceph-mon[34775]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:49.600 INFO:teuthology.orchestra.run.smithi142.stdout:201863462930 2024-08-17T16:56:49.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T16:56:50.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:50 smithi152 ceph-mon[34775]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:50.360 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:50 smithi142 ceph-mon[40020]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:50.361 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:50 smithi142 ceph-mon[31182]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:50.385 INFO:teuthology.orchestra.run.smithi142.stdout:133143986205 2024-08-17T16:56:50.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T16:56:51.633 INFO:teuthology.orchestra.run.smithi142.stdout:77309411367 2024-08-17T16:56:51.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T16:56:52.035 INFO:teuthology.orchestra.run.smithi142.stdout:167503724570 2024-08-17T16:56:52.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T16:56:52.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:52 smithi152 ceph-mon[34775]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:52.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:52 smithi142 ceph-mon[40020]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:52.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:52 smithi142 ceph-mon[31182]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:52.520 INFO:teuthology.orchestra.run.smithi142.stdout:236223201296 2024-08-17T16:56:52.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T16:56:52.555 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:52.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:56:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:56:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:56:52.855 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:52.856 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:52.856 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:52.856 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:54.212 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:54 smithi142 ceph-mon[40020]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:54.212 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:54 smithi142 ceph-mon[31182]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:54.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:54 smithi152 ceph-mon[34775]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:55.873 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:55 smithi142 bash[73836]: Writing manifest to image destination 2024-08-17T16:56:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:56 smithi152 ceph-mon[34775]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:56.353 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:56 smithi142 ceph-mon[40020]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:56.354 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:55 smithi142 podman[73836]: 2024-08-17 16:56:55.919104245 +0000 UTC m=+3.302297745 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T16:56:56.354 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:56 smithi142 ceph-mon[31182]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:56.622 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:56 smithi142 podman[73836]: 2024-08-17 16:56:56.45393048 +0000 UTC m=+3.837124020 container create dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:58.016 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 podman[73836]: 2024-08-17 16:56:57.638356588 +0000 UTC m=+5.021550094 container init dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:58.016 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 podman[73836]: 2024-08-17 16:56:57.644172766 +0000 UTC m=+5.027366264 container start dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.648Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.648Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.648Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.648Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T16:56:58.017 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T16:56:58.018 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T16:56:58.018 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T16:56:58.018 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T16:56:58.019 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T16:56:58.019 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T16:56:58.019 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T16:56:58.020 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.649Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T16:56:58.021 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T16:56:58.022 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T16:56:58.023 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T16:56:58.023 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[74149]: ts=2024-08-17T16:56:57.650Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T16:56:58.023 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 bash[73836]: dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f 2024-08-17T16:56:58.023 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 16:56:57 smithi142 systemd[1]: Started Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:58.318 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:58 smithi152 ceph-mon[34775]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:58.318 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:58.318 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.318 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.318 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.351 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[31182]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:58.351 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:58.351 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.351 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.351 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.352 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[40020]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:56:58.352 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:58.352 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.352 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.352 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:56:58.405 INFO:teuthology.orchestra.run.smithi142.stdout:111669149730 2024-08-17T16:56:58.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:56:58 smithi152 systemd[1]: Starting Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:58.889 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:58.889 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:56:59.031 INFO:teuthology.orchestra.run.smithi142.stdout:34359738418 2024-08-17T16:56:59.040 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:56:58 smithi152 bash[66918]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-17T16:56:59.286 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:59 smithi142 ceph-mon[31182]: Deploying daemon node-exporter.b on smithi152 2024-08-17T16:56:59.286 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:59 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2447457542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:56:59.286 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:56:59 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1068804025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:56:59.286 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:59 smithi142 ceph-mon[40020]: Deploying daemon node-exporter.b on smithi152 2024-08-17T16:56:59.286 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:59 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2447457542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:56:59.286 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:56:59 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1068804025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:56:59.311 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149730 for osd.3 2024-08-17T16:56:59.311 DEBUG:teuthology.parallel:result is None 2024-08-17T16:56:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:59 smithi152 ceph-mon[34775]: Deploying daemon node-exporter.b on smithi152 2024-08-17T16:56:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:59 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2447457542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:56:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:56:59 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1068804025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:56:59.323 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:56:59 smithi152 bash[66918]: Getting image source signatures 2024-08-17T16:56:59.323 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:56:59 smithi152 bash[66918]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-17T16:56:59.323 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:56:59 smithi152 bash[66918]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-17T16:56:59.323 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:56:59 smithi152 bash[66918]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-17T16:57:00.095 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:00 smithi142 ceph-mon[40020]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:00.096 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:00 smithi142 ceph-mon[31182]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:00.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:00 smithi152 ceph-mon[34775]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:01.018 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:00 smithi152 bash[66918]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-17T16:57:01.018 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:00 smithi152 bash[66918]: Writing manifest to image destination 2024-08-17T16:57:01.267 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:01.310 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 podman[66918]: 2024-08-17 16:57:01.009318219 +0000 UTC m=+2.420040622 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T16:57:01.310 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 podman[66918]: 2024-08-17 16:57:01.083151231 +0000 UTC m=+2.493873689 container create 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:01.310 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 podman[66918]: 2024-08-17 16:57:01.310039624 +0000 UTC m=+2.720762045 container init 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 podman[66918]: 2024-08-17 16:57:01.314906625 +0000 UTC m=+2.725629029 container start 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.318Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.318Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.319Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.319Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.319Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.319Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.319Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T16:57:01.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T16:57:01.575 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T16:57:01.576 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T16:57:01.577 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T16:57:01.578 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T16:57:01.579 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[66970]: ts=2024-08-17T16:57:01.320Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T16:57:01.579 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 bash[66918]: 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 2024-08-17T16:57:01.579 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 16:57:01 smithi152 systemd[1]: Started Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:01.817 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2024-08-17T16:57:01.817 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:02.339 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.340 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.341 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.341 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.341 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:02.341 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:02 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:02.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:57:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:57:02.976 INFO:teuthology.orchestra.run.smithi142.stdout:77309411369 2024-08-17T16:57:03.502 INFO:teuthology.orchestra.run.smithi142.stdout:133143986208 2024-08-17T16:57:03.788 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[40020]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:03.788 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:03.789 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/939384234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:57:03.789 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1820562392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:57:03.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[31182]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:03.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:03.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/939384234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:57:03.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:03 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1820562392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:57:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:03 smithi152 ceph-mon[34775]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:03 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:03 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/939384234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:57:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:03 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1820562392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:57:05.568 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986205 got 133143986208 for osd.4 2024-08-17T16:57:05.568 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:05.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:05 smithi142 ceph-mon[40020]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:05.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:05 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:05.628 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:05 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:05 smithi142 ceph-mon[31182]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:05 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:05 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:05.628 INFO:teuthology.orchestra.run.smithi142.stdout:201863462935 2024-08-17T16:57:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:05 smithi152 ceph-mon[34775]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:05 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:05 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:06.648 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:06 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2379018608' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:57:06.649 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:06 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2379018608' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:57:06.676 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462935 for osd.6 2024-08-17T16:57:06.677 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:06 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2379018608' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:57:07.601 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:07 smithi142 ceph-mon[40020]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:07.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:07 smithi142 ceph-mon[31182]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:07 smithi152 ceph-mon[34775]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:08.010 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411369 for osd.2 2024-08-17T16:57:08.010 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:09.014 INFO:teuthology.orchestra.run.smithi142.stdout:236223201300 2024-08-17T16:57:09.504 INFO:teuthology.orchestra.run.smithi142.stdout:167503724574 2024-08-17T16:57:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:09 smithi152 ceph-mon[34775]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:09 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2112103391' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:57:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:09 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1233722245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:57:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:09 smithi142 ceph-mon[40020]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:09 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2112103391' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:57:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:09 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1233722245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:57:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:09 smithi142 ceph-mon[31182]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:09 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2112103391' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:57:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:09 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1233722245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:57:10.197 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724570 got 167503724574 for osd.5 2024-08-17T16:57:10.198 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:10.853 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201296 got 236223201300 for osd.7 2024-08-17T16:57:10.853 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:11.304 INFO:teuthology.orchestra.run.smithi142.stdout:55834574897 2024-08-17T16:57:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:11 smithi152 ceph-mon[34775]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:11 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2501938977' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:57:11.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:11 smithi142 ceph-mon[31182]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:11.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:11 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2501938977' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:57:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:11 smithi142 ceph-mon[40020]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:11 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2501938977' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:57:11.999 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574897 for osd.1 2024-08-17T16:57:11.999 DEBUG:teuthology.parallel:result is None 2024-08-17T16:57:11.999 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T16:57:11.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:57:12.291 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:12 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:12.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:57:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:57:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:12 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:12.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:12 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:13.651 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:13 smithi142 ceph-mon[40020]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:13.651 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:13 smithi142 ceph-mon[31182]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:13 smithi152 ceph-mon[34775]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:14.763 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:57:14.763 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T16:57:15.539 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-08-17T16:57:14.479741+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385772,"kb_used_data":6924,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575444,"statfs":{"total":767960285184,"available":765517254656,"internally_reserved":0,"allocated":7090176,"data_stored":4003308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002056"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927356+0000","last_change":"2024-08-17T16:56:17.492610+0000","last_active":"2024-08-17T16:56:21.927356+0000","last_peered":"2024-08-17T16:56:21.927356+0000","last_clean":"2024-08-17T16:56:21.927356+0000","last_became_active":"2024-08-17T16:56:12.470076+0000","last_became_peered":"2024-08-17T16:56:12.470076+0000","last_unstale":"2024-08-17T16:56:21.927356+0000","last_undegraded":"2024-08-17T16:56:21.927356+0000","last_fullsized":"2024-08-17T16:56:21.927356+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:24.652500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.740330+0000","last_change":"2024-08-17T16:56:17.485823+0000","last_active":"2024-08-17T16:56:21.740330+0000","last_peered":"2024-08-17T16:56:21.740330+0000","last_clean":"2024-08-17T16:56:21.740330+0000","last_became_active":"2024-08-17T16:56:12.468663+0000","last_became_peered":"2024-08-17T16:56:12.468663+0000","last_unstale":"2024-08-17T16:56:21.740330+0000","last_undegraded":"2024-08-17T16:56:21.740330+0000","last_fullsized":"2024-08-17T16:56:21.740330+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:25.628917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037574,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927379+0000","last_change":"2024-08-17T16:56:17.492672+0000","last_active":"2024-08-17T16:56:21.927379+0000","last_peered":"2024-08-17T16:56:21.927379+0000","last_clean":"2024-08-17T16:56:21.927379+0000","last_became_active":"2024-08-17T16:56:12.468762+0000","last_became_peered":"2024-08-17T16:56:12.468762+0000","last_unstale":"2024-08-17T16:56:21.927379+0000","last_undegraded":"2024-08-17T16:56:21.927379+0000","last_fullsized":"2024-08-17T16:56:21.927379+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:15.340939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.740339+0000","last_change":"2024-08-17T16:56:17.485649+0000","last_active":"2024-08-17T16:56:21.740339+0000","last_peered":"2024-08-17T16:56:21.740339+0000","last_clean":"2024-08-17T16:56:21.740339+0000","last_became_active":"2024-08-17T16:56:12.470313+0000","last_became_peered":"2024-08-17T16:56:12.470313+0000","last_unstale":"2024-08-17T16:56:21.740339+0000","last_undegraded":"2024-08-17T16:56:21.740339+0000","last_fullsized":"2024-08-17T16:56:21.740339+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:41.233588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927554+0000","last_change":"2024-08-17T16:56:17.493126+0000","last_active":"2024-08-17T16:56:21.927554+0000","last_peered":"2024-08-17T16:56:21.927554+0000","last_clean":"2024-08-17T16:56:21.927554+0000","last_became_active":"2024-08-17T16:56:12.468605+0000","last_became_peered":"2024-08-17T16:56:12.468605+0000","last_unstale":"2024-08-17T16:56:21.927554+0000","last_undegraded":"2024-08-17T16:56:21.927554+0000","last_fullsized":"2024-08-17T16:56:21.927554+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:55.016828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.926687+0000","last_change":"2024-08-17T16:56:17.485644+0000","last_active":"2024-08-17T16:56:21.926687+0000","last_peered":"2024-08-17T16:56:21.926687+0000","last_clean":"2024-08-17T16:56:21.926687+0000","last_became_active":"2024-08-17T16:56:12.897452+0000","last_became_peered":"2024-08-17T16:56:12.897452+0000","last_unstale":"2024-08-17T16:56:21.926687+0000","last_undegraded":"2024-08-17T16:56:21.926687+0000","last_fullsized":"2024-08-17T16:56:21.926687+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:18.635731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927573+0000","last_change":"2024-08-17T16:56:17.492695+0000","last_active":"2024-08-17T16:56:21.927573+0000","last_peered":"2024-08-17T16:56:21.927573+0000","last_clean":"2024-08-17T16:56:21.927573+0000","last_became_active":"2024-08-17T16:56:12.469470+0000","last_became_peered":"2024-08-17T16:56:12.469470+0000","last_unstale":"2024-08-17T16:56:21.927573+0000","last_undegraded":"2024-08-17T16:56:21.927573+0000","last_fullsized":"2024-08-17T16:56:21.927573+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:02.237538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927281+0000","last_change":"2024-08-17T16:56:17.492706+0000","last_active":"2024-08-17T16:56:21.927281+0000","last_peered":"2024-08-17T16:56:21.927281+0000","last_clean":"2024-08-17T16:56:21.927281+0000","last_became_active":"2024-08-17T16:56:12.470130+0000","last_became_peered":"2024-08-17T16:56:12.470130+0000","last_unstale":"2024-08-17T16:56:21.927281+0000","last_undegraded":"2024-08-17T16:56:21.927281+0000","last_fullsized":"2024-08-17T16:56:21.927281+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:00.149068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":272,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-08-17T16:56:17.492342+0000","last_change":"2024-08-17T16:55:45.748632+0000","last_active":"2024-08-17T16:56:17.492342+0000","last_peered":"2024-08-17T16:56:17.492342+0000","last_clean":"2024-08-17T16:56:17.492342+0000","last_became_active":"2024-08-17T16:55:45.748242+0000","last_became_peered":"2024-08-17T16:55:45.748242+0000","last_unstale":"2024-08-17T16:56:17.492342+0000","last_undegraded":"2024-08-17T16:56:17.492342+0000","last_fullsized":"2024-08-17T16:56:17.492342+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:22.553778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":869493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47399999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.439},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.34699999999999998}]},{"osd":5,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.39800000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1261568,"data_stored":869512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.751}]},{"osd":1,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.70999999999999996}]},{"osd":2,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60899999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.505}]},{"osd":5,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Aug 17 16:56:42 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.63400000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":278821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.54200000000000004,"15min":0.55000000000000004},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.68999999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.53800000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.68899999999999995,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.52800000000000002,"15min":0.53600000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70099999999999996,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.63400000000000001,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.53700000000000003,"15min":0.54400000000000004},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.69999999999999996,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.501},{"interface":"front","average":{"1min":0.504,"5min":0.53700000000000003,"15min":0.54300000000000004},"min":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.61099999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.5,"15min":0.502},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56299999999999994,"15min":0.56799999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.73799999999999999,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.56599999999999995}]},{"osd":4,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.39400000000000002,"15min":0.38800000000000001},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.374}]},{"osd":6,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Aug 17 16:56:45 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.52100000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":644,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":659456,"data_stored":278860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64000000000000001,"5min":1.151,"15min":1.151},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.48599999999999999,"15min":0.48499999999999999},"min":{"1min":0.34100000000000003,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.59099999999999997}]},{"osd":1,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.48199999999999998,"15min":0.47899999999999998},"min":{"1min":0.36799999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.498,"15min":0.496},"min":{"1min":0.39000000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67700000000000005,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.39300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.67800000000000005,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.5},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.50900000000000001,"15min":0.50600000000000001},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":0.627,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51000000000000001,"15min":0.503},"min":{"1min":0.34399999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71299999999999997,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.48699999999999999,"15min":0.48499999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66200000000000003,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Aug 17 16:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.32300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Aug 17 16:56:19 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Aug 17 16:56:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.62},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.60199999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":644,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":659456,"data_stored":278860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:15 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.39000000000000001,"15min":0.39100000000000001},"min":{"1min":0.23999999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.52300000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.41199999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:56:15 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.45700000000000002,"15min":0.45900000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.432,"15min":0.434},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.434}]},{"osd":2,"last update":"Sat Aug 17 16:56:15 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.45800000000000002,"15min":0.46000000000000002},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51900000000000002,"15min":0.52000000000000002},"min":{"1min":0.35599999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.69799999999999995}]},{"osd":5,"last update":"Sat Aug 17 16:57:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53900000000000003,"15min":0.54200000000000004},"min":{"1min":0.38500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.501,"5min":0.53100000000000003,"15min":0.53600000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66900000000000004,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Aug 17 16:56:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Aug 17 16:56:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.71599999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":279125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:51 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.34300000000000003},"min":{"1min":0.249,"5min":0.186,"15min":0.186},"max":{"1min":0.44800000000000001,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.436},{"interface":"front","average":{"1min":0.379,"5min":0.35599999999999998,"15min":0.34899999999999998},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:56:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39400000000000002,"15min":0.38400000000000001},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.60099999999999998,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.375},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.41099999999999998,"15min":0.40500000000000003},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:56:17 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.42399999999999999,"15min":0.42199999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58599999999999997,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.47599999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:56:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58499999999999996,"15min":0.58199999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.82899999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56599999999999995,"15min":0.56899999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.81899999999999995}]},{"osd":6,"last update":"Sat Aug 17 16:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.77100000000000002}]},{"osd":7,"last update":"Sat Aug 17 16:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.624},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.78400000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":279144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.376,"15min":0.36299999999999999},"min":{"1min":0.27700000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.52500000000000002,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.34300000000000003},"min":{"1min":0.24199999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.58099999999999996,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:56:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.26500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.58099999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39500000000000002,"15min":0.39100000000000001},"min":{"1min":0.29099999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56899999999999995,"5min":0.626,"15min":0.626},"last":0.24099999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.42699999999999999,"15min":0.42299999999999999},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69899999999999995,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.309}]},{"osd":4,"last update":"Sat Aug 17 16:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41399999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78900000000000003,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57399999999999995,"15min":0.57299999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84199999999999997,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.63200000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:57:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54300000000000004,"15min":0.54200000000000004},"min":{"1min":0.38200000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.73699999999999999,"5min":0.753,"15min":0.753},"last":0.77700000000000002}]},{"osd":6,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Aug 17 16:56:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.749},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.75800000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":869493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.33200000000000002},"min":{"1min":0.216,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.218},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36499999999999999,"15min":0.35699999999999998},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.53100000000000003,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Aug 17 16:56:49 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55800000000000005,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.316},{"interface":"front","average":{"1min":0.376,"5min":0.38900000000000001,"15min":0.39400000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.52900000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.24299999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:56:14 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.46200000000000002,"15min":0.46000000000000002},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.61399999999999999,"5min":0.629,"15min":0.629},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.42499999999999999,"15min":0.42699999999999999},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.54200000000000004,"5min":0.624,"15min":0.624},"last":0.29399999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:56:35 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54100000000000004,"15min":0.54000000000000004},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.73799999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:56:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.55200000000000005,"15min":0.55800000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.499},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:57:15.541 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:57:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:15 smithi152 ceph-mon[34775]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:15 smithi152 ceph-mon[34775]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:57:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:15 smithi142 ceph-mon[40020]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:15 smithi142 ceph-mon[40020]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:57:15.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:15 smithi142 ceph-mon[31182]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:15.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:15 smithi142 ceph-mon[31182]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:57:16.037 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:17.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:17 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:17 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:18.071 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:18 smithi142 ceph-mon[40020]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:18.071 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:18 smithi142 ceph-mon[40020]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T16:57:18.071 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:18 smithi142 ceph-mon[40020]: Reconfiguring daemon alertmanager.a on smithi142 2024-08-17T16:57:18.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:18 smithi142 ceph-mon[31182]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:18.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:18 smithi142 ceph-mon[31182]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T16:57:18.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:18 smithi142 ceph-mon[31182]: Reconfiguring daemon alertmanager.a on smithi142 2024-08-17T16:57:18.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:18 smithi152 ceph-mon[34775]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:18.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:18 smithi152 ceph-mon[34775]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T16:57:18.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:18 smithi152 ceph-mon[34775]: Reconfiguring daemon alertmanager.a on smithi142 2024-08-17T16:57:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:57:19.353 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T16:57:20.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:20 smithi152 ceph-mon[34775]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:20.329 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:20 smithi142 ceph-mon[40020]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:20.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:20 smithi142 ceph-mon[31182]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:20.701 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-08-17T16:57:18.480552+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385772,"kb_used_data":6924,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575444,"statfs":{"total":767960285184,"available":765517254656,"internally_reserved":0,"allocated":7090176,"data_stored":4003308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002142"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927356+0000","last_change":"2024-08-17T16:56:17.492610+0000","last_active":"2024-08-17T16:56:21.927356+0000","last_peered":"2024-08-17T16:56:21.927356+0000","last_clean":"2024-08-17T16:56:21.927356+0000","last_became_active":"2024-08-17T16:56:12.470076+0000","last_became_peered":"2024-08-17T16:56:12.470076+0000","last_unstale":"2024-08-17T16:56:21.927356+0000","last_undegraded":"2024-08-17T16:56:21.927356+0000","last_fullsized":"2024-08-17T16:56:21.927356+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:24.652500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.740330+0000","last_change":"2024-08-17T16:56:17.485823+0000","last_active":"2024-08-17T16:56:21.740330+0000","last_peered":"2024-08-17T16:56:21.740330+0000","last_clean":"2024-08-17T16:56:21.740330+0000","last_became_active":"2024-08-17T16:56:12.468663+0000","last_became_peered":"2024-08-17T16:56:12.468663+0000","last_unstale":"2024-08-17T16:56:21.740330+0000","last_undegraded":"2024-08-17T16:56:21.740330+0000","last_fullsized":"2024-08-17T16:56:21.740330+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:25.628917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037574,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927379+0000","last_change":"2024-08-17T16:56:17.492672+0000","last_active":"2024-08-17T16:56:21.927379+0000","last_peered":"2024-08-17T16:56:21.927379+0000","last_clean":"2024-08-17T16:56:21.927379+0000","last_became_active":"2024-08-17T16:56:12.468762+0000","last_became_peered":"2024-08-17T16:56:12.468762+0000","last_unstale":"2024-08-17T16:56:21.927379+0000","last_undegraded":"2024-08-17T16:56:21.927379+0000","last_fullsized":"2024-08-17T16:56:21.927379+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:15.340939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.740339+0000","last_change":"2024-08-17T16:56:17.485649+0000","last_active":"2024-08-17T16:56:21.740339+0000","last_peered":"2024-08-17T16:56:21.740339+0000","last_clean":"2024-08-17T16:56:21.740339+0000","last_became_active":"2024-08-17T16:56:12.470313+0000","last_became_peered":"2024-08-17T16:56:12.470313+0000","last_unstale":"2024-08-17T16:56:21.740339+0000","last_undegraded":"2024-08-17T16:56:21.740339+0000","last_fullsized":"2024-08-17T16:56:21.740339+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:41.233588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927554+0000","last_change":"2024-08-17T16:56:17.493126+0000","last_active":"2024-08-17T16:56:21.927554+0000","last_peered":"2024-08-17T16:56:21.927554+0000","last_clean":"2024-08-17T16:56:21.927554+0000","last_became_active":"2024-08-17T16:56:12.468605+0000","last_became_peered":"2024-08-17T16:56:12.468605+0000","last_unstale":"2024-08-17T16:56:21.927554+0000","last_undegraded":"2024-08-17T16:56:21.927554+0000","last_fullsized":"2024-08-17T16:56:21.927554+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:55.016828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.926687+0000","last_change":"2024-08-17T16:56:17.485644+0000","last_active":"2024-08-17T16:56:21.926687+0000","last_peered":"2024-08-17T16:56:21.926687+0000","last_clean":"2024-08-17T16:56:21.926687+0000","last_became_active":"2024-08-17T16:56:12.897452+0000","last_became_peered":"2024-08-17T16:56:12.897452+0000","last_unstale":"2024-08-17T16:56:21.926687+0000","last_undegraded":"2024-08-17T16:56:21.926687+0000","last_fullsized":"2024-08-17T16:56:21.926687+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:18.635731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927573+0000","last_change":"2024-08-17T16:56:17.492695+0000","last_active":"2024-08-17T16:56:21.927573+0000","last_peered":"2024-08-17T16:56:21.927573+0000","last_clean":"2024-08-17T16:56:21.927573+0000","last_became_active":"2024-08-17T16:56:12.469470+0000","last_became_peered":"2024-08-17T16:56:12.469470+0000","last_unstale":"2024-08-17T16:56:21.927573+0000","last_undegraded":"2024-08-17T16:56:21.927573+0000","last_fullsized":"2024-08-17T16:56:21.927573+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:02.237538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:56:21.927281+0000","last_change":"2024-08-17T16:56:17.492706+0000","last_active":"2024-08-17T16:56:21.927281+0000","last_peered":"2024-08-17T16:56:21.927281+0000","last_clean":"2024-08-17T16:56:21.927281+0000","last_became_active":"2024-08-17T16:56:12.470130+0000","last_became_peered":"2024-08-17T16:56:12.470130+0000","last_unstale":"2024-08-17T16:56:21.927281+0000","last_undegraded":"2024-08-17T16:56:21.927281+0000","last_fullsized":"2024-08-17T16:56:21.927281+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:00.149068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":272,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-08-17T16:56:17.492342+0000","last_change":"2024-08-17T16:55:45.748632+0000","last_active":"2024-08-17T16:56:17.492342+0000","last_peered":"2024-08-17T16:56:17.492342+0000","last_clean":"2024-08-17T16:56:17.492342+0000","last_became_active":"2024-08-17T16:55:45.748242+0000","last_became_peered":"2024-08-17T16:55:45.748242+0000","last_unstale":"2024-08-17T16:56:17.492342+0000","last_undegraded":"2024-08-17T16:56:17.492342+0000","last_fullsized":"2024-08-17T16:56:17.492342+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:22.553778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":869493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.61099999999999999}]},{"osd":2,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.625},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.379},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.39500000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.442}]},{"osd":6,"last update":"Sat Aug 17 16:56:43 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.41999999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1261568,"data_stored":869512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.42999999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.497},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.40400000000000003}]},{"osd":2,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.47699999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Aug 17 16:56:42 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.56499999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":278821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.54200000000000004,"15min":0.55000000000000004},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.68999999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.53200000000000003,"15min":0.53800000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.68899999999999995,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.52800000000000002,"15min":0.53600000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70099999999999996,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.63400000000000001,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.53700000000000003,"15min":0.54400000000000004},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.69999999999999996,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.53700000000000003,"15min":0.54300000000000004},"min":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.61099999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.44}]},{"osd":3,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.5,"15min":0.502},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56299999999999994,"15min":0.56799999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.73799999999999999,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.374},{"interface":"front","average":{"1min":0.433,"5min":0.39400000000000002,"15min":0.38800000000000001},"min":{"1min":0.28599999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.35899999999999999}]},{"osd":7,"last update":"Sat Aug 17 16:56:45 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46899999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":644,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":659456,"data_stored":278860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64000000000000001,"5min":1.151,"15min":1.151},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.48599999999999999,"15min":0.48499999999999999},"min":{"1min":0.34100000000000003,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.59099999999999997}]},{"osd":1,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.48199999999999998,"15min":0.47899999999999998},"min":{"1min":0.36799999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.498,"15min":0.496},"min":{"1min":0.39000000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67700000000000005,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.39300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.67800000000000005,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.5},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.50900000000000001,"15min":0.50600000000000001},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":0.627,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Aug 17 16:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51000000000000001,"15min":0.503},"min":{"1min":0.34399999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71299999999999997,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.48699999999999999,"15min":0.48499999999999999},"min":{"1min":0.374,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66200000000000003,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Aug 17 16:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.32300000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Aug 17 16:56:19 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Aug 17 16:56:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.62},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.60199999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298000,"kb_used_data":644,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447152,"statfs":{"total":95995035648,"available":95689883648,"internally_reserved":0,"allocated":659456,"data_stored":278860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:16 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.38300000000000001,"15min":0.38900000000000001},"min":{"1min":0.26600000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.59499999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Aug 17 16:57:16 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44900000000000001,"15min":0.45700000000000002},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.42899999999999999,"15min":0.433},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.57599999999999996,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:57:16 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.44900000000000001,"15min":0.45700000000000002},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61899999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.42299999999999999,"15min":0.42699999999999999},"min":{"1min":0.309,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.52600000000000002,"5min":0.622,"15min":0.622},"last":0.309}]},{"osd":4,"last update":"Sat Aug 17 16:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51900000000000002,"15min":0.52000000000000002},"min":{"1min":0.35599999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Aug 17 16:57:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53900000000000003,"15min":0.54200000000000004},"min":{"1min":0.38500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.53100000000000003,"15min":0.53600000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66900000000000004,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:56:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Aug 17 16:56:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.624},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":279125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:51 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.34300000000000003},"min":{"1min":0.249,"5min":0.186,"15min":0.186},"max":{"1min":0.44800000000000001,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.379,"5min":0.35599999999999998,"15min":0.34899999999999998},"min":{"1min":0.23000000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Aug 17 16:56:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39400000000000002,"15min":0.38400000000000001},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.60099999999999998,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.37},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.41099999999999998,"15min":0.40500000000000003},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.442}]},{"osd":3,"last update":"Sat Aug 17 16:56:17 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.42399999999999999,"15min":0.42199999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58599999999999997,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.26100000000000001}]},{"osd":4,"last update":"Sat Aug 17 16:56:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58499999999999996,"15min":0.58199999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.58199999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:57:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56599999999999995,"15min":0.56899999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.59899999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Aug 17 16:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.64200000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":279144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:56:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.376,"15min":0.36299999999999999},"min":{"1min":0.27700000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.52500000000000002,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.34300000000000003},"min":{"1min":0.24199999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.58099999999999996,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.377}]},{"osd":2,"last update":"Sat Aug 17 16:56:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.375,"15min":0.36699999999999999},"min":{"1min":0.26500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.58099999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39500000000000002,"15min":0.39100000000000001},"min":{"1min":0.29099999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56899999999999995,"5min":0.626,"15min":0.626},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:57:16 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.41199999999999998,"15min":0.41799999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.60099999999999998,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.38,"5min":0.434,"15min":0.443},"min":{"1min":0.28199999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.57999999999999996,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:56:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41399999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78900000000000003,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57399999999999995,"15min":0.57299999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84199999999999997,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.745}]},{"osd":5,"last update":"Sat Aug 17 16:57:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54300000000000004,"15min":0.54200000000000004},"min":{"1min":0.38200000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.73699999999999999,"5min":0.753,"15min":0.753},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.76300000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:56:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.753}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":869493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:56:23 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.33200000000000002},"min":{"1min":0.216,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36499999999999999,"15min":0.35699999999999998},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.53100000000000003,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.315}]},{"osd":2,"last update":"Sat Aug 17 16:56:49 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55800000000000005,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.373},{"interface":"front","average":{"1min":0.376,"5min":0.38900000000000001,"15min":0.39400000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.52900000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:57:15 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.44900000000000001,"15min":0.45600000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.58799999999999997,"5min":0.629,"15min":0.629},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.41599999999999998,"15min":0.42399999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.28699999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:56:35 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54100000000000004,"15min":0.54000000000000004},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.73799999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Aug 17 16:56:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.55200000000000005,"15min":0.55800000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Aug 17 16:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.439},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Aug 17 16:56:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.47799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:57:20.703 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T16:57:20.703 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T16:57:20.704 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T16:57:20.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T16:57:21.261 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:21 smithi142 ceph-mon[31182]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:57:21.262 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:21 smithi142 systemd[1]: Stopping Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:21.262 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:21 smithi142 ceph-mon[40020]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:57:21.301 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:21 smithi152 ceph-mon[34775]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:57:21.872 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:21 smithi142 podman[76447]: 2024-08-17 16:57:21.743752493 +0000 UTC m=+0.470136553 container died f66e788267b09afa364e0cc53d9b7085bbb3fb4f10bbb534a38ff3fb8db013de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=condescending_yalow, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-08-17T16:57:22.291 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:22 smithi152 ceph-mon[34775]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:22.364 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:22 smithi142 ceph-mon[40020]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:22.364 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:22 smithi142 ceph-mon[31182]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:22.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:57:22 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:57:23.115 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[71138]: ts=2024-08-17T16:57:22.737Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:57:23.115 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:22 smithi142 podman[76538]: 2024-08-17 16:57:22.836698545 +0000 UTC m=+0.836773133 container died 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:57:23.354 INFO:teuthology.orchestra.run.smithi142.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T16:57:23.372 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:23 smithi142 podman[76538]: 2024-08-17 16:57:23.244583756 +0000 UTC m=+1.244658331 container cleanup 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:23.372 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:23 smithi142 bash[76538]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T16:57:24.115 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T16:57:24.116 INFO:tasks.cephadm:Setup complete, yielding 2024-08-17T16:57:24.116 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-08-17T16:57:24.130 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi142.front.sepia.ceph.com 2024-08-17T16:57:24.130 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-08-17T16:57:24.309 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:24 smithi142 ceph-mon[40020]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:24.309 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:24 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3717349581' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:57:24.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:24 smithi142 ceph-mon[31182]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:24.310 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:24 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3717349581' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:57:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:24 smithi152 ceph-mon[34775]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:24 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3717349581' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:57:24.516 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:24.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:24 smithi142 podman[76605]: 2024-08-17 16:57:24.308239076 +0000 UTC m=+1.551485577 container remove 2791e630e3564d1b02a99a664592b435c57545e7f14dd2e775675db686e932b3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:24.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:24 smithi142 podman[76605]: 2024-08-17 16:57:24.434213585 +0000 UTC m=+1.677460078 volume remove 653ecd4409fe7c667e3bd0b511a2d4c550841280372ecc8de6bccf870f9bc581 2024-08-17T16:57:25.191 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 podman[76753]: 2024-08-17 16:57:25.020962782 +0000 UTC m=+0.490979319 container died fc82dfb7f1d785096ed616a28010d77ea084d61b08e2c92cae9da961c79a2158 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=amazing_goldberg, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T16:57:25.616 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T16:57:25.616 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 systemd[1]: Stopped Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:25.616 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 systemd[1]: Starting Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:25.867 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 podman[76866]: 2024-08-17 16:57:25.659637201 +0000 UTC m=+0.024616572 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-17T16:57:25.867 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 podman[76866]: 2024-08-17 16:57:25.766504101 +0000 UTC m=+0.131483499 volume create 85b3067985d2297f071fa90a2297808231305b11d92a6cf9ab9464f483a374dd 2024-08-17T16:57:25.867 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:25 smithi142 podman[76866]: 2024-08-17 16:57:25.866932423 +0000 UTC m=+0.231911782 container create d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-mon[40020]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-mon[31182]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:26 smithi152 ceph-mon[34775]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:26.590 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 podman[76866]: 2024-08-17 16:57:26.220788629 +0000 UTC m=+0.585767985 container init d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 podman[76866]: 2024-08-17 16:57:26.225640036 +0000 UTC m=+0.590619393 container start d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.238Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.238Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.239Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.142 port=9094 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.240Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.263Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.263Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.266Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:26.266Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T16:57:26.591 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 bash[76866]: d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 2024-08-17T16:57:26.592 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:26 smithi142 systemd[1]: Started Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: Reconfiguring daemon prometheus.a on smithi152 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:27.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T16:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: Reconfiguring daemon prometheus.a on smithi152 2024-08-17T16:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: Reconfiguring daemon prometheus.a on smithi152 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T16:57:28.521 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T16:57:28.521 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:28 smithi152 ceph-mon[34775]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T16:57:28.521 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T16:57:28.521 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:28.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T16:57:28.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[40020]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T16:57:28.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T16:57:28.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:28.623 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:28.241Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001060058s 2024-08-17T16:57:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T16:57:28.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[31182]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T16:57:28.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T16:57:28.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:28.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[31159]: 2024-08-17T16:57:28.379+0000 7f4acc07e640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:57:29.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-08-17T16:57:29.478 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:29.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: fsmap foo:0 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: Saving service mds.foo spec with placement count:2 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:29.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[31182]: pgmap v49: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: fsmap foo:0 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: Saving service mds.foo spec with placement count:2 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:29.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:29 smithi142 ceph-mon[40020]: pgmap v49: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:29.725 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 systemd[1]: Stopping Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:29.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T16:57:29.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T16:57:29.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T16:57:29.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: fsmap foo:0 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: Saving service mds.foo spec with placement count:2 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:29.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-mon[34775]: pgmap v49: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:30.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.723Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:57:30.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T16:57:30.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T16:57:30.075 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[64569]: ts=2024-08-17T16:57:29.724Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T16:57:30.075 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:29 smithi152 podman[67858]: 2024-08-17 16:57:29.734354977 +0000 UTC m=+0.112686342 container died a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:30.397 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 podman[67858]: 2024-08-17 16:57:30.263812195 +0000 UTC m=+0.642143549 container cleanup a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:30.397 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 bash[67858]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T16:57:30.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:30 smithi142 ceph-mon[40020]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T16:57:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:30 smithi142 ceph-mon[31182]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T16:57:30.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:30 smithi152 ceph-mon[34775]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T16:57:30.679 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 podman[67875]: 2024-08-17 16:57:30.451660974 +0000 UTC m=+0.718957038 container remove a7f64ba6e2907c1ddc871f31c8e004b42d851321057491a48bccaf203d5b27a0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:31.021 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T16:57:31.021 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 systemd[1]: Stopped Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:31.021 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.115s CPU time. 2024-08-17T16:57:31.021 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 systemd[1]: Starting Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:31.021 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 podman[67971]: 2024-08-17 16:57:30.916159564 +0000 UTC m=+0.114225040 container create fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:31.021 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:30 smithi152 podman[67971]: 2024-08-17 16:57:30.819972207 +0000 UTC m=+0.018037625 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T16:57:31.406 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 podman[67971]: 2024-08-17 16:57:31.134820456 +0000 UTC m=+0.332885886 container init fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:31.406 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 podman[67971]: 2024-08-17 16:57:31.14010764 +0000 UTC m=+0.338173061 container start fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:31.406 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.164Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T16:57:31.406 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.164Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T16:57:31.406 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.164Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi152 (none))" 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.164Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.164Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.165Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.166Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.167Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.167Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.168Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.168Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.367µs 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.168Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.170Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.170Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-08-17T16:57:31.407 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.170Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.69µs wal_replay_duration=1.795337ms wbl_replay_duration=111ns total_replay_duration=1.845966ms 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.171Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.171Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.171Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.185Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.415965ms db_storage=674ns remote_storage=1.356µs web_handler=441ns query_engine=622ns scrape=398.576µs scrape_sd=25.618µs notify=22.271µs notify_sd=6.255µs rules=13.733321ms tracing=5.19µs 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.185Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T16:57:31.185Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 bash[67971]: fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 2024-08-17T16:57:31.408 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 16:57:31 smithi152 systemd[1]: Started Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: pgmap v52: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi142.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: pgmap v52: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi142.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:57:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STOPPING 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STOPPED 2024-08-17T16:57:31.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STARTING 2024-08-17T16:57:31.626 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Serving on http://:::9283 2024-08-17T16:57:31.626 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STARTED 2024-08-17T16:57:31.626 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STOPPING 2024-08-17T16:57:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T16:57:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: pgmap v52: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi142.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:31.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:31 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:32.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-08-17T16:57:32.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:57:32.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STOPPED 2024-08-17T16:57:32.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:31] ENGINE Bus STARTING 2024-08-17T16:57:32.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Serving on http://:::9283 2024-08-17T16:57:32.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Bus STARTED 2024-08-17T16:57:32.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Bus STOPPING 2024-08-17T16:57:32.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi142.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Bus STOPPED 2024-08-17T16:57:32.681 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Bus STARTING 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Serving on http://:::9283 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:57:32] ENGINE Bus STARTED 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi142.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:57:32.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:32 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:57:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:32 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:57:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:32 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi142.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:57:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:32 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:32 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:57:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:32 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:57:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:32 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:57:32.920 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:33 smithi142 ceph-mon[40020]: pgmap v53: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:33 smithi142 ceph-mon[31182]: pgmap v53: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:33.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:33 smithi152 ceph-mon[34775]: pgmap v53: 73 pgs: 7 creating+activating, 24 creating+peering, 42 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:36.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:35 smithi142 ceph-mon[40020]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:36.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:35 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:36.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:35 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:36.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:35 smithi142 ceph-mon[31182]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:36.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:35 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:36.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:35 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:36.072 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T16:57:36.081 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T16:57:36.081 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T16:57:36.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T16:57:36.082 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:35 smithi152 ceph-mon[34775]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:36.082 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:35 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:36.082 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:35 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:36.372 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 16:57:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T16:57:36.244Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004372863s 2024-08-17T16:57:36.883 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:38.117 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[40020]: Deploying daemon mds.foo.smithi142.ndmoid on smithi142 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:37 smithi142 ceph-mon[31182]: Deploying daemon mds.foo.smithi142.ndmoid on smithi142 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:57:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:37 smithi152 ceph-mon[34775]: Deploying daemon mds.foo.smithi142.ndmoid on smithi142 2024-08-17T16:57:38.368 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:38] "GET /metrics HTTP/1.1" 200 206135 "" "Prometheus/2.43.0" 2024-08-17T16:57:39.459 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:57:40.029 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:39 smithi142 ceph-mon[40020]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:40.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:39 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/926102176' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:57:40.030 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:39 smithi142 ceph-mon[31182]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:40.030 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:39 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/926102176' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:57:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:39 smithi152 ceph-mon[34775]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:39 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/926102176' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:57:40.792 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":20,"flags":0,"active_gid":14445,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6800","nonce":3356817710},{"type":"v1","addr":"172.21.15.142:6801","nonce":3356817710}]},"active_addr":"172.21.15.142:6801/3356817710","active_change":"2024-08-17T16:56:12.454867+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":61,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":2472242802}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":2627951813}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":1554300251}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":639679378}]}]} 2024-08-17T16:57:40.795 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T16:57:40.796 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T16:57:40.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:57:41.426 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:42.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: daemon mds.foo.smithi142.ndmoid assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: Cluster is now healthy 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:boot 2024-08-17T16:57:42.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:creating} 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[40020]: daemon mds.foo.smithi142.ndmoid is now active in filesystem foo as rank 0 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:57:42.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: daemon mds.foo.smithi142.ndmoid assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: Cluster is now healthy 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:boot 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:creating} 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T16:57:42.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:41 smithi142 ceph-mon[31182]: daemon mds.foo.smithi142.ndmoid is now active in filesystem foo as rank 0 2024-08-17T16:57:42.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:42.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: daemon mds.foo.smithi142.ndmoid assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: Cluster is now healthy 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:boot 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:creating} 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T16:57:42.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:41 smithi152 ceph-mon[34775]: daemon mds.foo.smithi142.ndmoid is now active in filesystem foo as rank 0 2024-08-17T16:57:42.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:57:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[40020]: Deploying daemon mds.foo.smithi152.tacamo on smithi152 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[40020]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:active 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[31182]: Deploying daemon mds.foo.smithi152.tacamo on smithi152 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[31182]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:active 2024-08-17T16:57:43.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:42 smithi142 ceph-mon[31182]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 2024-08-17T16:57:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:57:43.193 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":71,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T16:57:30.395283+0000","last_up_change":"2024-08-17T16:55:41.730067+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:57:27.258162+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:57:27.611866+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6803","nonce":3785349577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6805","nonce":3785349577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6809","nonce":3785349577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6807","nonce":3785349577}]},"public_addr":"172.21.15.142:6803/3785349577","cluster_addr":"172.21.15.142:6805/3785349577","heartbeat_back_addr":"172.21.15.142:6809/3785349577","heartbeat_front_addr":"172.21.15.142:6807/3785349577","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2041524993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2041524993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2041524993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2041524993}]},"public_addr":"172.21.15.142:6811/2041524993","cluster_addr":"172.21.15.142:6813/2041524993","heartbeat_back_addr":"172.21.15.142:6817/2041524993","heartbeat_front_addr":"172.21.15.142:6815/2041524993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6819","nonce":3610394488}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6821","nonce":3610394488}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6825","nonce":3610394488}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6823","nonce":3610394488}]},"public_addr":"172.21.15.142:6819/3610394488","cluster_addr":"172.21.15.142:6821/3610394488","heartbeat_back_addr":"172.21.15.142:6825/3610394488","heartbeat_front_addr":"172.21.15.142:6823/3610394488","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6827","nonce":2365382971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6829","nonce":2365382971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6833","nonce":2365382971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6831","nonce":2365382971}]},"public_addr":"172.21.15.142:6827/2365382971","cluster_addr":"172.21.15.142:6829/2365382971","heartbeat_back_addr":"172.21.15.142:6833/2365382971","heartbeat_front_addr":"172.21.15.142:6831/2365382971","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:57:43.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:42 smithi152 ceph-mon[34775]: Deploying daemon mds.foo.smithi152.tacamo on smithi152 2024-08-17T16:57:43.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:43.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:42 smithi152 ceph-mon[34775]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:active 2024-08-17T16:57:43.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:42 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 2024-08-17T16:57:43.827 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T16:57:43.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:57:43.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:43 smithi152 ceph-mon[34775]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:43.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:43 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3953717462' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:57:44.093 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:44.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:43 smithi142 ceph-mon[40020]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:44.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:43 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3953717462' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:57:44.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:43 smithi142 ceph-mon[31182]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:57:44.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:43 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3953717462' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:57:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:45.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: mds.? [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] up:boot 2024-08-17T16:57:45.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:45.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T16:57:45.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:45 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:45.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: mds.? [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] up:boot 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T16:57:45.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: mds.? [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] up:boot 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T16:57:45.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:45 smithi142 ceph-mon[31182]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:46.044 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:57:46.044 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":71,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T16:57:30.395283+0000","last_up_change":"2024-08-17T16:55:41.730067+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:57:27.258162+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:57:27.611866+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6803","nonce":3785349577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6805","nonce":3785349577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6809","nonce":3785349577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":3785349577},{"type":"v1","addr":"172.21.15.142:6807","nonce":3785349577}]},"public_addr":"172.21.15.142:6803/3785349577","cluster_addr":"172.21.15.142:6805/3785349577","heartbeat_back_addr":"172.21.15.142:6809/3785349577","heartbeat_front_addr":"172.21.15.142:6807/3785349577","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2041524993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2041524993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2041524993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2041524993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2041524993}]},"public_addr":"172.21.15.142:6811/2041524993","cluster_addr":"172.21.15.142:6813/2041524993","heartbeat_back_addr":"172.21.15.142:6817/2041524993","heartbeat_front_addr":"172.21.15.142:6815/2041524993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6819","nonce":3610394488}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6821","nonce":3610394488}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6825","nonce":3610394488}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":3610394488},{"type":"v1","addr":"172.21.15.142:6823","nonce":3610394488}]},"public_addr":"172.21.15.142:6819/3610394488","cluster_addr":"172.21.15.142:6821/3610394488","heartbeat_back_addr":"172.21.15.142:6825/3610394488","heartbeat_front_addr":"172.21.15.142:6823/3610394488","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6827","nonce":2365382971}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6829","nonce":2365382971}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6833","nonce":2365382971}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":2365382971},{"type":"v1","addr":"172.21.15.142:6831","nonce":2365382971}]},"public_addr":"172.21.15.142:6827/2365382971","cluster_addr":"172.21.15.142:6829/2365382971","heartbeat_back_addr":"172.21.15.142:6833/2365382971","heartbeat_front_addr":"172.21.15.142:6831/2365382971","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:57:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:46 smithi152 ceph-mon[34775]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:46 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2150082997' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:57:46.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:46 smithi142 ceph-mon[40020]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:46.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:46 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2150082997' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:57:46.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:46 smithi142 ceph-mon[31182]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:46.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:46 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2150082997' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:57:46.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T16:57:46.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T16:57:46.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T16:57:46.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T16:57:46.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T16:57:46.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T16:57:46.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T16:57:46.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T16:57:47.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:47 smithi142 ceph-mon[31182]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:active 2024-08-17T16:57:47.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:47 smithi142 ceph-mon[31182]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:47 smithi142 ceph-mon[40020]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:active 2024-08-17T16:57:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:47 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:47 smithi152 ceph-mon[34775]: mds.? [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] up:active 2024-08-17T16:57:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:47 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:47.465 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.477 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.477 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.478 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.481 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.482 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.484 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:47.494 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:57:48.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[40020]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:48.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[31182]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:48.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:48 smithi152 ceph-mon[34775]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:48 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:48 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:48 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:57:48.477 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:48] "GET /metrics HTTP/1.1" 200 239147 "" "Prometheus/2.43.0" 2024-08-17T16:57:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:49 smithi142 ceph-mon[40020]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:49 smithi142 ceph-mon[40020]: mds.? [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] up:standby 2024-08-17T16:57:49.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:49 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:49 smithi142 ceph-mon[31182]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:49 smithi142 ceph-mon[31182]: mds.? [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] up:standby 2024-08-17T16:57:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:49 smithi142 ceph-mon[31182]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:49 smithi152 ceph-mon[34775]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:49 smithi152 ceph-mon[34775]: mds.? [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] up:standby 2024-08-17T16:57:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:49 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T16:57:51.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:51 smithi142 ceph-mon[40020]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:51.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:51 smithi142 ceph-mon[31182]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:51 smithi152 ceph-mon[34775]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:52.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:57:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:57:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:53 smithi152 ceph-mon[34775]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:54.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:53 smithi142 ceph-mon[40020]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:53 smithi142 ceph-mon[31182]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:55.895 INFO:teuthology.orchestra.run.smithi142.stdout:201863462945 2024-08-17T16:57:55.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T16:57:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:55 smithi152 ceph-mon[34775]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:56.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:55 smithi142 ceph-mon[40020]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:56.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:55 smithi142 ceph-mon[31182]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T16:57:58.095 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:57 smithi142 ceph-mon[40020]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:57:58.095 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:57 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:57 smithi142 ceph-mon[31182]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:57 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:58.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:57 smithi152 ceph-mon[34775]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:57:58.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:57 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:58.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:57:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:57:58] "GET /metrics HTTP/1.1" 200 239667 "" "Prometheus/2.43.0" 2024-08-17T16:57:59.194 INFO:teuthology.orchestra.run.smithi142.stdout:133143986220 2024-08-17T16:57:59.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T16:57:59.947 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:57:59 smithi142 ceph-mon[40020]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:57:59.948 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:57:59 smithi142 ceph-mon[31182]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:58:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:57:59 smithi152 ceph-mon[34775]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:58:02.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:01 smithi142 ceph-mon[40020]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:01 smithi142 ceph-mon[31182]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:02.172 INFO:teuthology.orchestra.run.smithi142.stdout:167503724585 2024-08-17T16:58:02.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T16:58:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:01 smithi152 ceph-mon[34775]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:02.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:58:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:58:03.486 INFO:teuthology.orchestra.run.smithi142.stdout:55834574908 2024-08-17T16:58:03.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T16:58:04.031 INFO:teuthology.orchestra.run.smithi142.stdout:34359738432 2024-08-17T16:58:04.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T16:58:04.101 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:03 smithi152 ceph-mon[34775]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:04.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:03 smithi142 ceph-mon[40020]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:04.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:03 smithi142 ceph-mon[31182]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:04.765 INFO:teuthology.orchestra.run.smithi142.stdout:236223201311 2024-08-17T16:58:04.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T16:58:05.216 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:05.217 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:05.217 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:05.607 INFO:teuthology.orchestra.run.smithi142.stdout:77309411383 2024-08-17T16:58:05.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T16:58:06.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:05 smithi142 ceph-mon[40020]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:06.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:05 smithi142 ceph-mon[31182]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:05 smithi152 ceph-mon[34775]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[40020]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:07.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:07.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:07.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:07 smithi142 ceph-mon[31182]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:07.632 INFO:teuthology.orchestra.run.smithi142.stdout:111669149743 2024-08-17T16:58:07.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T16:58:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:07.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:07 smithi152 ceph-mon[34775]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:08.588 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:08] "GET /metrics HTTP/1.1" 200 239667 "" "Prometheus/2.43.0" 2024-08-17T16:58:09.804 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:09 smithi142 ceph-mon[40020]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:09.804 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:09 smithi142 ceph-mon[31182]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:09 smithi152 ceph-mon[34775]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:10.567 INFO:teuthology.orchestra.run.smithi142.stdout:201863462949 2024-08-17T16:58:10.804 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:11.142 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:11.143 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:11.143 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:11.258 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:10 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:11.258 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:10 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:11.258 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:10 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3912161088' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:58:11.259 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:10 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:11.259 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:10 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:11.259 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:10 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3912161088' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:58:11.275 INFO:teuthology.orchestra.run.smithi142.stdout:167503724587 2024-08-17T16:58:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:10 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:10 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:10 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3912161088' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:58:11.966 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724585 got 167503724587 for osd.5 2024-08-17T16:58:11.967 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:12.206 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:11 smithi142 ceph-mon[31182]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:12.207 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:11 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1900344814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:58:12.207 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:11 smithi142 ceph-mon[40020]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:12.207 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:11 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1900344814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:58:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:11 smithi152 ceph-mon[34775]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:11 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1900344814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:58:12.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:58:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:58:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-08-17T16:58:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-08-17T16:58:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-08-17T16:58:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-08-17T16:58:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:13.611 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462945 got 201863462949 for osd.6 2024-08-17T16:58:13.611 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:14.299 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:14 smithi152 ceph-mon[34775]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:14 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-08-17T16:58:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:14 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-08-17T16:58:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:14 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-08-17T16:58:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:14 smithi152 ceph-mon[34775]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[31182]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[31182]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[40020]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-08-17T16:58:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-08-17T16:58:14.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-08-17T16:58:14.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:14 smithi142 ceph-mon[40020]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T16:58:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:15 smithi152 ceph-mon[34775]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T16:58:15.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:15 smithi142 ceph-mon[40020]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T16:58:15.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:15 smithi142 ceph-mon[31182]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T16:58:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:16 smithi142 ceph-mon[31182]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:16 smithi142 ceph-mon[40020]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:16 smithi152 ceph-mon[34775]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:17.675 INFO:teuthology.orchestra.run.smithi142.stdout:133143986224 2024-08-17T16:58:18.309 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:18 smithi142 ceph-mon[40020]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:18.309 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:18 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2085733606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:58:18.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:18 smithi142 ceph-mon[31182]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:18.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:18 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2085733606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:58:18.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:18 smithi152 ceph-mon[34775]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:18.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:18 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2085733606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:58:18.586 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:18] "GET /metrics HTTP/1.1" 200 239640 "" "Prometheus/2.43.0" 2024-08-17T16:58:19.051 INFO:teuthology.orchestra.run.smithi142.stdout:55834574912 2024-08-17T16:58:19.941 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574908 got 55834574912 for osd.1 2024-08-17T16:58:19.941 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:20.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:20 smithi142 ceph-mon[31182]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:20.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:20 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1873769119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:58:20.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:20 smithi142 ceph-mon[40020]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:20.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:20 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1873769119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:58:20.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:20 smithi152 ceph-mon[34775]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:20.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:20 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1873769119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:58:21.725 INFO:teuthology.orchestra.run.smithi142.stdout:111669149748 2024-08-17T16:58:21.913 INFO:teuthology.orchestra.run.smithi142.stdout:77309411386 2024-08-17T16:58:22.265 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:22 smithi142 ceph-mon[40020]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:22.265 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:22 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3680914479' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:58:22.265 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:22 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3920993850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:58:22.265 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:22 smithi142 ceph-mon[31182]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:22.265 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:22 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3680914479' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:58:22.265 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:22 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3920993850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:58:22.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:22 smithi152 ceph-mon[34775]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:22.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:22 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3680914479' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:58:22.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:22 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3920993850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:58:22.664 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-08-17T16:58:22.665 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:22.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:58:22 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:58:23.313 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986220 got 133143986224 for osd.4 2024-08-17T16:58:23.313 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:23.644 INFO:teuthology.orchestra.run.smithi142.stdout:34359738436 2024-08-17T16:58:23.846 INFO:teuthology.orchestra.run.smithi142.stdout:236223201315 2024-08-17T16:58:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:24 smithi142 ceph-mon[31182]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:24 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2202838347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:58:24.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:24 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/312399612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:58:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:24 smithi142 ceph-mon[40020]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:24 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2202838347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:58:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:24 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/312399612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:58:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:24 smithi152 ceph-mon[34775]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:24 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2202838347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:58:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:24 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/312399612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:58:24.605 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201315 for osd.7 2024-08-17T16:58:24.606 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:25.007 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149743 got 111669149748 for osd.3 2024-08-17T16:58:25.007 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:25.576 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738432 got 34359738436 for osd.0 2024-08-17T16:58:25.576 DEBUG:teuthology.parallel:result is None 2024-08-17T16:58:25.576 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T16:58:25.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:58:26.217 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:26 smithi152 ceph-mon[34775]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:26.340 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:26 smithi142 ceph-mon[40020]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:26.340 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:26 smithi142 ceph-mon[31182]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:28.223 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:28 smithi142 ceph-mon[40020]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:28.223 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:28.223 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:28 smithi142 ceph-mon[31182]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:28.223 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:28 smithi152 ceph-mon[34775]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:28.514 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:28] "GET /metrics HTTP/1.1" 200 239675 "" "Prometheus/2.43.0" 2024-08-17T16:58:29.245 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:58:29.246 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T16:58:29.932 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":83,"stamp":"2024-08-17T16:58:28.494580+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386564,"kb_used_data":7692,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574652,"statfs":{"total":767960285184,"available":765516443648,"internally_reserved":0,"allocated":7876608,"data_stored":4413738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:58:14.057872+0000","last_change":"2024-08-17T16:58:14.057872+0000","last_active":"2024-08-17T16:58:14.057872+0000","last_peered":"2024-08-17T16:58:14.057872+0000","last_clean":"2024-08-17T16:58:14.057872+0000","last_became_active":"2024-08-17T16:58:14.057545+0000","last_became_peered":"2024-08-17T16:58:14.057545+0000","last_unstale":"2024-08-17T16:58:14.057872+0000","last_undegraded":"2024-08-17T16:58:14.057872+0000","last_fullsized":"2024-08-17T16:58:14.057872+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:53:02.696571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013521+0000","last_change":"2024-08-17T16:57:28.388724+0000","last_active":"2024-08-17T16:58:13.013521+0000","last_peered":"2024-08-17T16:58:13.013521+0000","last_clean":"2024-08-17T16:58:13.013521+0000","last_became_active":"2024-08-17T16:57:28.388374+0000","last_became_peered":"2024-08-17T16:57:28.388374+0000","last_unstale":"2024-08-17T16:58:13.013521+0000","last_undegraded":"2024-08-17T16:58:13.013521+0000","last_fullsized":"2024-08-17T16:58:13.013521+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:00:03.425629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010335+0000","last_change":"2024-08-17T16:57:28.770576+0000","last_active":"2024-08-17T16:58:13.010335+0000","last_peered":"2024-08-17T16:58:13.010335+0000","last_clean":"2024-08-17T16:58:13.010335+0000","last_became_active":"2024-08-17T16:57:28.770338+0000","last_became_peered":"2024-08-17T16:57:28.770338+0000","last_unstale":"2024-08-17T16:58:13.010335+0000","last_undegraded":"2024-08-17T16:58:13.010335+0000","last_fullsized":"2024-08-17T16:58:13.010335+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:50.039329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008958+0000","last_change":"2024-08-17T16:57:28.389122+0000","last_active":"2024-08-17T16:58:13.008958+0000","last_peered":"2024-08-17T16:58:13.008958+0000","last_clean":"2024-08-17T16:58:13.008958+0000","last_became_active":"2024-08-17T16:57:28.388809+0000","last_became_peered":"2024-08-17T16:57:28.388809+0000","last_unstale":"2024-08-17T16:58:13.008958+0000","last_undegraded":"2024-08-17T16:58:13.008958+0000","last_fullsized":"2024-08-17T16:58:13.008958+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:04.416967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009660+0000","last_change":"2024-08-17T16:57:28.771324+0000","last_active":"2024-08-17T16:58:13.009660+0000","last_peered":"2024-08-17T16:58:13.009660+0000","last_clean":"2024-08-17T16:58:13.009660+0000","last_became_active":"2024-08-17T16:57:28.771053+0000","last_became_peered":"2024-08-17T16:57:28.771053+0000","last_unstale":"2024-08-17T16:58:13.009660+0000","last_undegraded":"2024-08-17T16:58:13.009660+0000","last_fullsized":"2024-08-17T16:58:13.009660+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:33.216252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013226+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T16:58:13.013226+0000","last_peered":"2024-08-17T16:58:13.013226+0000","last_clean":"2024-08-17T16:58:13.013226+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T16:58:13.013226+0000","last_undegraded":"2024-08-17T16:58:13.013226+0000","last_fullsized":"2024-08-17T16:58:13.013226+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011449+0000","last_change":"2024-08-17T16:57:30.217818+0000","last_active":"2024-08-17T16:58:13.011449+0000","last_peered":"2024-08-17T16:58:13.011449+0000","last_clean":"2024-08-17T16:58:13.011449+0000","last_became_active":"2024-08-17T16:57:30.217533+0000","last_became_peered":"2024-08-17T16:57:30.217533+0000","last_unstale":"2024-08-17T16:58:13.011449+0000","last_undegraded":"2024-08-17T16:58:13.011449+0000","last_fullsized":"2024-08-17T16:58:13.011449+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:26:29.439464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.267994+0000","last_change":"2024-08-17T16:57:28.384794+0000","last_active":"2024-08-17T16:58:13.267994+0000","last_peered":"2024-08-17T16:58:13.267994+0000","last_clean":"2024-08-17T16:58:13.267994+0000","last_became_active":"2024-08-17T16:57:28.384535+0000","last_became_peered":"2024-08-17T16:57:28.384535+0000","last_unstale":"2024-08-17T16:58:13.267994+0000","last_undegraded":"2024-08-17T16:58:13.267994+0000","last_fullsized":"2024-08-17T16:58:13.267994+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:00.999336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013831+0000","last_change":"2024-08-17T16:57:28.389191+0000","last_active":"2024-08-17T16:58:13.013831+0000","last_peered":"2024-08-17T16:58:13.013831+0000","last_clean":"2024-08-17T16:58:13.013831+0000","last_became_active":"2024-08-17T16:57:28.388931+0000","last_became_peered":"2024-08-17T16:57:28.388931+0000","last_unstale":"2024-08-17T16:58:13.013831+0000","last_undegraded":"2024-08-17T16:58:13.013831+0000","last_fullsized":"2024-08-17T16:58:13.013831+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:05:53.117510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010182+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T16:58:13.010182+0000","last_peered":"2024-08-17T16:58:13.010182+0000","last_clean":"2024-08-17T16:58:13.010182+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T16:58:13.010182+0000","last_undegraded":"2024-08-17T16:58:13.010182+0000","last_fullsized":"2024-08-17T16:58:13.010182+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013886+0000","last_change":"2024-08-17T16:57:28.391656+0000","last_active":"2024-08-17T16:58:13.013886+0000","last_peered":"2024-08-17T16:58:13.013886+0000","last_clean":"2024-08-17T16:58:13.013886+0000","last_became_active":"2024-08-17T16:57:28.391401+0000","last_became_peered":"2024-08-17T16:57:28.391401+0000","last_unstale":"2024-08-17T16:58:13.013886+0000","last_undegraded":"2024-08-17T16:58:13.013886+0000","last_fullsized":"2024-08-17T16:58:13.013886+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:07.880114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.007891+0000","last_change":"2024-08-17T16:57:29.408130+0000","last_active":"2024-08-17T16:58:13.007891+0000","last_peered":"2024-08-17T16:58:13.007891+0000","last_clean":"2024-08-17T16:58:13.007891+0000","last_became_active":"2024-08-17T16:57:29.407695+0000","last_became_peered":"2024-08-17T16:57:29.407695+0000","last_unstale":"2024-08-17T16:58:13.007891+0000","last_undegraded":"2024-08-17T16:58:13.007891+0000","last_fullsized":"2024-08-17T16:58:13.007891+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:27.859334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013914+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T16:58:13.013914+0000","last_peered":"2024-08-17T16:58:13.013914+0000","last_clean":"2024-08-17T16:58:13.013914+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T16:58:13.013914+0000","last_undegraded":"2024-08-17T16:58:13.013914+0000","last_fullsized":"2024-08-17T16:58:13.013914+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.267746+0000","last_change":"2024-08-17T16:57:30.217703+0000","last_active":"2024-08-17T16:58:13.267746+0000","last_peered":"2024-08-17T16:58:13.267746+0000","last_clean":"2024-08-17T16:58:13.267746+0000","last_became_active":"2024-08-17T16:57:30.217312+0000","last_became_peered":"2024-08-17T16:57:30.217312+0000","last_unstale":"2024-08-17T16:58:13.267746+0000","last_undegraded":"2024-08-17T16:58:13.267746+0000","last_fullsized":"2024-08-17T16:58:13.267746+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:10.443728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010441+0000","last_change":"2024-08-17T16:57:28.386096+0000","last_active":"2024-08-17T16:58:13.010441+0000","last_peered":"2024-08-17T16:58:13.010441+0000","last_clean":"2024-08-17T16:58:13.010441+0000","last_became_active":"2024-08-17T16:57:28.385810+0000","last_became_peered":"2024-08-17T16:57:28.385810+0000","last_unstale":"2024-08-17T16:58:13.010441+0000","last_undegraded":"2024-08-17T16:58:13.010441+0000","last_fullsized":"2024-08-17T16:58:13.010441+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:43.215952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009244+0000","last_change":"2024-08-17T16:57:28.772065+0000","last_active":"2024-08-17T16:58:13.009244+0000","last_peered":"2024-08-17T16:58:13.009244+0000","last_clean":"2024-08-17T16:58:13.009244+0000","last_became_active":"2024-08-17T16:57:28.771736+0000","last_became_peered":"2024-08-17T16:57:28.771736+0000","last_unstale":"2024-08-17T16:58:13.009244+0000","last_undegraded":"2024-08-17T16:58:13.009244+0000","last_fullsized":"2024-08-17T16:58:13.009244+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:02.662730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010958+0000","last_change":"2024-08-17T16:57:28.389633+0000","last_active":"2024-08-17T16:58:13.010958+0000","last_peered":"2024-08-17T16:58:13.010958+0000","last_clean":"2024-08-17T16:58:13.010958+0000","last_became_active":"2024-08-17T16:57:28.389355+0000","last_became_peered":"2024-08-17T16:57:28.389355+0000","last_unstale":"2024-08-17T16:58:13.010958+0000","last_undegraded":"2024-08-17T16:58:13.010958+0000","last_fullsized":"2024-08-17T16:58:13.010958+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:09.796711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008097+0000","last_change":"2024-08-17T16:57:29.404439+0000","last_active":"2024-08-17T16:58:13.008097+0000","last_peered":"2024-08-17T16:58:13.008097+0000","last_clean":"2024-08-17T16:58:13.008097+0000","last_became_active":"2024-08-17T16:57:29.404033+0000","last_became_peered":"2024-08-17T16:57:29.404033+0000","last_unstale":"2024-08-17T16:58:13.008097+0000","last_undegraded":"2024-08-17T16:58:13.008097+0000","last_fullsized":"2024-08-17T16:58:13.008097+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:52.322809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.267874+0000","last_change":"2024-08-17T16:57:28.386622+0000","last_active":"2024-08-17T16:58:13.267874+0000","last_peered":"2024-08-17T16:58:13.267874+0000","last_clean":"2024-08-17T16:58:13.267874+0000","last_became_active":"2024-08-17T16:57:28.386335+0000","last_became_peered":"2024-08-17T16:57:28.386335+0000","last_unstale":"2024-08-17T16:58:13.267874+0000","last_undegraded":"2024-08-17T16:58:13.267874+0000","last_fullsized":"2024-08-17T16:58:13.267874+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:09.308835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008365+0000","last_change":"2024-08-17T16:57:30.007765+0000","last_active":"2024-08-17T16:58:13.008365+0000","last_peered":"2024-08-17T16:58:13.008365+0000","last_clean":"2024-08-17T16:58:13.008365+0000","last_became_active":"2024-08-17T16:57:30.007479+0000","last_became_peered":"2024-08-17T16:57:30.007479+0000","last_unstale":"2024-08-17T16:58:13.008365+0000","last_undegraded":"2024-08-17T16:58:13.008365+0000","last_fullsized":"2024-08-17T16:58:13.008365+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:33:53.873135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008207+0000","last_change":"2024-08-17T16:57:28.391237+0000","last_active":"2024-08-17T16:58:13.008207+0000","last_peered":"2024-08-17T16:58:13.008207+0000","last_clean":"2024-08-17T16:58:13.008207+0000","last_became_active":"2024-08-17T16:57:28.390939+0000","last_became_peered":"2024-08-17T16:57:28.390939+0000","last_unstale":"2024-08-17T16:58:13.008207+0000","last_undegraded":"2024-08-17T16:58:13.008207+0000","last_fullsized":"2024-08-17T16:58:13.008207+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:21.419059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008541+0000","last_change":"2024-08-17T16:57:29.407933+0000","last_active":"2024-08-17T16:58:13.008541+0000","last_peered":"2024-08-17T16:58:13.008541+0000","last_clean":"2024-08-17T16:58:13.008541+0000","last_became_active":"2024-08-17T16:57:29.407620+0000","last_became_peered":"2024-08-17T16:57:29.407620+0000","last_unstale":"2024-08-17T16:58:13.008541+0000","last_undegraded":"2024-08-17T16:58:13.008541+0000","last_fullsized":"2024-08-17T16:58:13.008541+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:27.092779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010957+0000","last_change":"2024-08-17T16:57:28.392865+0000","last_active":"2024-08-17T16:58:13.010957+0000","last_peered":"2024-08-17T16:58:13.010957+0000","last_clean":"2024-08-17T16:58:13.010957+0000","last_became_active":"2024-08-17T16:57:28.391905+0000","last_became_peered":"2024-08-17T16:57:28.391905+0000","last_unstale":"2024-08-17T16:58:13.010957+0000","last_undegraded":"2024-08-17T16:58:13.010957+0000","last_fullsized":"2024-08-17T16:58:13.010957+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:51.664933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:58:14.015634+0000","last_change":"2024-08-17T16:58:14.015634+0000","last_active":"2024-08-17T16:58:14.015634+0000","last_peered":"2024-08-17T16:58:14.015634+0000","last_clean":"2024-08-17T16:58:14.015634+0000","last_became_active":"2024-08-17T16:58:14.015258+0000","last_became_peered":"2024-08-17T16:58:14.015258+0000","last_unstale":"2024-08-17T16:58:14.015634+0000","last_undegraded":"2024-08-17T16:58:14.015634+0000","last_fullsized":"2024-08-17T16:58:14.015634+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:48.766414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":279,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013093+0000","last_change":"2024-08-17T16:55:45.748632+0000","last_active":"2024-08-17T16:58:13.013093+0000","last_peered":"2024-08-17T16:58:13.013093+0000","last_clean":"2024-08-17T16:58:13.013093+0000","last_became_active":"2024-08-17T16:55:45.748242+0000","last_became_peered":"2024-08-17T16:55:45.748242+0000","last_unstale":"2024-08-17T16:58:13.013093+0000","last_undegraded":"2024-08-17T16:58:13.013093+0000","last_fullsized":"2024-08-17T16:58:13.013093+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:22.553778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010372+0000","last_change":"2024-08-17T16:56:17.492706+0000","last_active":"2024-08-17T16:58:13.010372+0000","last_peered":"2024-08-17T16:58:13.010372+0000","last_clean":"2024-08-17T16:58:13.010372+0000","last_became_active":"2024-08-17T16:56:12.470130+0000","last_became_peered":"2024-08-17T16:56:12.470130+0000","last_unstale":"2024-08-17T16:58:13.010372+0000","last_undegraded":"2024-08-17T16:58:13.010372+0000","last_fullsized":"2024-08-17T16:58:13.010372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:00.149068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010353+0000","last_change":"2024-08-17T16:57:28.386845+0000","last_active":"2024-08-17T16:58:13.010353+0000","last_peered":"2024-08-17T16:58:13.010353+0000","last_clean":"2024-08-17T16:58:13.010353+0000","last_became_active":"2024-08-17T16:57:28.386539+0000","last_became_peered":"2024-08-17T16:57:28.386539+0000","last_unstale":"2024-08-17T16:58:13.010353+0000","last_undegraded":"2024-08-17T16:58:13.010353+0000","last_fullsized":"2024-08-17T16:58:13.010353+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.239896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.268064+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T16:58:13.268064+0000","last_peered":"2024-08-17T16:58:13.268064+0000","last_clean":"2024-08-17T16:58:13.268064+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T16:58:13.268064+0000","last_undegraded":"2024-08-17T16:58:13.268064+0000","last_fullsized":"2024-08-17T16:58:13.268064+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013077+0000","last_change":"2024-08-17T16:56:17.492695+0000","last_active":"2024-08-17T16:58:13.013077+0000","last_peered":"2024-08-17T16:58:13.013077+0000","last_clean":"2024-08-17T16:58:13.013077+0000","last_became_active":"2024-08-17T16:56:12.469470+0000","last_became_peered":"2024-08-17T16:56:12.469470+0000","last_unstale":"2024-08-17T16:58:13.013077+0000","last_undegraded":"2024-08-17T16:58:13.013077+0000","last_fullsized":"2024-08-17T16:58:13.013077+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:02.237538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008999+0000","last_change":"2024-08-17T16:57:28.389255+0000","last_active":"2024-08-17T16:58:13.008999+0000","last_peered":"2024-08-17T16:58:13.008999+0000","last_clean":"2024-08-17T16:58:13.008999+0000","last_became_active":"2024-08-17T16:57:28.388979+0000","last_became_peered":"2024-08-17T16:57:28.388979+0000","last_unstale":"2024-08-17T16:58:13.008999+0000","last_undegraded":"2024-08-17T16:58:13.008999+0000","last_fullsized":"2024-08-17T16:58:13.008999+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:20.131361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009591+0000","last_change":"2024-08-17T16:57:28.404457+0000","last_active":"2024-08-17T16:58:13.009591+0000","last_peered":"2024-08-17T16:58:13.009591+0000","last_clean":"2024-08-17T16:58:13.009591+0000","last_became_active":"2024-08-17T16:57:28.404142+0000","last_became_peered":"2024-08-17T16:57:28.404142+0000","last_unstale":"2024-08-17T16:58:13.009591+0000","last_undegraded":"2024-08-17T16:58:13.009591+0000","last_fullsized":"2024-08-17T16:58:13.009591+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:15.012100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010610+0000","last_change":"2024-08-17T16:56:17.485644+0000","last_active":"2024-08-17T16:58:13.010610+0000","last_peered":"2024-08-17T16:58:13.010610+0000","last_clean":"2024-08-17T16:58:13.010610+0000","last_became_active":"2024-08-17T16:56:12.897452+0000","last_became_peered":"2024-08-17T16:56:12.897452+0000","last_unstale":"2024-08-17T16:58:13.010610+0000","last_undegraded":"2024-08-17T16:58:13.010610+0000","last_fullsized":"2024-08-17T16:58:13.010610+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:18.635731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012007+0000","last_change":"2024-08-17T16:57:28.395682+0000","last_active":"2024-08-17T16:58:13.012007+0000","last_peered":"2024-08-17T16:58:13.012007+0000","last_clean":"2024-08-17T16:58:13.012007+0000","last_became_active":"2024-08-17T16:57:28.395416+0000","last_became_peered":"2024-08-17T16:57:28.395416+0000","last_unstale":"2024-08-17T16:58:13.012007+0000","last_undegraded":"2024-08-17T16:58:13.012007+0000","last_fullsized":"2024-08-17T16:58:13.012007+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:36.466479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012624+0000","last_change":"2024-08-17T16:57:29.400497+0000","last_active":"2024-08-17T16:58:13.012624+0000","last_peered":"2024-08-17T16:58:13.012624+0000","last_clean":"2024-08-17T16:58:13.012624+0000","last_became_active":"2024-08-17T16:57:29.399887+0000","last_became_peered":"2024-08-17T16:57:29.399887+0000","last_unstale":"2024-08-17T16:58:13.012624+0000","last_undegraded":"2024-08-17T16:58:13.012624+0000","last_fullsized":"2024-08-17T16:58:13.012624+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:45.015421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":28,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010427+0000","last_change":"2024-08-17T16:56:17.493126+0000","last_active":"2024-08-17T16:58:13.010427+0000","last_peered":"2024-08-17T16:58:13.010427+0000","last_clean":"2024-08-17T16:58:13.010427+0000","last_became_active":"2024-08-17T16:56:12.468605+0000","last_became_peered":"2024-08-17T16:56:12.468605+0000","last_unstale":"2024-08-17T16:58:13.010427+0000","last_undegraded":"2024-08-17T16:58:13.010427+0000","last_fullsized":"2024-08-17T16:58:13.010427+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:55.016828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008509+0000","last_change":"2024-08-17T16:57:28.389765+0000","last_active":"2024-08-17T16:58:13.008509+0000","last_peered":"2024-08-17T16:58:13.008509+0000","last_clean":"2024-08-17T16:58:13.008509+0000","last_became_active":"2024-08-17T16:57:28.388956+0000","last_became_peered":"2024-08-17T16:57:28.388956+0000","last_unstale":"2024-08-17T16:58:13.008509+0000","last_undegraded":"2024-08-17T16:58:13.008509+0000","last_fullsized":"2024-08-17T16:58:13.008509+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:15:05.329554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012729+0000","last_change":"2024-08-17T16:57:30.007794+0000","last_active":"2024-08-17T16:58:13.012729+0000","last_peered":"2024-08-17T16:58:13.012729+0000","last_clean":"2024-08-17T16:58:13.012729+0000","last_became_active":"2024-08-17T16:57:30.007373+0000","last_became_peered":"2024-08-17T16:57:30.007373+0000","last_unstale":"2024-08-17T16:58:13.012729+0000","last_undegraded":"2024-08-17T16:58:13.012729+0000","last_fullsized":"2024-08-17T16:58:13.012729+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:15:52.481603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010457+0000","last_change":"2024-08-17T16:57:28.386646+0000","last_active":"2024-08-17T16:58:13.010457+0000","last_peered":"2024-08-17T16:58:13.010457+0000","last_clean":"2024-08-17T16:58:13.010457+0000","last_became_active":"2024-08-17T16:57:28.386354+0000","last_became_peered":"2024-08-17T16:57:28.386354+0000","last_unstale":"2024-08-17T16:58:13.010457+0000","last_undegraded":"2024-08-17T16:58:13.010457+0000","last_fullsized":"2024-08-17T16:58:13.010457+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:01.797414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012754+0000","last_change":"2024-08-17T16:56:17.485649+0000","last_active":"2024-08-17T16:58:13.012754+0000","last_peered":"2024-08-17T16:58:13.012754+0000","last_clean":"2024-08-17T16:58:13.012754+0000","last_became_active":"2024-08-17T16:56:12.470313+0000","last_became_peered":"2024-08-17T16:56:12.470313+0000","last_unstale":"2024-08-17T16:58:13.012754+0000","last_undegraded":"2024-08-17T16:58:13.012754+0000","last_fullsized":"2024-08-17T16:58:13.012754+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:41.233588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012134+0000","last_change":"2024-08-17T16:57:29.405267+0000","last_active":"2024-08-17T16:58:13.012134+0000","last_peered":"2024-08-17T16:58:13.012134+0000","last_clean":"2024-08-17T16:58:13.012134+0000","last_became_active":"2024-08-17T16:57:29.404623+0000","last_became_peered":"2024-08-17T16:57:29.404623+0000","last_unstale":"2024-08-17T16:58:13.012134+0000","last_undegraded":"2024-08-17T16:58:13.012134+0000","last_fullsized":"2024-08-17T16:58:13.012134+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:26:16.926666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012717+0000","last_change":"2024-08-17T16:57:28.390807+0000","last_active":"2024-08-17T16:58:13.012717+0000","last_peered":"2024-08-17T16:58:13.012717+0000","last_clean":"2024-08-17T16:58:13.012717+0000","last_became_active":"2024-08-17T16:57:28.390518+0000","last_became_peered":"2024-08-17T16:57:28.390518+0000","last_unstale":"2024-08-17T16:58:13.012717+0000","last_undegraded":"2024-08-17T16:58:13.012717+0000","last_fullsized":"2024-08-17T16:58:13.012717+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:30.871316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013849+0000","last_change":"2024-08-17T16:56:17.492672+0000","last_active":"2024-08-17T16:58:13.013849+0000","last_peered":"2024-08-17T16:58:13.013849+0000","last_clean":"2024-08-17T16:58:13.013849+0000","last_became_active":"2024-08-17T16:56:12.468762+0000","last_became_peered":"2024-08-17T16:56:12.468762+0000","last_unstale":"2024-08-17T16:58:13.013849+0000","last_undegraded":"2024-08-17T16:58:13.013849+0000","last_fullsized":"2024-08-17T16:58:13.013849+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:15.340939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009312+0000","last_change":"2024-08-17T16:57:28.771223+0000","last_active":"2024-08-17T16:58:13.009312+0000","last_peered":"2024-08-17T16:58:13.009312+0000","last_clean":"2024-08-17T16:58:13.009312+0000","last_became_active":"2024-08-17T16:57:28.770448+0000","last_became_peered":"2024-08-17T16:57:28.770448+0000","last_unstale":"2024-08-17T16:58:13.009312+0000","last_undegraded":"2024-08-17T16:58:13.009312+0000","last_fullsized":"2024-08-17T16:58:13.009312+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:55:02.545226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011532+0000","last_change":"2024-08-17T16:57:28.394186+0000","last_active":"2024-08-17T16:58:13.011532+0000","last_peered":"2024-08-17T16:58:13.011532+0000","last_clean":"2024-08-17T16:58:13.011532+0000","last_became_active":"2024-08-17T16:57:28.393859+0000","last_became_peered":"2024-08-17T16:57:28.393859+0000","last_unstale":"2024-08-17T16:58:13.011532+0000","last_undegraded":"2024-08-17T16:58:13.011532+0000","last_fullsized":"2024-08-17T16:58:13.011532+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:05:28.285878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012634+0000","last_change":"2024-08-17T16:56:17.485823+0000","last_active":"2024-08-17T16:58:13.012634+0000","last_peered":"2024-08-17T16:58:13.012634+0000","last_clean":"2024-08-17T16:58:13.012634+0000","last_became_active":"2024-08-17T16:56:12.468663+0000","last_became_peered":"2024-08-17T16:56:12.468663+0000","last_unstale":"2024-08-17T16:58:13.012634+0000","last_undegraded":"2024-08-17T16:58:13.012634+0000","last_fullsized":"2024-08-17T16:58:13.012634+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:25.628917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037574,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010799+0000","last_change":"2024-08-17T16:57:30.217819+0000","last_active":"2024-08-17T16:58:13.010799+0000","last_peered":"2024-08-17T16:58:13.010799+0000","last_clean":"2024-08-17T16:58:13.010799+0000","last_became_active":"2024-08-17T16:57:30.217532+0000","last_became_peered":"2024-08-17T16:57:30.217532+0000","last_unstale":"2024-08-17T16:58:13.010799+0000","last_undegraded":"2024-08-17T16:58:13.010799+0000","last_fullsized":"2024-08-17T16:58:13.010799+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:34:52.766321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009746+0000","last_change":"2024-08-17T16:57:28.392762+0000","last_active":"2024-08-17T16:58:13.009746+0000","last_peered":"2024-08-17T16:58:13.009746+0000","last_clean":"2024-08-17T16:58:13.009746+0000","last_became_active":"2024-08-17T16:57:28.392464+0000","last_became_peered":"2024-08-17T16:57:28.392464+0000","last_unstale":"2024-08-17T16:58:13.009746+0000","last_undegraded":"2024-08-17T16:58:13.009746+0000","last_fullsized":"2024-08-17T16:58:13.009746+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:50.811869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.268118+0000","last_change":"2024-08-17T16:56:17.492610+0000","last_active":"2024-08-17T16:58:13.268118+0000","last_peered":"2024-08-17T16:58:13.268118+0000","last_clean":"2024-08-17T16:58:13.268118+0000","last_became_active":"2024-08-17T16:56:12.470076+0000","last_became_peered":"2024-08-17T16:56:12.470076+0000","last_unstale":"2024-08-17T16:58:13.268118+0000","last_undegraded":"2024-08-17T16:58:13.268118+0000","last_fullsized":"2024-08-17T16:58:13.268118+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:24.652500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008105+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T16:58:13.008105+0000","last_peered":"2024-08-17T16:58:13.008105+0000","last_clean":"2024-08-17T16:58:13.008105+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T16:58:13.008105+0000","last_undegraded":"2024-08-17T16:58:13.008105+0000","last_fullsized":"2024-08-17T16:58:13.008105+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008325+0000","last_change":"2024-08-17T16:57:29.406069+0000","last_active":"2024-08-17T16:58:13.008325+0000","last_peered":"2024-08-17T16:58:13.008325+0000","last_clean":"2024-08-17T16:58:13.008325+0000","last_became_active":"2024-08-17T16:57:29.405563+0000","last_became_peered":"2024-08-17T16:57:29.405563+0000","last_unstale":"2024-08-17T16:58:13.008325+0000","last_undegraded":"2024-08-17T16:58:13.008325+0000","last_fullsized":"2024-08-17T16:58:13.008325+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:19.544585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008425+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T16:58:13.008425+0000","last_peered":"2024-08-17T16:58:13.008425+0000","last_clean":"2024-08-17T16:58:13.008425+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T16:58:13.008425+0000","last_undegraded":"2024-08-17T16:58:13.008425+0000","last_fullsized":"2024-08-17T16:58:13.008425+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012230+0000","last_change":"2024-08-17T16:57:30.007854+0000","last_active":"2024-08-17T16:58:13.012230+0000","last_peered":"2024-08-17T16:58:13.012230+0000","last_clean":"2024-08-17T16:58:13.012230+0000","last_became_active":"2024-08-17T16:57:30.007585+0000","last_became_peered":"2024-08-17T16:57:30.007585+0000","last_unstale":"2024-08-17T16:58:13.012230+0000","last_undegraded":"2024-08-17T16:58:13.012230+0000","last_fullsized":"2024-08-17T16:58:13.012230+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:05.848015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013673+0000","last_change":"2024-08-17T16:57:28.388049+0000","last_active":"2024-08-17T16:58:13.013673+0000","last_peered":"2024-08-17T16:58:13.013673+0000","last_clean":"2024-08-17T16:58:13.013673+0000","last_became_active":"2024-08-17T16:57:28.387714+0000","last_became_peered":"2024-08-17T16:57:28.387714+0000","last_unstale":"2024-08-17T16:58:13.013673+0000","last_undegraded":"2024-08-17T16:58:13.013673+0000","last_fullsized":"2024-08-17T16:58:13.013673+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:00:45.274024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010423+0000","last_change":"2024-08-17T16:57:28.392245+0000","last_active":"2024-08-17T16:58:13.010423+0000","last_peered":"2024-08-17T16:58:13.010423+0000","last_clean":"2024-08-17T16:58:13.010423+0000","last_became_active":"2024-08-17T16:57:28.391802+0000","last_became_peered":"2024-08-17T16:57:28.391802+0000","last_unstale":"2024-08-17T16:58:13.010423+0000","last_undegraded":"2024-08-17T16:58:13.010423+0000","last_fullsized":"2024-08-17T16:58:13.010423+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:26.229482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010455+0000","last_change":"2024-08-17T16:57:30.217947+0000","last_active":"2024-08-17T16:58:13.010455+0000","last_peered":"2024-08-17T16:58:13.010455+0000","last_clean":"2024-08-17T16:58:13.010455+0000","last_became_active":"2024-08-17T16:57:30.217663+0000","last_became_peered":"2024-08-17T16:57:30.217663+0000","last_unstale":"2024-08-17T16:58:13.010455+0000","last_undegraded":"2024-08-17T16:58:13.010455+0000","last_fullsized":"2024-08-17T16:58:13.010455+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:13:04.408554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010102+0000","last_change":"2024-08-17T16:57:28.390093+0000","last_active":"2024-08-17T16:58:13.010102+0000","last_peered":"2024-08-17T16:58:13.010102+0000","last_clean":"2024-08-17T16:58:13.010102+0000","last_became_active":"2024-08-17T16:57:28.389406+0000","last_became_peered":"2024-08-17T16:57:28.389406+0000","last_unstale":"2024-08-17T16:58:13.010102+0000","last_undegraded":"2024-08-17T16:58:13.010102+0000","last_fullsized":"2024-08-17T16:58:13.010102+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:37:25.033928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010653+0000","last_change":"2024-08-17T16:57:29.407134+0000","last_active":"2024-08-17T16:58:13.010653+0000","last_peered":"2024-08-17T16:58:13.010653+0000","last_clean":"2024-08-17T16:58:13.010653+0000","last_became_active":"2024-08-17T16:57:29.406793+0000","last_became_peered":"2024-08-17T16:57:29.406793+0000","last_unstale":"2024-08-17T16:58:13.010653+0000","last_undegraded":"2024-08-17T16:58:13.010653+0000","last_fullsized":"2024-08-17T16:58:13.010653+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:35.415132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.268090+0000","last_change":"2024-08-17T16:57:29.404899+0000","last_active":"2024-08-17T16:58:13.268090+0000","last_peered":"2024-08-17T16:58:13.268090+0000","last_clean":"2024-08-17T16:58:13.268090+0000","last_became_active":"2024-08-17T16:57:29.404598+0000","last_became_peered":"2024-08-17T16:57:29.404598+0000","last_unstale":"2024-08-17T16:58:13.268090+0000","last_undegraded":"2024-08-17T16:58:13.268090+0000","last_fullsized":"2024-08-17T16:58:13.268090+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:35.693871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010874+0000","last_change":"2024-08-17T16:57:28.393050+0000","last_active":"2024-08-17T16:58:13.010874+0000","last_peered":"2024-08-17T16:58:13.010874+0000","last_clean":"2024-08-17T16:58:13.010874+0000","last_became_active":"2024-08-17T16:57:28.392255+0000","last_became_peered":"2024-08-17T16:57:28.392255+0000","last_unstale":"2024-08-17T16:58:13.010874+0000","last_undegraded":"2024-08-17T16:58:13.010874+0000","last_fullsized":"2024-08-17T16:58:13.010874+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:12.512872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009521+0000","last_change":"2024-08-17T16:57:28.771483+0000","last_active":"2024-08-17T16:58:13.009521+0000","last_peered":"2024-08-17T16:58:13.009521+0000","last_clean":"2024-08-17T16:58:13.009521+0000","last_became_active":"2024-08-17T16:57:28.770542+0000","last_became_peered":"2024-08-17T16:57:28.770542+0000","last_unstale":"2024-08-17T16:58:13.009521+0000","last_undegraded":"2024-08-17T16:58:13.009521+0000","last_fullsized":"2024-08-17T16:58:13.009521+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:19.111134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011817+0000","last_change":"2024-08-17T16:57:28.396203+0000","last_active":"2024-08-17T16:58:13.011817+0000","last_peered":"2024-08-17T16:58:13.011817+0000","last_clean":"2024-08-17T16:58:13.011817+0000","last_became_active":"2024-08-17T16:57:28.395859+0000","last_became_peered":"2024-08-17T16:57:28.395859+0000","last_unstale":"2024-08-17T16:58:13.011817+0000","last_undegraded":"2024-08-17T16:58:13.011817+0000","last_fullsized":"2024-08-17T16:58:13.011817+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:40.723145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010377+0000","last_change":"2024-08-17T16:57:29.406190+0000","last_active":"2024-08-17T16:58:13.010377+0000","last_peered":"2024-08-17T16:58:13.010377+0000","last_clean":"2024-08-17T16:58:13.010377+0000","last_became_active":"2024-08-17T16:57:29.405901+0000","last_became_peered":"2024-08-17T16:57:29.405901+0000","last_unstale":"2024-08-17T16:58:13.010377+0000","last_undegraded":"2024-08-17T16:58:13.010377+0000","last_fullsized":"2024-08-17T16:58:13.010377+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:18:17.849549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008157+0000","last_change":"2024-08-17T16:57:28.396890+0000","last_active":"2024-08-17T16:58:13.008157+0000","last_peered":"2024-08-17T16:58:13.008157+0000","last_clean":"2024-08-17T16:58:13.008157+0000","last_became_active":"2024-08-17T16:57:28.396586+0000","last_became_peered":"2024-08-17T16:57:28.396586+0000","last_unstale":"2024-08-17T16:58:13.008157+0000","last_undegraded":"2024-08-17T16:58:13.008157+0000","last_fullsized":"2024-08-17T16:58:13.008157+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:12:36.093665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010630+0000","last_change":"2024-08-17T16:57:28.770481+0000","last_active":"2024-08-17T16:58:13.010630+0000","last_peered":"2024-08-17T16:58:13.010630+0000","last_clean":"2024-08-17T16:58:13.010630+0000","last_became_active":"2024-08-17T16:57:28.769896+0000","last_became_peered":"2024-08-17T16:57:28.769896+0000","last_unstale":"2024-08-17T16:58:13.010630+0000","last_undegraded":"2024-08-17T16:58:13.010630+0000","last_fullsized":"2024-08-17T16:58:13.010630+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:55.034956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009340+0000","last_change":"2024-08-17T16:57:28.384785+0000","last_active":"2024-08-17T16:58:13.009340+0000","last_peered":"2024-08-17T16:58:13.009340+0000","last_clean":"2024-08-17T16:58:13.009340+0000","last_became_active":"2024-08-17T16:57:28.384517+0000","last_became_peered":"2024-08-17T16:57:28.384517+0000","last_unstale":"2024-08-17T16:58:13.009340+0000","last_undegraded":"2024-08-17T16:58:13.009340+0000","last_fullsized":"2024-08-17T16:58:13.009340+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:22.745434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:58:14.015303+0000","last_change":"2024-08-17T16:58:14.015303+0000","last_active":"2024-08-17T16:58:14.015303+0000","last_peered":"2024-08-17T16:58:14.015303+0000","last_clean":"2024-08-17T16:58:14.015303+0000","last_became_active":"2024-08-17T16:58:14.014997+0000","last_became_peered":"2024-08-17T16:58:14.014997+0000","last_unstale":"2024-08-17T16:58:14.015303+0000","last_undegraded":"2024-08-17T16:58:14.015303+0000","last_fullsized":"2024-08-17T16:58:14.015303+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:06.009612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010393+0000","last_change":"2024-08-17T16:57:28.385986+0000","last_active":"2024-08-17T16:58:13.010393+0000","last_peered":"2024-08-17T16:58:13.010393+0000","last_clean":"2024-08-17T16:58:13.010393+0000","last_became_active":"2024-08-17T16:57:28.385674+0000","last_became_peered":"2024-08-17T16:57:28.385674+0000","last_unstale":"2024-08-17T16:58:13.010393+0000","last_undegraded":"2024-08-17T16:58:13.010393+0000","last_fullsized":"2024-08-17T16:58:13.010393+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:23.831195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.007910+0000","last_change":"2024-08-17T16:57:29.406195+0000","last_active":"2024-08-17T16:58:13.007910+0000","last_peered":"2024-08-17T16:58:13.007910+0000","last_clean":"2024-08-17T16:58:13.007910+0000","last_became_active":"2024-08-17T16:57:29.405895+0000","last_became_peered":"2024-08-17T16:57:29.405895+0000","last_unstale":"2024-08-17T16:58:13.007910+0000","last_undegraded":"2024-08-17T16:58:13.007910+0000","last_fullsized":"2024-08-17T16:58:13.007910+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:15:09.912787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010480+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T16:58:13.010480+0000","last_peered":"2024-08-17T16:58:13.010480+0000","last_clean":"2024-08-17T16:58:13.010480+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T16:58:13.010480+0000","last_undegraded":"2024-08-17T16:58:13.010480+0000","last_fullsized":"2024-08-17T16:58:13.010480+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011016+0000","last_change":"2024-08-17T16:57:30.007691+0000","last_active":"2024-08-17T16:58:13.011016+0000","last_peered":"2024-08-17T16:58:13.011016+0000","last_clean":"2024-08-17T16:58:13.011016+0000","last_became_active":"2024-08-17T16:57:30.007407+0000","last_became_peered":"2024-08-17T16:57:30.007407+0000","last_unstale":"2024-08-17T16:58:13.011016+0000","last_undegraded":"2024-08-17T16:58:13.011016+0000","last_fullsized":"2024-08-17T16:58:13.011016+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:20.536064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011569+0000","last_change":"2024-08-17T16:57:28.389294+0000","last_active":"2024-08-17T16:58:13.011569+0000","last_peered":"2024-08-17T16:58:13.011569+0000","last_clean":"2024-08-17T16:58:13.011569+0000","last_became_active":"2024-08-17T16:57:28.389009+0000","last_became_peered":"2024-08-17T16:57:28.389009+0000","last_unstale":"2024-08-17T16:58:13.011569+0000","last_undegraded":"2024-08-17T16:58:13.011569+0000","last_fullsized":"2024-08-17T16:58:13.011569+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:28:50.290891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.007813+0000","last_change":"2024-08-17T16:57:29.408953+0000","last_active":"2024-08-17T16:58:13.007813+0000","last_peered":"2024-08-17T16:58:13.007813+0000","last_clean":"2024-08-17T16:58:13.007813+0000","last_became_active":"2024-08-17T16:57:29.408650+0000","last_became_peered":"2024-08-17T16:57:29.408650+0000","last_unstale":"2024-08-17T16:58:13.007813+0000","last_undegraded":"2024-08-17T16:58:13.007813+0000","last_fullsized":"2024-08-17T16:58:13.007813+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:41.577346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010023+0000","last_change":"2024-08-17T16:57:28.390261+0000","last_active":"2024-08-17T16:58:13.010023+0000","last_peered":"2024-08-17T16:58:13.010023+0000","last_clean":"2024-08-17T16:58:13.010023+0000","last_became_active":"2024-08-17T16:57:28.389807+0000","last_became_peered":"2024-08-17T16:57:28.389807+0000","last_unstale":"2024-08-17T16:58:13.010023+0000","last_undegraded":"2024-08-17T16:58:13.010023+0000","last_fullsized":"2024-08-17T16:58:13.010023+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:45.568192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298700,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446452,"statfs":{"total":95995035648,"available":95689166848,"internally_reserved":0,"allocated":1372160,"data_stored":921688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.49399999999999999,"15min":0.48899999999999999},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.36599999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70099999999999996,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.629}]},{"osd":1,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48099999999999998,"15min":0.47599999999999998},"min":{"1min":0.36299999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.65600000000000003}]},{"osd":2,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.47899999999999998},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.73899999999999999,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65500000000000003,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.378,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71399999999999997,"5min":0.747,"15min":0.747},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.66700000000000004,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.435,"15min":0.433},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.621,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.623,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47799999999999998,"15min":0.48299999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.68000000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.32500000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63300000000000001,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71199999999999997,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.51800000000000002,"15min":0.52400000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.66000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58199999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298688,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446464,"statfs":{"total":95995035648,"available":95689179136,"internally_reserved":0,"allocated":1363968,"data_stored":921561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.315,"15min":0.315},"max":{"1min":0.68600000000000005,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.48999999999999999,"15min":0.49199999999999999},"min":{"1min":0.38,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.61699999999999999,"5min":0.751,"15min":0.751},"last":0.498}]},{"osd":1,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.5,"15min":0.496},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.64900000000000002,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.503,"15min":0.51000000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64500000000000002,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.67800000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52200000000000002,"15min":0.52800000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.66800000000000004,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.50800000000000001,"15min":0.51200000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.629,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74299999999999999,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.74299999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.42999999999999999,"15min":0.42199999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.435,"15min":0.436},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.42699999999999999,"15min":0.41499999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:57:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.44400000000000001,"15min":0.441},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.67000000000000004,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.495,"15min":0.496},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.65300000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.42699999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":757760,"data_stored":329888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.53500000000000003,"15min":0.54700000000000004},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.72999999999999998,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.53600000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.76300000000000001,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.628}]},{"osd":1,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52500000000000002,"15min":0.53500000000000003},"min":{"1min":0.28699999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.76000000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.52500000000000002,"15min":0.54000000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52900000000000003,"15min":0.54100000000000004},"min":{"1min":0.29299999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.70499999999999996,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53500000000000003,"15min":0.54200000000000004},"min":{"1min":0.35099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73099999999999998,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.498,"15min":0.501},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.752,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78300000000000003,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.39100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53400000000000003,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.314,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.57699999999999996,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Aug 17 16:58:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.45400000000000001,"15min":0.44400000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.66200000000000003,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.49099999999999999,"15min":0.47999999999999998},"min":{"1min":0.34999999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.71899999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Aug 17 16:57:46 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.49099999999999999,"15min":0.48999999999999999},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.65900000000000003,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.55500000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298112,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447040,"statfs":{"total":95995035648,"available":95689768960,"internally_reserved":0,"allocated":770048,"data_stored":330985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.51500000000000001,"15min":0.51400000000000001},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69899999999999995,"5min":1.151,"15min":1.151},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.49299999999999999,"15min":0.48699999999999999},"min":{"1min":0.33200000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48699999999999999,"15min":0.48099999999999998},"min":{"1min":0.35899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64000000000000001,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.5,"15min":0.496},"min":{"1min":0.38100000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.37,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68899999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51300000000000001,"15min":0.50700000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.51800000000000002,"15min":0.50600000000000001},"min":{"1min":0.42099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72099999999999997,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.504},{"interface":"front","average":{"1min":0.497,"5min":0.48899999999999999,"15min":0.48599999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67400000000000004,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.35199999999999998}]},{"osd":5,"last update":"Sat Aug 17 16:58:00 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.35299999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53600000000000003,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.27600000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.55100000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.29799999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:58:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.40899999999999997,"15min":0.40500000000000003},"min":{"1min":0.34599999999999997,"5min":0.223,"15min":0.223},"max":{"1min":0.57099999999999995,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60099999999999998,"5min":0.745,"15min":0.745},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:57:44 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.46500000000000002,"15min":0.46000000000000002},"min":{"1min":0.34799999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.501,"15min":0.501},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.44500000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298092,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447060,"statfs":{"total":95995035648,"available":95689789440,"internally_reserved":0,"allocated":749568,"data_stored":329377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.374,"15min":0.38600000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.53800000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.38900000000000001,"15min":0.40899999999999997},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.45700000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.434,"15min":0.45200000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.53200000000000003,"5min":0.628,"15min":0.628},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.41399999999999998,"15min":0.42799999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.375}]},{"osd":2,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.42799999999999999,"15min":0.45000000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.498,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.40799999999999997,"15min":0.42199999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.47899999999999998,"5min":0.622,"15min":0.622},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70999999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.52600000000000002,"15min":0.51600000000000001},"min":{"1min":0.436,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Aug 17 16:58:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.39800000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.53300000000000003,"15min":0.53700000000000003},"min":{"1min":0.374,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71199999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:58:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.76000000000000001,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Aug 17 16:57:42 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.53800000000000003,"15min":0.52700000000000002},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53800000000000003,"15min":0.53100000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.66600000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298100,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447052,"statfs":{"total":95995035648,"available":95689781248,"internally_reserved":0,"allocated":757760,"data_stored":330156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:52 2024","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.35299999999999998,"15min":0.34300000000000003},"min":{"1min":0.218,"5min":0.186,"15min":0.186},"max":{"1min":0.51400000000000001,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.52000000000000002,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat Aug 17 16:57:52 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.57399999999999995,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.27700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54300000000000004,"5min":0.751,"15min":0.751},"last":0.36499999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.41099999999999998,"15min":0.41699999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.50600000000000001,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.375},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.40100000000000002,"15min":0.40899999999999997},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.52800000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:57:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55100000000000005,"15min":0.54600000000000004},"min":{"1min":0.40500000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80900000000000005,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58999999999999997,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82899999999999996,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.68500000000000005}]},{"osd":5,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.40300000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80700000000000005,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.628},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81899999999999995,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:58:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.44600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84999999999999998,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79600000000000004,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.67100000000000004}]},{"osd":7,"last update":"Sat Aug 17 16:57:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.39300000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.74299999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79100000000000004,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.53700000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":753664,"data_stored":329571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:32 2024","interfaces":[{"interface":"back","average":{"1min":0.313,"5min":0.36699999999999999,"15min":0.35999999999999999},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.52900000000000003,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.214},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.36099999999999999,"15min":0.34300000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.60099999999999998,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:57:50 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.22600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.51400000000000001,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.57699999999999996,"5min":0.626,"15min":0.626},"last":0.30599999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:58:17 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40300000000000002,"15min":0.41499999999999998},"min":{"1min":0.26500000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51400000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.42099999999999999,"15min":0.439},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55900000000000005,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.27800000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:57:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54800000000000004,"15min":0.55000000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71099999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.41599999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.745,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.497}]},{"osd":5,"last update":"Sat Aug 17 16:58:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54100000000000004,"15min":0.53700000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78700000000000003,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.439},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Aug 17 16:58:23 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.58299999999999996,"15min":0.59499999999999997},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.748,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:57:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38300000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77900000000000003,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.48899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298680,"kb_used_data":1320,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446472,"statfs":{"total":95995035648,"available":95689187328,"internally_reserved":0,"allocated":1351680,"data_stored":920512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:57:29 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.34699999999999998,"15min":0.33300000000000002},"min":{"1min":0.218,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.49099999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.44,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:57:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.29799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.41799999999999998,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.31900000000000001,"5min":0.374,"15min":0.38900000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.44500000000000001,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.307}]},{"osd":3,"last update":"Sat Aug 17 16:58:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.42999999999999999,"15min":0.45000000000000001},"min":{"1min":0.28000000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56100000000000005,"5min":0.629,"15min":0.629},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40100000000000002,"15min":0.41899999999999998},"min":{"1min":0.23599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.53000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.375}]},{"osd":4,"last update":"Sat Aug 17 16:57:36 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52300000000000002,"15min":0.52000000000000002},"min":{"1min":0.34000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70999999999999996,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.434},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53400000000000003,"15min":0.53800000000000003},"min":{"1min":0.40600000000000003,"5min":0.379,"15min":0.379},"max":{"1min":0.66700000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54400000000000004,"15min":0.55500000000000005},"min":{"1min":0.41399999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55700000000000005,"15min":0.56699999999999995},"min":{"1min":0.40500000000000003,"5min":0.31,"15min":0.31},"max":{"1min":0.71499999999999997,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:58:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.54600000000000004,"15min":0.56000000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72499999999999998,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.57899999999999996,"15min":0.59699999999999998},"min":{"1min":0.42299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77500000000000002,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:57:48 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69799999999999995,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.53500000000000003,"15min":0.54100000000000004},"min":{"1min":0.40500000000000003,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.60699999999999998,"5min":0.755,"15min":0.755},"last":0.40999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:58:29.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:58:30.190 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:30.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[40020]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:30.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:30.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:30.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.314 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[31182]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:30.314 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.314 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.314 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:30.314 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:30.314 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:29 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:29 smithi152 ceph-mon[34775]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:29 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:30 smithi152 ceph-mon[34775]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:58:31.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:30 smithi142 ceph-mon[40020]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:58:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:30 smithi142 ceph-mon[31182]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:58:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:31 smithi142 ceph-mon[40020]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:32.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:31 smithi142 ceph-mon[31182]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:32.137 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:58:32.138 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T16:58:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:31 smithi152 ceph-mon[34775]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:32.702 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":84,"stamp":"2024-08-17T16:58:30.495168+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386564,"kb_used_data":7692,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574652,"statfs":{"total":767960285184,"available":765516443648,"internally_reserved":0,"allocated":7876608,"data_stored":4413738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002774"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:58:14.057872+0000","last_change":"2024-08-17T16:58:14.057872+0000","last_active":"2024-08-17T16:58:14.057872+0000","last_peered":"2024-08-17T16:58:14.057872+0000","last_clean":"2024-08-17T16:58:14.057872+0000","last_became_active":"2024-08-17T16:58:14.057545+0000","last_became_peered":"2024-08-17T16:58:14.057545+0000","last_unstale":"2024-08-17T16:58:14.057872+0000","last_undegraded":"2024-08-17T16:58:14.057872+0000","last_fullsized":"2024-08-17T16:58:14.057872+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:53:02.696571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013521+0000","last_change":"2024-08-17T16:57:28.388724+0000","last_active":"2024-08-17T16:58:13.013521+0000","last_peered":"2024-08-17T16:58:13.013521+0000","last_clean":"2024-08-17T16:58:13.013521+0000","last_became_active":"2024-08-17T16:57:28.388374+0000","last_became_peered":"2024-08-17T16:57:28.388374+0000","last_unstale":"2024-08-17T16:58:13.013521+0000","last_undegraded":"2024-08-17T16:58:13.013521+0000","last_fullsized":"2024-08-17T16:58:13.013521+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:00:03.425629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010335+0000","last_change":"2024-08-17T16:57:28.770576+0000","last_active":"2024-08-17T16:58:13.010335+0000","last_peered":"2024-08-17T16:58:13.010335+0000","last_clean":"2024-08-17T16:58:13.010335+0000","last_became_active":"2024-08-17T16:57:28.770338+0000","last_became_peered":"2024-08-17T16:57:28.770338+0000","last_unstale":"2024-08-17T16:58:13.010335+0000","last_undegraded":"2024-08-17T16:58:13.010335+0000","last_fullsized":"2024-08-17T16:58:13.010335+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:34:50.039329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008958+0000","last_change":"2024-08-17T16:57:28.389122+0000","last_active":"2024-08-17T16:58:13.008958+0000","last_peered":"2024-08-17T16:58:13.008958+0000","last_clean":"2024-08-17T16:58:13.008958+0000","last_became_active":"2024-08-17T16:57:28.388809+0000","last_became_peered":"2024-08-17T16:57:28.388809+0000","last_unstale":"2024-08-17T16:58:13.008958+0000","last_undegraded":"2024-08-17T16:58:13.008958+0000","last_fullsized":"2024-08-17T16:58:13.008958+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:04.416967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009660+0000","last_change":"2024-08-17T16:57:28.771324+0000","last_active":"2024-08-17T16:58:13.009660+0000","last_peered":"2024-08-17T16:58:13.009660+0000","last_clean":"2024-08-17T16:58:13.009660+0000","last_became_active":"2024-08-17T16:57:28.771053+0000","last_became_peered":"2024-08-17T16:57:28.771053+0000","last_unstale":"2024-08-17T16:58:13.009660+0000","last_undegraded":"2024-08-17T16:58:13.009660+0000","last_fullsized":"2024-08-17T16:58:13.009660+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:33.216252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013226+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T16:58:13.013226+0000","last_peered":"2024-08-17T16:58:13.013226+0000","last_clean":"2024-08-17T16:58:13.013226+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T16:58:13.013226+0000","last_undegraded":"2024-08-17T16:58:13.013226+0000","last_fullsized":"2024-08-17T16:58:13.013226+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011449+0000","last_change":"2024-08-17T16:57:30.217818+0000","last_active":"2024-08-17T16:58:13.011449+0000","last_peered":"2024-08-17T16:58:13.011449+0000","last_clean":"2024-08-17T16:58:13.011449+0000","last_became_active":"2024-08-17T16:57:30.217533+0000","last_became_peered":"2024-08-17T16:57:30.217533+0000","last_unstale":"2024-08-17T16:58:13.011449+0000","last_undegraded":"2024-08-17T16:58:13.011449+0000","last_fullsized":"2024-08-17T16:58:13.011449+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:26:29.439464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.267994+0000","last_change":"2024-08-17T16:57:28.384794+0000","last_active":"2024-08-17T16:58:13.267994+0000","last_peered":"2024-08-17T16:58:13.267994+0000","last_clean":"2024-08-17T16:58:13.267994+0000","last_became_active":"2024-08-17T16:57:28.384535+0000","last_became_peered":"2024-08-17T16:57:28.384535+0000","last_unstale":"2024-08-17T16:58:13.267994+0000","last_undegraded":"2024-08-17T16:58:13.267994+0000","last_fullsized":"2024-08-17T16:58:13.267994+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:00.999336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013831+0000","last_change":"2024-08-17T16:57:28.389191+0000","last_active":"2024-08-17T16:58:13.013831+0000","last_peered":"2024-08-17T16:58:13.013831+0000","last_clean":"2024-08-17T16:58:13.013831+0000","last_became_active":"2024-08-17T16:57:28.388931+0000","last_became_peered":"2024-08-17T16:57:28.388931+0000","last_unstale":"2024-08-17T16:58:13.013831+0000","last_undegraded":"2024-08-17T16:58:13.013831+0000","last_fullsized":"2024-08-17T16:58:13.013831+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:05:53.117510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010182+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T16:58:13.010182+0000","last_peered":"2024-08-17T16:58:13.010182+0000","last_clean":"2024-08-17T16:58:13.010182+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T16:58:13.010182+0000","last_undegraded":"2024-08-17T16:58:13.010182+0000","last_fullsized":"2024-08-17T16:58:13.010182+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013886+0000","last_change":"2024-08-17T16:57:28.391656+0000","last_active":"2024-08-17T16:58:13.013886+0000","last_peered":"2024-08-17T16:58:13.013886+0000","last_clean":"2024-08-17T16:58:13.013886+0000","last_became_active":"2024-08-17T16:57:28.391401+0000","last_became_peered":"2024-08-17T16:57:28.391401+0000","last_unstale":"2024-08-17T16:58:13.013886+0000","last_undegraded":"2024-08-17T16:58:13.013886+0000","last_fullsized":"2024-08-17T16:58:13.013886+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:07.880114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.007891+0000","last_change":"2024-08-17T16:57:29.408130+0000","last_active":"2024-08-17T16:58:13.007891+0000","last_peered":"2024-08-17T16:58:13.007891+0000","last_clean":"2024-08-17T16:58:13.007891+0000","last_became_active":"2024-08-17T16:57:29.407695+0000","last_became_peered":"2024-08-17T16:57:29.407695+0000","last_unstale":"2024-08-17T16:58:13.007891+0000","last_undegraded":"2024-08-17T16:58:13.007891+0000","last_fullsized":"2024-08-17T16:58:13.007891+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:27.859334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013914+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T16:58:13.013914+0000","last_peered":"2024-08-17T16:58:13.013914+0000","last_clean":"2024-08-17T16:58:13.013914+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T16:58:13.013914+0000","last_undegraded":"2024-08-17T16:58:13.013914+0000","last_fullsized":"2024-08-17T16:58:13.013914+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.267746+0000","last_change":"2024-08-17T16:57:30.217703+0000","last_active":"2024-08-17T16:58:13.267746+0000","last_peered":"2024-08-17T16:58:13.267746+0000","last_clean":"2024-08-17T16:58:13.267746+0000","last_became_active":"2024-08-17T16:57:30.217312+0000","last_became_peered":"2024-08-17T16:57:30.217312+0000","last_unstale":"2024-08-17T16:58:13.267746+0000","last_undegraded":"2024-08-17T16:58:13.267746+0000","last_fullsized":"2024-08-17T16:58:13.267746+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:10.443728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010441+0000","last_change":"2024-08-17T16:57:28.386096+0000","last_active":"2024-08-17T16:58:13.010441+0000","last_peered":"2024-08-17T16:58:13.010441+0000","last_clean":"2024-08-17T16:58:13.010441+0000","last_became_active":"2024-08-17T16:57:28.385810+0000","last_became_peered":"2024-08-17T16:57:28.385810+0000","last_unstale":"2024-08-17T16:58:13.010441+0000","last_undegraded":"2024-08-17T16:58:13.010441+0000","last_fullsized":"2024-08-17T16:58:13.010441+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:43.215952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009244+0000","last_change":"2024-08-17T16:57:28.772065+0000","last_active":"2024-08-17T16:58:13.009244+0000","last_peered":"2024-08-17T16:58:13.009244+0000","last_clean":"2024-08-17T16:58:13.009244+0000","last_became_active":"2024-08-17T16:57:28.771736+0000","last_became_peered":"2024-08-17T16:57:28.771736+0000","last_unstale":"2024-08-17T16:58:13.009244+0000","last_undegraded":"2024-08-17T16:58:13.009244+0000","last_fullsized":"2024-08-17T16:58:13.009244+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:02.662730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010958+0000","last_change":"2024-08-17T16:57:28.389633+0000","last_active":"2024-08-17T16:58:13.010958+0000","last_peered":"2024-08-17T16:58:13.010958+0000","last_clean":"2024-08-17T16:58:13.010958+0000","last_became_active":"2024-08-17T16:57:28.389355+0000","last_became_peered":"2024-08-17T16:57:28.389355+0000","last_unstale":"2024-08-17T16:58:13.010958+0000","last_undegraded":"2024-08-17T16:58:13.010958+0000","last_fullsized":"2024-08-17T16:58:13.010958+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:09.796711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008097+0000","last_change":"2024-08-17T16:57:29.404439+0000","last_active":"2024-08-17T16:58:13.008097+0000","last_peered":"2024-08-17T16:58:13.008097+0000","last_clean":"2024-08-17T16:58:13.008097+0000","last_became_active":"2024-08-17T16:57:29.404033+0000","last_became_peered":"2024-08-17T16:57:29.404033+0000","last_unstale":"2024-08-17T16:58:13.008097+0000","last_undegraded":"2024-08-17T16:58:13.008097+0000","last_fullsized":"2024-08-17T16:58:13.008097+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:52.322809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.267874+0000","last_change":"2024-08-17T16:57:28.386622+0000","last_active":"2024-08-17T16:58:13.267874+0000","last_peered":"2024-08-17T16:58:13.267874+0000","last_clean":"2024-08-17T16:58:13.267874+0000","last_became_active":"2024-08-17T16:57:28.386335+0000","last_became_peered":"2024-08-17T16:57:28.386335+0000","last_unstale":"2024-08-17T16:58:13.267874+0000","last_undegraded":"2024-08-17T16:58:13.267874+0000","last_fullsized":"2024-08-17T16:58:13.267874+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:09.308835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008365+0000","last_change":"2024-08-17T16:57:30.007765+0000","last_active":"2024-08-17T16:58:13.008365+0000","last_peered":"2024-08-17T16:58:13.008365+0000","last_clean":"2024-08-17T16:58:13.008365+0000","last_became_active":"2024-08-17T16:57:30.007479+0000","last_became_peered":"2024-08-17T16:57:30.007479+0000","last_unstale":"2024-08-17T16:58:13.008365+0000","last_undegraded":"2024-08-17T16:58:13.008365+0000","last_fullsized":"2024-08-17T16:58:13.008365+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:33:53.873135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008207+0000","last_change":"2024-08-17T16:57:28.391237+0000","last_active":"2024-08-17T16:58:13.008207+0000","last_peered":"2024-08-17T16:58:13.008207+0000","last_clean":"2024-08-17T16:58:13.008207+0000","last_became_active":"2024-08-17T16:57:28.390939+0000","last_became_peered":"2024-08-17T16:57:28.390939+0000","last_unstale":"2024-08-17T16:58:13.008207+0000","last_undegraded":"2024-08-17T16:58:13.008207+0000","last_fullsized":"2024-08-17T16:58:13.008207+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:21.419059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008541+0000","last_change":"2024-08-17T16:57:29.407933+0000","last_active":"2024-08-17T16:58:13.008541+0000","last_peered":"2024-08-17T16:58:13.008541+0000","last_clean":"2024-08-17T16:58:13.008541+0000","last_became_active":"2024-08-17T16:57:29.407620+0000","last_became_peered":"2024-08-17T16:57:29.407620+0000","last_unstale":"2024-08-17T16:58:13.008541+0000","last_undegraded":"2024-08-17T16:58:13.008541+0000","last_fullsized":"2024-08-17T16:58:13.008541+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:27.092779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010957+0000","last_change":"2024-08-17T16:57:28.392865+0000","last_active":"2024-08-17T16:58:13.010957+0000","last_peered":"2024-08-17T16:58:13.010957+0000","last_clean":"2024-08-17T16:58:13.010957+0000","last_became_active":"2024-08-17T16:57:28.391905+0000","last_became_peered":"2024-08-17T16:57:28.391905+0000","last_unstale":"2024-08-17T16:58:13.010957+0000","last_undegraded":"2024-08-17T16:58:13.010957+0000","last_fullsized":"2024-08-17T16:58:13.010957+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:19:51.664933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:58:14.015634+0000","last_change":"2024-08-17T16:58:14.015634+0000","last_active":"2024-08-17T16:58:14.015634+0000","last_peered":"2024-08-17T16:58:14.015634+0000","last_clean":"2024-08-17T16:58:14.015634+0000","last_became_active":"2024-08-17T16:58:14.015258+0000","last_became_peered":"2024-08-17T16:58:14.015258+0000","last_unstale":"2024-08-17T16:58:14.015634+0000","last_undegraded":"2024-08-17T16:58:14.015634+0000","last_fullsized":"2024-08-17T16:58:14.015634+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:48.766414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"62'128","reported_seq":279,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013093+0000","last_change":"2024-08-17T16:55:45.748632+0000","last_active":"2024-08-17T16:58:13.013093+0000","last_peered":"2024-08-17T16:58:13.013093+0000","last_clean":"2024-08-17T16:58:13.013093+0000","last_became_active":"2024-08-17T16:55:45.748242+0000","last_became_peered":"2024-08-17T16:55:45.748242+0000","last_unstale":"2024-08-17T16:58:13.013093+0000","last_undegraded":"2024-08-17T16:58:13.013093+0000","last_fullsized":"2024-08-17T16:58:13.013093+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:33:22.553778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010372+0000","last_change":"2024-08-17T16:56:17.492706+0000","last_active":"2024-08-17T16:58:13.010372+0000","last_peered":"2024-08-17T16:58:13.010372+0000","last_clean":"2024-08-17T16:58:13.010372+0000","last_became_active":"2024-08-17T16:56:12.470130+0000","last_became_peered":"2024-08-17T16:56:12.470130+0000","last_unstale":"2024-08-17T16:58:13.010372+0000","last_undegraded":"2024-08-17T16:58:13.010372+0000","last_fullsized":"2024-08-17T16:58:13.010372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:00.149068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010353+0000","last_change":"2024-08-17T16:57:28.386845+0000","last_active":"2024-08-17T16:58:13.010353+0000","last_peered":"2024-08-17T16:58:13.010353+0000","last_clean":"2024-08-17T16:58:13.010353+0000","last_became_active":"2024-08-17T16:57:28.386539+0000","last_became_peered":"2024-08-17T16:57:28.386539+0000","last_unstale":"2024-08-17T16:58:13.010353+0000","last_undegraded":"2024-08-17T16:58:13.010353+0000","last_fullsized":"2024-08-17T16:58:13.010353+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.239896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.268064+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T16:58:13.268064+0000","last_peered":"2024-08-17T16:58:13.268064+0000","last_clean":"2024-08-17T16:58:13.268064+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T16:58:13.268064+0000","last_undegraded":"2024-08-17T16:58:13.268064+0000","last_fullsized":"2024-08-17T16:58:13.268064+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013077+0000","last_change":"2024-08-17T16:56:17.492695+0000","last_active":"2024-08-17T16:58:13.013077+0000","last_peered":"2024-08-17T16:58:13.013077+0000","last_clean":"2024-08-17T16:58:13.013077+0000","last_became_active":"2024-08-17T16:56:12.469470+0000","last_became_peered":"2024-08-17T16:56:12.469470+0000","last_unstale":"2024-08-17T16:58:13.013077+0000","last_undegraded":"2024-08-17T16:58:13.013077+0000","last_fullsized":"2024-08-17T16:58:13.013077+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:02.237538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008999+0000","last_change":"2024-08-17T16:57:28.389255+0000","last_active":"2024-08-17T16:58:13.008999+0000","last_peered":"2024-08-17T16:58:13.008999+0000","last_clean":"2024-08-17T16:58:13.008999+0000","last_became_active":"2024-08-17T16:57:28.388979+0000","last_became_peered":"2024-08-17T16:57:28.388979+0000","last_unstale":"2024-08-17T16:58:13.008999+0000","last_undegraded":"2024-08-17T16:58:13.008999+0000","last_fullsized":"2024-08-17T16:58:13.008999+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:20.131361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009591+0000","last_change":"2024-08-17T16:57:28.404457+0000","last_active":"2024-08-17T16:58:13.009591+0000","last_peered":"2024-08-17T16:58:13.009591+0000","last_clean":"2024-08-17T16:58:13.009591+0000","last_became_active":"2024-08-17T16:57:28.404142+0000","last_became_peered":"2024-08-17T16:57:28.404142+0000","last_unstale":"2024-08-17T16:58:13.009591+0000","last_undegraded":"2024-08-17T16:58:13.009591+0000","last_fullsized":"2024-08-17T16:58:13.009591+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:15.012100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010610+0000","last_change":"2024-08-17T16:56:17.485644+0000","last_active":"2024-08-17T16:58:13.010610+0000","last_peered":"2024-08-17T16:58:13.010610+0000","last_clean":"2024-08-17T16:58:13.010610+0000","last_became_active":"2024-08-17T16:56:12.897452+0000","last_became_peered":"2024-08-17T16:56:12.897452+0000","last_unstale":"2024-08-17T16:58:13.010610+0000","last_undegraded":"2024-08-17T16:58:13.010610+0000","last_fullsized":"2024-08-17T16:58:13.010610+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:44:18.635731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012007+0000","last_change":"2024-08-17T16:57:28.395682+0000","last_active":"2024-08-17T16:58:13.012007+0000","last_peered":"2024-08-17T16:58:13.012007+0000","last_clean":"2024-08-17T16:58:13.012007+0000","last_became_active":"2024-08-17T16:57:28.395416+0000","last_became_peered":"2024-08-17T16:57:28.395416+0000","last_unstale":"2024-08-17T16:58:13.012007+0000","last_undegraded":"2024-08-17T16:58:13.012007+0000","last_fullsized":"2024-08-17T16:58:13.012007+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:36.466479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012624+0000","last_change":"2024-08-17T16:57:29.400497+0000","last_active":"2024-08-17T16:58:13.012624+0000","last_peered":"2024-08-17T16:58:13.012624+0000","last_clean":"2024-08-17T16:58:13.012624+0000","last_became_active":"2024-08-17T16:57:29.399887+0000","last_became_peered":"2024-08-17T16:57:29.399887+0000","last_unstale":"2024-08-17T16:58:13.012624+0000","last_undegraded":"2024-08-17T16:58:13.012624+0000","last_fullsized":"2024-08-17T16:58:13.012624+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:45.015421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":28,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010427+0000","last_change":"2024-08-17T16:56:17.493126+0000","last_active":"2024-08-17T16:58:13.010427+0000","last_peered":"2024-08-17T16:58:13.010427+0000","last_clean":"2024-08-17T16:58:13.010427+0000","last_became_active":"2024-08-17T16:56:12.468605+0000","last_became_peered":"2024-08-17T16:56:12.468605+0000","last_unstale":"2024-08-17T16:58:13.010427+0000","last_undegraded":"2024-08-17T16:58:13.010427+0000","last_fullsized":"2024-08-17T16:58:13.010427+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:55.016828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008509+0000","last_change":"2024-08-17T16:57:28.389765+0000","last_active":"2024-08-17T16:58:13.008509+0000","last_peered":"2024-08-17T16:58:13.008509+0000","last_clean":"2024-08-17T16:58:13.008509+0000","last_became_active":"2024-08-17T16:57:28.388956+0000","last_became_peered":"2024-08-17T16:57:28.388956+0000","last_unstale":"2024-08-17T16:58:13.008509+0000","last_undegraded":"2024-08-17T16:58:13.008509+0000","last_fullsized":"2024-08-17T16:58:13.008509+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:15:05.329554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012729+0000","last_change":"2024-08-17T16:57:30.007794+0000","last_active":"2024-08-17T16:58:13.012729+0000","last_peered":"2024-08-17T16:58:13.012729+0000","last_clean":"2024-08-17T16:58:13.012729+0000","last_became_active":"2024-08-17T16:57:30.007373+0000","last_became_peered":"2024-08-17T16:57:30.007373+0000","last_unstale":"2024-08-17T16:58:13.012729+0000","last_undegraded":"2024-08-17T16:58:13.012729+0000","last_fullsized":"2024-08-17T16:58:13.012729+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:15:52.481603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010457+0000","last_change":"2024-08-17T16:57:28.386646+0000","last_active":"2024-08-17T16:58:13.010457+0000","last_peered":"2024-08-17T16:58:13.010457+0000","last_clean":"2024-08-17T16:58:13.010457+0000","last_became_active":"2024-08-17T16:57:28.386354+0000","last_became_peered":"2024-08-17T16:57:28.386354+0000","last_unstale":"2024-08-17T16:58:13.010457+0000","last_undegraded":"2024-08-17T16:58:13.010457+0000","last_fullsized":"2024-08-17T16:58:13.010457+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:01.797414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012754+0000","last_change":"2024-08-17T16:56:17.485649+0000","last_active":"2024-08-17T16:58:13.012754+0000","last_peered":"2024-08-17T16:58:13.012754+0000","last_clean":"2024-08-17T16:58:13.012754+0000","last_became_active":"2024-08-17T16:56:12.470313+0000","last_became_peered":"2024-08-17T16:56:12.470313+0000","last_unstale":"2024-08-17T16:58:13.012754+0000","last_undegraded":"2024-08-17T16:58:13.012754+0000","last_fullsized":"2024-08-17T16:58:13.012754+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:41.233588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012134+0000","last_change":"2024-08-17T16:57:29.405267+0000","last_active":"2024-08-17T16:58:13.012134+0000","last_peered":"2024-08-17T16:58:13.012134+0000","last_clean":"2024-08-17T16:58:13.012134+0000","last_became_active":"2024-08-17T16:57:29.404623+0000","last_became_peered":"2024-08-17T16:57:29.404623+0000","last_unstale":"2024-08-17T16:58:13.012134+0000","last_undegraded":"2024-08-17T16:58:13.012134+0000","last_fullsized":"2024-08-17T16:58:13.012134+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:26:16.926666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012717+0000","last_change":"2024-08-17T16:57:28.390807+0000","last_active":"2024-08-17T16:58:13.012717+0000","last_peered":"2024-08-17T16:58:13.012717+0000","last_clean":"2024-08-17T16:58:13.012717+0000","last_became_active":"2024-08-17T16:57:28.390518+0000","last_became_peered":"2024-08-17T16:57:28.390518+0000","last_unstale":"2024-08-17T16:58:13.012717+0000","last_undegraded":"2024-08-17T16:58:13.012717+0000","last_fullsized":"2024-08-17T16:58:13.012717+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:30.871316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013849+0000","last_change":"2024-08-17T16:56:17.492672+0000","last_active":"2024-08-17T16:58:13.013849+0000","last_peered":"2024-08-17T16:58:13.013849+0000","last_clean":"2024-08-17T16:58:13.013849+0000","last_became_active":"2024-08-17T16:56:12.468762+0000","last_became_peered":"2024-08-17T16:56:12.468762+0000","last_unstale":"2024-08-17T16:58:13.013849+0000","last_undegraded":"2024-08-17T16:58:13.013849+0000","last_fullsized":"2024-08-17T16:58:13.013849+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:15.340939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009312+0000","last_change":"2024-08-17T16:57:28.771223+0000","last_active":"2024-08-17T16:58:13.009312+0000","last_peered":"2024-08-17T16:58:13.009312+0000","last_clean":"2024-08-17T16:58:13.009312+0000","last_became_active":"2024-08-17T16:57:28.770448+0000","last_became_peered":"2024-08-17T16:57:28.770448+0000","last_unstale":"2024-08-17T16:58:13.009312+0000","last_undegraded":"2024-08-17T16:58:13.009312+0000","last_fullsized":"2024-08-17T16:58:13.009312+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:55:02.545226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011532+0000","last_change":"2024-08-17T16:57:28.394186+0000","last_active":"2024-08-17T16:58:13.011532+0000","last_peered":"2024-08-17T16:58:13.011532+0000","last_clean":"2024-08-17T16:58:13.011532+0000","last_became_active":"2024-08-17T16:57:28.393859+0000","last_became_peered":"2024-08-17T16:57:28.393859+0000","last_unstale":"2024-08-17T16:58:13.011532+0000","last_undegraded":"2024-08-17T16:58:13.011532+0000","last_fullsized":"2024-08-17T16:58:13.011532+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:05:28.285878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012634+0000","last_change":"2024-08-17T16:56:17.485823+0000","last_active":"2024-08-17T16:58:13.012634+0000","last_peered":"2024-08-17T16:58:13.012634+0000","last_clean":"2024-08-17T16:58:13.012634+0000","last_became_active":"2024-08-17T16:56:12.468663+0000","last_became_peered":"2024-08-17T16:56:12.468663+0000","last_unstale":"2024-08-17T16:58:13.012634+0000","last_undegraded":"2024-08-17T16:58:13.012634+0000","last_fullsized":"2024-08-17T16:58:13.012634+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:25.628917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037574,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010799+0000","last_change":"2024-08-17T16:57:30.217819+0000","last_active":"2024-08-17T16:58:13.010799+0000","last_peered":"2024-08-17T16:58:13.010799+0000","last_clean":"2024-08-17T16:58:13.010799+0000","last_became_active":"2024-08-17T16:57:30.217532+0000","last_became_peered":"2024-08-17T16:57:30.217532+0000","last_unstale":"2024-08-17T16:58:13.010799+0000","last_undegraded":"2024-08-17T16:58:13.010799+0000","last_fullsized":"2024-08-17T16:58:13.010799+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:34:52.766321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009746+0000","last_change":"2024-08-17T16:57:28.392762+0000","last_active":"2024-08-17T16:58:13.009746+0000","last_peered":"2024-08-17T16:58:13.009746+0000","last_clean":"2024-08-17T16:58:13.009746+0000","last_became_active":"2024-08-17T16:57:28.392464+0000","last_became_peered":"2024-08-17T16:57:28.392464+0000","last_unstale":"2024-08-17T16:58:13.009746+0000","last_undegraded":"2024-08-17T16:58:13.009746+0000","last_fullsized":"2024-08-17T16:58:13.009746+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:50.811869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.268118+0000","last_change":"2024-08-17T16:56:17.492610+0000","last_active":"2024-08-17T16:58:13.268118+0000","last_peered":"2024-08-17T16:58:13.268118+0000","last_clean":"2024-08-17T16:58:13.268118+0000","last_became_active":"2024-08-17T16:56:12.470076+0000","last_became_peered":"2024-08-17T16:56:12.470076+0000","last_unstale":"2024-08-17T16:58:13.268118+0000","last_undegraded":"2024-08-17T16:58:13.268118+0000","last_fullsized":"2024-08-17T16:58:13.268118+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:24.652500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008105+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T16:58:13.008105+0000","last_peered":"2024-08-17T16:58:13.008105+0000","last_clean":"2024-08-17T16:58:13.008105+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T16:58:13.008105+0000","last_undegraded":"2024-08-17T16:58:13.008105+0000","last_fullsized":"2024-08-17T16:58:13.008105+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008325+0000","last_change":"2024-08-17T16:57:29.406069+0000","last_active":"2024-08-17T16:58:13.008325+0000","last_peered":"2024-08-17T16:58:13.008325+0000","last_clean":"2024-08-17T16:58:13.008325+0000","last_became_active":"2024-08-17T16:57:29.405563+0000","last_became_peered":"2024-08-17T16:57:29.405563+0000","last_unstale":"2024-08-17T16:58:13.008325+0000","last_undegraded":"2024-08-17T16:58:13.008325+0000","last_fullsized":"2024-08-17T16:58:13.008325+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:19.544585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008425+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T16:58:13.008425+0000","last_peered":"2024-08-17T16:58:13.008425+0000","last_clean":"2024-08-17T16:58:13.008425+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T16:58:13.008425+0000","last_undegraded":"2024-08-17T16:58:13.008425+0000","last_fullsized":"2024-08-17T16:58:13.008425+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.012230+0000","last_change":"2024-08-17T16:57:30.007854+0000","last_active":"2024-08-17T16:58:13.012230+0000","last_peered":"2024-08-17T16:58:13.012230+0000","last_clean":"2024-08-17T16:58:13.012230+0000","last_became_active":"2024-08-17T16:57:30.007585+0000","last_became_peered":"2024-08-17T16:57:30.007585+0000","last_unstale":"2024-08-17T16:58:13.012230+0000","last_undegraded":"2024-08-17T16:58:13.012230+0000","last_fullsized":"2024-08-17T16:58:13.012230+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:05.848015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.013673+0000","last_change":"2024-08-17T16:57:28.388049+0000","last_active":"2024-08-17T16:58:13.013673+0000","last_peered":"2024-08-17T16:58:13.013673+0000","last_clean":"2024-08-17T16:58:13.013673+0000","last_became_active":"2024-08-17T16:57:28.387714+0000","last_became_peered":"2024-08-17T16:57:28.387714+0000","last_unstale":"2024-08-17T16:58:13.013673+0000","last_undegraded":"2024-08-17T16:58:13.013673+0000","last_fullsized":"2024-08-17T16:58:13.013673+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:00:45.274024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010423+0000","last_change":"2024-08-17T16:57:28.392245+0000","last_active":"2024-08-17T16:58:13.010423+0000","last_peered":"2024-08-17T16:58:13.010423+0000","last_clean":"2024-08-17T16:58:13.010423+0000","last_became_active":"2024-08-17T16:57:28.391802+0000","last_became_peered":"2024-08-17T16:57:28.391802+0000","last_unstale":"2024-08-17T16:58:13.010423+0000","last_undegraded":"2024-08-17T16:58:13.010423+0000","last_fullsized":"2024-08-17T16:58:13.010423+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:26.229482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010455+0000","last_change":"2024-08-17T16:57:30.217947+0000","last_active":"2024-08-17T16:58:13.010455+0000","last_peered":"2024-08-17T16:58:13.010455+0000","last_clean":"2024-08-17T16:58:13.010455+0000","last_became_active":"2024-08-17T16:57:30.217663+0000","last_became_peered":"2024-08-17T16:57:30.217663+0000","last_unstale":"2024-08-17T16:58:13.010455+0000","last_undegraded":"2024-08-17T16:58:13.010455+0000","last_fullsized":"2024-08-17T16:58:13.010455+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:13:04.408554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010102+0000","last_change":"2024-08-17T16:57:28.390093+0000","last_active":"2024-08-17T16:58:13.010102+0000","last_peered":"2024-08-17T16:58:13.010102+0000","last_clean":"2024-08-17T16:58:13.010102+0000","last_became_active":"2024-08-17T16:57:28.389406+0000","last_became_peered":"2024-08-17T16:57:28.389406+0000","last_unstale":"2024-08-17T16:58:13.010102+0000","last_undegraded":"2024-08-17T16:58:13.010102+0000","last_fullsized":"2024-08-17T16:58:13.010102+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:37:25.033928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010653+0000","last_change":"2024-08-17T16:57:29.407134+0000","last_active":"2024-08-17T16:58:13.010653+0000","last_peered":"2024-08-17T16:58:13.010653+0000","last_clean":"2024-08-17T16:58:13.010653+0000","last_became_active":"2024-08-17T16:57:29.406793+0000","last_became_peered":"2024-08-17T16:57:29.406793+0000","last_unstale":"2024-08-17T16:58:13.010653+0000","last_undegraded":"2024-08-17T16:58:13.010653+0000","last_fullsized":"2024-08-17T16:58:13.010653+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:35.415132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.268090+0000","last_change":"2024-08-17T16:57:29.404899+0000","last_active":"2024-08-17T16:58:13.268090+0000","last_peered":"2024-08-17T16:58:13.268090+0000","last_clean":"2024-08-17T16:58:13.268090+0000","last_became_active":"2024-08-17T16:57:29.404598+0000","last_became_peered":"2024-08-17T16:57:29.404598+0000","last_unstale":"2024-08-17T16:58:13.268090+0000","last_undegraded":"2024-08-17T16:58:13.268090+0000","last_fullsized":"2024-08-17T16:58:13.268090+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:35.693871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010874+0000","last_change":"2024-08-17T16:57:28.393050+0000","last_active":"2024-08-17T16:58:13.010874+0000","last_peered":"2024-08-17T16:58:13.010874+0000","last_clean":"2024-08-17T16:58:13.010874+0000","last_became_active":"2024-08-17T16:57:28.392255+0000","last_became_peered":"2024-08-17T16:57:28.392255+0000","last_unstale":"2024-08-17T16:58:13.010874+0000","last_undegraded":"2024-08-17T16:58:13.010874+0000","last_fullsized":"2024-08-17T16:58:13.010874+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:12.512872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009521+0000","last_change":"2024-08-17T16:57:28.771483+0000","last_active":"2024-08-17T16:58:13.009521+0000","last_peered":"2024-08-17T16:58:13.009521+0000","last_clean":"2024-08-17T16:58:13.009521+0000","last_became_active":"2024-08-17T16:57:28.770542+0000","last_became_peered":"2024-08-17T16:57:28.770542+0000","last_unstale":"2024-08-17T16:58:13.009521+0000","last_undegraded":"2024-08-17T16:58:13.009521+0000","last_fullsized":"2024-08-17T16:58:13.009521+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:19.111134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011817+0000","last_change":"2024-08-17T16:57:28.396203+0000","last_active":"2024-08-17T16:58:13.011817+0000","last_peered":"2024-08-17T16:58:13.011817+0000","last_clean":"2024-08-17T16:58:13.011817+0000","last_became_active":"2024-08-17T16:57:28.395859+0000","last_became_peered":"2024-08-17T16:57:28.395859+0000","last_unstale":"2024-08-17T16:58:13.011817+0000","last_undegraded":"2024-08-17T16:58:13.011817+0000","last_fullsized":"2024-08-17T16:58:13.011817+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:40.723145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010377+0000","last_change":"2024-08-17T16:57:29.406190+0000","last_active":"2024-08-17T16:58:13.010377+0000","last_peered":"2024-08-17T16:58:13.010377+0000","last_clean":"2024-08-17T16:58:13.010377+0000","last_became_active":"2024-08-17T16:57:29.405901+0000","last_became_peered":"2024-08-17T16:57:29.405901+0000","last_unstale":"2024-08-17T16:58:13.010377+0000","last_undegraded":"2024-08-17T16:58:13.010377+0000","last_fullsized":"2024-08-17T16:58:13.010377+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:18:17.849549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.008157+0000","last_change":"2024-08-17T16:57:28.396890+0000","last_active":"2024-08-17T16:58:13.008157+0000","last_peered":"2024-08-17T16:58:13.008157+0000","last_clean":"2024-08-17T16:58:13.008157+0000","last_became_active":"2024-08-17T16:57:28.396586+0000","last_became_peered":"2024-08-17T16:57:28.396586+0000","last_unstale":"2024-08-17T16:58:13.008157+0000","last_undegraded":"2024-08-17T16:58:13.008157+0000","last_fullsized":"2024-08-17T16:58:13.008157+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:12:36.093665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010630+0000","last_change":"2024-08-17T16:57:28.770481+0000","last_active":"2024-08-17T16:58:13.010630+0000","last_peered":"2024-08-17T16:58:13.010630+0000","last_clean":"2024-08-17T16:58:13.010630+0000","last_became_active":"2024-08-17T16:57:28.769896+0000","last_became_peered":"2024-08-17T16:57:28.769896+0000","last_unstale":"2024-08-17T16:58:13.010630+0000","last_undegraded":"2024-08-17T16:58:13.010630+0000","last_fullsized":"2024-08-17T16:58:13.010630+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:55.034956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.009340+0000","last_change":"2024-08-17T16:57:28.384785+0000","last_active":"2024-08-17T16:58:13.009340+0000","last_peered":"2024-08-17T16:58:13.009340+0000","last_clean":"2024-08-17T16:58:13.009340+0000","last_became_active":"2024-08-17T16:57:28.384517+0000","last_became_peered":"2024-08-17T16:57:28.384517+0000","last_unstale":"2024-08-17T16:58:13.009340+0000","last_undegraded":"2024-08-17T16:58:13.009340+0000","last_fullsized":"2024-08-17T16:58:13.009340+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:22.745434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:58:14.015303+0000","last_change":"2024-08-17T16:58:14.015303+0000","last_active":"2024-08-17T16:58:14.015303+0000","last_peered":"2024-08-17T16:58:14.015303+0000","last_clean":"2024-08-17T16:58:14.015303+0000","last_became_active":"2024-08-17T16:58:14.014997+0000","last_became_peered":"2024-08-17T16:58:14.014997+0000","last_unstale":"2024-08-17T16:58:14.015303+0000","last_undegraded":"2024-08-17T16:58:14.015303+0000","last_fullsized":"2024-08-17T16:58:14.015303+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:06.009612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010393+0000","last_change":"2024-08-17T16:57:28.385986+0000","last_active":"2024-08-17T16:58:13.010393+0000","last_peered":"2024-08-17T16:58:13.010393+0000","last_clean":"2024-08-17T16:58:13.010393+0000","last_became_active":"2024-08-17T16:57:28.385674+0000","last_became_peered":"2024-08-17T16:57:28.385674+0000","last_unstale":"2024-08-17T16:58:13.010393+0000","last_undegraded":"2024-08-17T16:58:13.010393+0000","last_fullsized":"2024-08-17T16:58:13.010393+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:40:23.831195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.007910+0000","last_change":"2024-08-17T16:57:29.406195+0000","last_active":"2024-08-17T16:58:13.007910+0000","last_peered":"2024-08-17T16:58:13.007910+0000","last_clean":"2024-08-17T16:58:13.007910+0000","last_became_active":"2024-08-17T16:57:29.405895+0000","last_became_peered":"2024-08-17T16:57:29.405895+0000","last_unstale":"2024-08-17T16:58:13.007910+0000","last_undegraded":"2024-08-17T16:58:13.007910+0000","last_fullsized":"2024-08-17T16:58:13.007910+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:15:09.912787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010480+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T16:58:13.010480+0000","last_peered":"2024-08-17T16:58:13.010480+0000","last_clean":"2024-08-17T16:58:13.010480+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T16:58:13.010480+0000","last_undegraded":"2024-08-17T16:58:13.010480+0000","last_fullsized":"2024-08-17T16:58:13.010480+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011016+0000","last_change":"2024-08-17T16:57:30.007691+0000","last_active":"2024-08-17T16:58:13.011016+0000","last_peered":"2024-08-17T16:58:13.011016+0000","last_clean":"2024-08-17T16:58:13.011016+0000","last_became_active":"2024-08-17T16:57:30.007407+0000","last_became_peered":"2024-08-17T16:57:30.007407+0000","last_unstale":"2024-08-17T16:58:13.011016+0000","last_undegraded":"2024-08-17T16:58:13.011016+0000","last_fullsized":"2024-08-17T16:58:13.011016+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:20.536064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.011569+0000","last_change":"2024-08-17T16:57:28.389294+0000","last_active":"2024-08-17T16:58:13.011569+0000","last_peered":"2024-08-17T16:58:13.011569+0000","last_clean":"2024-08-17T16:58:13.011569+0000","last_became_active":"2024-08-17T16:57:28.389009+0000","last_became_peered":"2024-08-17T16:57:28.389009+0000","last_unstale":"2024-08-17T16:58:13.011569+0000","last_undegraded":"2024-08-17T16:58:13.011569+0000","last_fullsized":"2024-08-17T16:58:13.011569+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:28:50.290891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.007813+0000","last_change":"2024-08-17T16:57:29.408953+0000","last_active":"2024-08-17T16:58:13.007813+0000","last_peered":"2024-08-17T16:58:13.007813+0000","last_clean":"2024-08-17T16:58:13.007813+0000","last_became_active":"2024-08-17T16:57:29.408650+0000","last_became_peered":"2024-08-17T16:57:29.408650+0000","last_unstale":"2024-08-17T16:58:13.007813+0000","last_undegraded":"2024-08-17T16:58:13.007813+0000","last_fullsized":"2024-08-17T16:58:13.007813+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:41.577346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T16:58:13.010023+0000","last_change":"2024-08-17T16:57:28.390261+0000","last_active":"2024-08-17T16:58:13.010023+0000","last_peered":"2024-08-17T16:58:13.010023+0000","last_clean":"2024-08-17T16:58:13.010023+0000","last_became_active":"2024-08-17T16:57:28.389807+0000","last_became_peered":"2024-08-17T16:57:28.389807+0000","last_unstale":"2024-08-17T16:58:13.010023+0000","last_undegraded":"2024-08-17T16:58:13.010023+0000","last_fullsized":"2024-08-17T16:58:13.010023+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:45.568192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298700,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446452,"statfs":{"total":95995035648,"available":95689166848,"internally_reserved":0,"allocated":1372160,"data_stored":921688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.49399999999999999,"15min":0.48899999999999999},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.36599999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70099999999999996,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.629}]},{"osd":1,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48099999999999998,"15min":0.47599999999999998},"min":{"1min":0.36299999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.65600000000000003}]},{"osd":2,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.47899999999999998},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.73899999999999999,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65500000000000003,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.378,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71399999999999997,"5min":0.747,"15min":0.747},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.66700000000000004,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.435,"15min":0.433},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.621,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.623,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47799999999999998,"15min":0.48299999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.68000000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.32500000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63300000000000001,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:57:47 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71199999999999997,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.51800000000000002,"15min":0.52400000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.66000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58199999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298688,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446464,"statfs":{"total":95995035648,"available":95689179136,"internally_reserved":0,"allocated":1363968,"data_stored":921561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.315,"15min":0.315},"max":{"1min":0.68600000000000005,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.48999999999999999,"15min":0.49199999999999999},"min":{"1min":0.38,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.61699999999999999,"5min":0.751,"15min":0.751},"last":0.498}]},{"osd":1,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.5,"15min":0.496},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.64900000000000002,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.503,"15min":0.51000000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64500000000000002,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.67800000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52200000000000002,"15min":0.52800000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.66800000000000004,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.50800000000000001,"15min":0.51200000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.629,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52800000000000002,"15min":0.52400000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74299999999999999,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.74299999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.42999999999999999,"15min":0.42199999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Aug 17 16:58:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.435,"15min":0.436},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.42699999999999999,"15min":0.41499999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:57:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.44400000000000001,"15min":0.441},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.67000000000000004,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.495,"15min":0.496},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.65300000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.42699999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":757760,"data_stored":329888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.53500000000000003,"15min":0.54700000000000004},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":0.72999999999999998,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.53600000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.76300000000000001,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.628}]},{"osd":1,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52500000000000002,"15min":0.53500000000000003},"min":{"1min":0.28699999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.76000000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.52500000000000002,"15min":0.54000000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52900000000000003,"15min":0.54100000000000004},"min":{"1min":0.29299999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.70499999999999996,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53500000000000003,"15min":0.54200000000000004},"min":{"1min":0.35099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73099999999999998,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.498,"15min":0.501},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.752,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78300000000000003,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.39100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53400000000000003,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.314,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.57699999999999996,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Aug 17 16:58:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.45400000000000001,"15min":0.44400000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.66200000000000003,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.49099999999999999,"15min":0.47999999999999998},"min":{"1min":0.34999999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.71899999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Aug 17 16:57:46 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.372,"5min":0.313,"15min":0.313},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.49099999999999999,"15min":0.48999999999999999},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.65900000000000003,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.55500000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298112,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447040,"statfs":{"total":95995035648,"available":95689768960,"internally_reserved":0,"allocated":770048,"data_stored":330985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.51500000000000001,"15min":0.51400000000000001},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69899999999999995,"5min":1.151,"15min":1.151},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.49299999999999999,"15min":0.48699999999999999},"min":{"1min":0.33200000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48699999999999999,"15min":0.48099999999999998},"min":{"1min":0.35899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64000000000000001,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.5,"15min":0.496},"min":{"1min":0.38100000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.37,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.68899999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51300000000000001,"15min":0.50700000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.51800000000000002,"15min":0.50600000000000001},"min":{"1min":0.42099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72099999999999997,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.504},{"interface":"front","average":{"1min":0.497,"5min":0.48899999999999999,"15min":0.48599999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67400000000000004,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.35199999999999998}]},{"osd":5,"last update":"Sat Aug 17 16:58:00 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.35299999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53600000000000003,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.27600000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.55100000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.29799999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:58:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.40899999999999997,"15min":0.40500000000000003},"min":{"1min":0.34599999999999997,"5min":0.223,"15min":0.223},"max":{"1min":0.57099999999999995,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60099999999999998,"5min":0.745,"15min":0.745},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:57:44 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.46500000000000002,"15min":0.46000000000000002},"min":{"1min":0.34799999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.501,"15min":0.501},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.44500000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298092,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447060,"statfs":{"total":95995035648,"available":95689789440,"internally_reserved":0,"allocated":749568,"data_stored":329377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.374,"15min":0.38600000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.53800000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.38900000000000001,"15min":0.40899999999999997},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.45700000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.434,"15min":0.45200000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.53200000000000003,"5min":0.628,"15min":0.628},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.41399999999999998,"15min":0.42799999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.375}]},{"osd":2,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.42799999999999999,"15min":0.45000000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.498,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.40799999999999997,"15min":0.42199999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.47899999999999998,"5min":0.622,"15min":0.622},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Aug 17 16:57:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52200000000000002,"15min":0.52100000000000002},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70999999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.52600000000000002,"15min":0.51600000000000001},"min":{"1min":0.436,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Aug 17 16:58:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.39800000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.53300000000000003,"15min":0.53700000000000003},"min":{"1min":0.374,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71199999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:58:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.76000000000000001,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Aug 17 16:57:42 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.53800000000000003,"15min":0.52700000000000002},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53800000000000003,"15min":0.53100000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.66600000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298100,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447052,"statfs":{"total":95995035648,"available":95689781248,"internally_reserved":0,"allocated":757760,"data_stored":330156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:52 2024","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.35299999999999998,"15min":0.34300000000000003},"min":{"1min":0.218,"5min":0.186,"15min":0.186},"max":{"1min":0.51400000000000001,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.52000000000000002,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat Aug 17 16:57:52 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.57399999999999995,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.27700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54300000000000004,"5min":0.751,"15min":0.751},"last":0.36499999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:58:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.41099999999999998,"15min":0.41699999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.50600000000000001,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.375},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.40100000000000002,"15min":0.40899999999999997},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.52800000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:57:35 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55100000000000005,"15min":0.54600000000000004},"min":{"1min":0.40500000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80900000000000005,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58999999999999997,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82899999999999996,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.68500000000000005}]},{"osd":5,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.40300000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80700000000000005,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.628},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.56200000000000006,"15min":0.56799999999999995},"min":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81899999999999995,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:58:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.44600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84999999999999998,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79600000000000004,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.67100000000000004}]},{"osd":7,"last update":"Sat Aug 17 16:57:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.39300000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.74299999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79100000000000004,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.53700000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":753664,"data_stored":329571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:57:32 2024","interfaces":[{"interface":"back","average":{"1min":0.313,"5min":0.36699999999999999,"15min":0.35999999999999999},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.52900000000000003,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.214},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.36099999999999999,"15min":0.34300000000000003},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.60099999999999998,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:57:50 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.22600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.51400000000000001,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.57699999999999996,"5min":0.626,"15min":0.626},"last":0.30599999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:58:17 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40300000000000002,"15min":0.41499999999999998},"min":{"1min":0.26500000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51400000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.42099999999999999,"15min":0.439},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55900000000000005,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.27800000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:57:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54800000000000004,"15min":0.55000000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71099999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.41599999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.745,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.497}]},{"osd":5,"last update":"Sat Aug 17 16:58:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54100000000000004,"15min":0.53700000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78700000000000003,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.439},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Aug 17 16:58:23 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.58299999999999996,"15min":0.59499999999999997},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.748,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:57:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38300000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77900000000000003,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.48899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298680,"kb_used_data":1320,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446472,"statfs":{"total":95995035648,"available":95689187328,"internally_reserved":0,"allocated":1351680,"data_stored":920512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:57:29 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.34699999999999998,"15min":0.33300000000000002},"min":{"1min":0.218,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.49099999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.36199999999999999,"15min":0.35599999999999998},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.44,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:57:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.29799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.41799999999999998,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.31900000000000001,"5min":0.374,"15min":0.38900000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.44500000000000001,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.307}]},{"osd":3,"last update":"Sat Aug 17 16:58:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.42999999999999999,"15min":0.45000000000000001},"min":{"1min":0.28000000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56100000000000005,"5min":0.629,"15min":0.629},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40100000000000002,"15min":0.41899999999999998},"min":{"1min":0.23599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.53000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.375}]},{"osd":4,"last update":"Sat Aug 17 16:57:36 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52300000000000002,"15min":0.52000000000000002},"min":{"1min":0.34000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70999999999999996,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.434},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53400000000000003,"15min":0.53800000000000003},"min":{"1min":0.40600000000000003,"5min":0.379,"15min":0.379},"max":{"1min":0.66700000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Aug 17 16:58:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54400000000000004,"15min":0.55500000000000005},"min":{"1min":0.41399999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55700000000000005,"15min":0.56699999999999995},"min":{"1min":0.40500000000000003,"5min":0.31,"15min":0.31},"max":{"1min":0.71499999999999997,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:58:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.54600000000000004,"15min":0.56000000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72499999999999998,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.57899999999999996,"15min":0.59699999999999998},"min":{"1min":0.42299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77500000000000002,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:57:48 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69799999999999995,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.53500000000000003,"15min":0.54100000000000004},"min":{"1min":0.40500000000000003,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.60699999999999998,"5min":0.755,"15min":0.755},"last":0.40999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:58:32.707 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T16:58:32.707 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T16:58:32.707 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T16:58:32.707 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T16:58:32.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:58:32 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:58:32.963 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:34.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:33 smithi142 ceph-mon[40020]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:58:34.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:33 smithi142 ceph-mon[40020]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:34.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:33 smithi142 ceph-mon[31182]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:58:34.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:33 smithi142 ceph-mon[31182]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:34.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:33 smithi152 ceph-mon[34775]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:58:34.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:33 smithi152 ceph-mon[34775]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:34.894 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T16:58:34.894 INFO:teuthology.orchestra.run.smithi142.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T16:58:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:34 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2896225004' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:58:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:34 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2896225004' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:58:35.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:34 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2896225004' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:58:35.652 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T16:58:35.653 INFO:teuthology.run_tasks:Running task print... 2024-08-17T16:58:35.662 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-08-17T16:58:35.662 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T16:58:35.680 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T16:58:35.680 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T16:58:35.681 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-17T16:58:35.681 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T16:58:35.683 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T16:58:35.683 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-08-17T16:58:35.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-08-17T16:58:35.684 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T16:58:35.684 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-08-17T16:58:35.684 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi142.front.sepia.ceph.com 2024-08-17T16:58:35.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-08-17T16:58:35.966 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:35.969 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:36.232 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:35 smithi142 ceph-mon[40020]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:36.232 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:35 smithi142 ceph-mon[31182]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:35 smithi152 ceph-mon[34775]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:38.071 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:37 smithi142 ceph-mon[40020]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:38.072 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:37 smithi142 ceph-mon[31182]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:37 smithi152 ceph-mon[34775]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:38.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:38] "GET /metrics HTTP/1.1" 200 239675 "" "Prometheus/2.43.0" 2024-08-17T16:58:39.474 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:58:39] ENGINE Bus STOPPING 2024-08-17T16:58:39.474 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:58:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:58:39.474 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:58:39] ENGINE Bus STOPPED 2024-08-17T16:58:39.474 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:58:39] ENGINE Bus STARTING 2024-08-17T16:58:39.475 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:58:39] ENGINE Serving on http://:::9283 2024-08-17T16:58:39.475 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:16:58:39] ENGINE Bus STARTED 2024-08-17T16:58:39.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set global log_to_journald false --force' 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: from='client.? ' entity='client.admin' 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2040842189' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:40.189 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2040842189' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:40.190 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:39 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:40.191 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_0' created 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2040842189' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:39 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:40.393 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:40.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-08-17T16:58:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:41 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2040842189' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T16:58:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:41 smithi152 ceph-mon[34775]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T16:58:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:41 smithi152 ceph-mon[34775]: pgmap v90: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:41 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2040842189' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T16:58:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:41 smithi142 ceph-mon[40020]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T16:58:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:41 smithi142 ceph-mon[40020]: pgmap v90: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:41 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2040842189' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T16:58:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:41 smithi142 ceph-mon[31182]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T16:58:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:41 smithi142 ceph-mon[31182]: pgmap v90: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:41.628 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:42 smithi142 ceph-mon[40020]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:58:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:42 smithi142 ceph-mon[40020]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T16:58:42.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:42 smithi142 ceph-mon[31182]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:58:42.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:42 smithi142 ceph-mon[31182]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T16:58:42.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:42 smithi152 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:58:42.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:42 smithi152 ceph-mon[34775]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T16:58:42.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:58:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:58:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[40020]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[40020]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[31182]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[31182]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:43 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:43 smithi152 ceph-mon[34775]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T16:58:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:43 smithi152 ceph-mon[34775]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:43 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:43 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:44.480 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-08-17T16:58:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:44 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1272290684' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:58:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:44 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:58:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:44 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1272290684' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:58:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:44 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:58:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:44 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1272290684' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:58:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:44 smithi142 ceph-mon[31182]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:58:45.023 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-08-17T16:58:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:45 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:58:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:45 smithi142 ceph-mon[40020]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T16:58:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:45 smithi142 ceph-mon[40020]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:45 smithi142 ceph-mon[31182]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:58:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:45 smithi142 ceph-mon[31182]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T16:58:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:45 smithi142 ceph-mon[31182]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:45.728 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:45 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:58:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:45 smithi152 ceph-mon[34775]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T16:58:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:45 smithi152 ceph-mon[34775]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:46.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:46 smithi142 ceph-mon[40020]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:58:46.615 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:46 smithi142 ceph-mon[40020]: Cluster is now healthy 2024-08-17T16:58:46.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:46 smithi142 ceph-mon[31182]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:58:46.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:46 smithi142 ceph-mon[31182]: Cluster is now healthy 2024-08-17T16:58:46.685 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-08-17T16:58:46.687 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T16:58:46.687 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T16:58:46.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T16:58:46.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T16:58:46.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_84876 2024-08-17T16:58:46.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T16:58:46.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T16:58:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:46 smithi152 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:58:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:46 smithi152 ceph-mon[34775]: Cluster is now healthy 2024-08-17T16:58:47.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 4375 4359 272.422 272.438 0.00483782 0.00365919 2024-08-17T16:58:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:47 smithi152 ceph-mon[34775]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:47.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:47 smithi142 ceph-mon[40020]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:47 smithi142 ceph-mon[31182]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T16:58:48.031 INFO:teuthology.orchestra.run.smithi142.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:48.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:48] "GET /metrics HTTP/1.1" 200 242017 "" "Prometheus/2.43.0" 2024-08-17T16:58:48.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 8288 8272 258.482 244.562 0.0040411 0.00386204 2024-08-17T16:58:48.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T16:58:49.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:49 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:49.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:49.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:49.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:49 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:49.414 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:49.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 11827 11811 246.044 221.188 0.00502768 0.00405794 2024-08-17T16:58:50.082 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:50 smithi142 ceph-mon[31182]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:50.082 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:50 smithi142 ceph-mon[31182]: pgmap v97: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-08-17T16:58:50.082 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:50 smithi142 ceph-mon[40020]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:50.082 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:50 smithi142 ceph-mon[40020]: pgmap v97: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-08-17T16:58:50.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:50 smithi152 ceph-mon[34775]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:58:50.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:50 smithi152 ceph-mon[34775]: pgmap v97: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-08-17T16:58:50.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 14969 14953 233.619 196.375 0.00588496 0.00427495 2024-08-17T16:58:51.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 17782 17766 222.054 175.812 0.00196576 0.00449639 2024-08-17T16:58:52.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:52 smithi152 ceph-mon[34775]: pgmap v98: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2024-08-17T16:58:52.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:52 smithi142 ceph-mon[31182]: pgmap v98: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2024-08-17T16:58:52.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:52 smithi142 ceph-mon[40020]: pgmap v98: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2024-08-17T16:58:52.762 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 20452 20436 212.854 166.875 0.00637383 0.00469354 2024-08-17T16:58:52.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:58:53.386 INFO:teuthology.orchestra.run.smithi142.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:alertmanager.a smithi142 *:9093,9094 running (87s) 24s ago 2m 27.8M - 0.25.0 c8568f914cd2 d120aaf39f51 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:grafana.a smithi152 *:3000 running (2m) 43s ago 2m 85.5M - 9.4.7 954c08fa6188 e049e3632985 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi142.ndmoid smithi142 running (72s) 24s ago 72s 13.7M - 17.2.7-1404-gf2c39d4d f69471c56189 bd857f554782 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi152.tacamo smithi152 running (69s) 43s ago 69s 13.9M - 17.2.7-1404-gf2c39d4d f69471c56189 8913efd6a27d 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.x smithi152 *:8443 running (6m) 43s ago 6m 399M - 17.2.7-1404-gf2c39d4d f69471c56189 77132185e2a3 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.y smithi142 *:9283,8765 running (7m) 24s ago 7m 445M - 17.2.7-1404-gf2c39d4d f69471c56189 c1b9a1fd281f 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a smithi142 running (7m) 24s ago 8m 47.4M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 5787ca46b374 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b smithi152 running (6m) 43s ago 6m 42.3M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 b18785ced07f 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c smithi142 running (6m) 24s ago 6m 35.4M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 24ff3a433074 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.a smithi142 *:9100 running (117s) 24s ago 115s 16.1M - 1.5.0 0da6a335fe13 dde7b19121b8 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.b smithi152 *:9100 running (112s) 43s ago 111s 9601k - 1.5.0 0da6a335fe13 83ebea2ae7c0 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 smithi142 running (6m) 24s ago 6m 43.2M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 cb819443ee72 2024-08-17T16:58:53.387 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 smithi142 running (5m) 24s ago 5m 43.3M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 5cd1d20f8eca 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 smithi142 running (5m) 24s ago 5m 41.5M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 b599f0d55518 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 smithi142 running (4m) 24s ago 4m 40.7M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 d53780492c5b 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 smithi152 running (4m) 43s ago 4m 43.7M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 7a1cab3a1049 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 smithi152 running (4m) 43s ago 4m 40.1M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 cac09b3f3a95 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 smithi152 running (3m) 43s ago 3m 39.8M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 81327b9b9e78 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 smithi152 running (3m) 43s ago 3m 42.0M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 0a45f0507de1 2024-08-17T16:58:53.388 INFO:teuthology.orchestra.run.smithi142.stdout:prometheus.a smithi152 *:9095 running (82s) 43s ago 2m 37.8M - 2.43.0 a07b618ecd1d fa63d07ad574 2024-08-17T16:58:53.763 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 23095 23079 206.041 165.188 0.00633813 0.00484812 2024-08-17T16:58:54.243 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for minority of mons to upgrade' 2024-08-17T16:58:54.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:54 smithi152 ceph-mon[34775]: pgmap v99: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2024-08-17T16:58:54.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:54 smithi142 ceph-mon[31182]: pgmap v99: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2024-08-17T16:58:54.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:54 smithi142 ceph-mon[40020]: pgmap v99: 89 pgs: 89 active+clean; 448 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2024-08-17T16:58:54.730 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:58:54.765 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 25611 25595 199.869 157.25 0.00708788 0.00499738 2024-08-17T16:58:55.313 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:55 smithi142 ceph-mon[31182]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:55.313 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:55 smithi142 ceph-mon[40020]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:55.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:55 smithi152 ceph-mon[34775]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:55.766 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 27975 27959 194.075 147.75 0.00634188 0.00514773 2024-08-17T16:58:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:56 smithi152 ceph-mon[34775]: pgmap v100: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:58:56.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:56 smithi142 ceph-mon[40020]: pgmap v100: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:58:56.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:56 smithi142 ceph-mon[31182]: pgmap v100: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:58:56.767 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 30330 30314 189.387 147.188 0.00480451 0.00527414 2024-08-17T16:58:57.766 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 32545 32529 184.755 138.438 0.00268465 0.00540766 2024-08-17T16:58:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-mon[31182]: pgmap v101: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T16:58:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-mon[40020]: pgmap v101: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T16:58:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:58.374 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:58:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:58:58] "GET /metrics HTTP/1.1" 200 242931 "" "Prometheus/2.43.0" 2024-08-17T16:58:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:58 smithi152 ceph-mon[34775]: pgmap v101: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T16:58:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:58:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:58.766 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 34612 34596 180.123 129.188 0.00351593 0.0055469 2024-08-17T16:58:59.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:58:59 smithi142 ceph-mon[31182]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-08-17T16:58:59.418 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:58:59 smithi142 ceph-mon[40020]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-08-17T16:58:59.525 INFO:teuthology.orchestra.run.smithi142.stdout:wait for minority of mons to upgrade 2024-08-17T16:58:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:58:59 smithi152 ceph-mon[34775]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-08-17T16:58:59.767 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 36808 36792 176.825 137.25 0.011938 0.00565065 2024-08-17T16:59:00.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T16:59:00.766 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 38848 38832 173.302 127.5 0.00870953 0.00576443 2024-08-17T16:59:00.837 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:01.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:01 smithi142 ceph-mon[31182]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:59:01.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:01 smithi142 ceph-mon[40020]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:59:01.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 40889 40873 170.239 127.562 0.00545215 0.00586913 2024-08-17T16:59:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:01 smithi152 ceph-mon[34775]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:59:02.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 42895 42879 167.435 125.375 0.00552075 0.0059674 2024-08-17T16:59:02.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:59:03.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 44646 44630 164.024 109.438 0.00548857 0.00609167 2024-08-17T16:59:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:03 smithi142 ceph-mon[40020]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:59:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:03 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2420867782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:03 smithi142 ceph-mon[31182]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:59:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:03 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2420867782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:03.971 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:03 smithi152 ceph-mon[34775]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-08-17T16:59:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:03 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2420867782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:04.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 46467 46451 161.234 113.812 0.00540541 0.00619722 2024-08-17T16:59:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:04 smithi142 ceph-mon[40020]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:04 smithi142 ceph-mon[31182]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:04 smithi152 ceph-mon[34775]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:05.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 48423 48407 159.179 122.25 0.0060427 0.00627644 2024-08-17T16:59:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:05 smithi142 ceph-mon[31182]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:05 smithi142 ceph-mon[31182]: pgmap v105: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 193 MiB/s wr, 3.08k op/s 2024-08-17T16:59:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:05 smithi142 ceph-mon[40020]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:05.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:05 smithi142 ceph-mon[40020]: pgmap v105: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 193 MiB/s wr, 3.08k op/s 2024-08-17T16:59:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:05 smithi152 ceph-mon[34775]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:05 smithi152 ceph-mon[34775]: pgmap v105: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 193 MiB/s wr, 3.08k op/s 2024-08-17T16:59:06.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T16:59:06.768989+0000 min lat: 0.00119736 max lat: 0.0982633 avg lat: 0.00636454 2024-08-17T16:59:06.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T16:59:06.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 50278 50262 157.015 115.938 0.00763444 0.00636454 2024-08-17T16:59:07.639 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:07.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 51867 51851 154.267 99.3125 0.00997599 0.0064768 2024-08-17T16:59:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:07 smithi142 ceph-mon[40020]: pgmap v106: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-17T16:59:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:07 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/521176701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:07 smithi142 ceph-mon[31182]: pgmap v106: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-17T16:59:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:07 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/521176701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:07 smithi152 ceph-mon[34775]: pgmap v106: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-17T16:59:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:07 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/521176701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:08.608 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:59:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:08] "GET /metrics HTTP/1.1" 200 242931 "" "Prometheus/2.43.0" 2024-08-17T16:59:08.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 53487 53471 151.855 101.25 0.00646364 0.00658059 2024-08-17T16:59:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:08 smithi142 ceph-mon[40020]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:08 smithi142 ceph-mon[31182]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:08 smithi152 ceph-mon[34775]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:09.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 55037 55021 149.465 96.875 0.00702417 0.00668539 2024-08-17T16:59:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:09 smithi142 ceph-mon[40020]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:09 smithi142 ceph-mon[40020]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-08-17T16:59:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:09 smithi142 ceph-mon[31182]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:09 smithi142 ceph-mon[31182]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-08-17T16:59:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:09 smithi152 ceph-mon[34775]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:09 smithi152 ceph-mon[34775]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-08-17T16:59:10.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 56388 56372 146.756 84.4375 0.0108808 0.00680916 2024-08-17T16:59:11.261 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:11.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:11 smithi142 ceph-mon[31182]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T16:59:11.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 57892 57876 144.642 94 0.00683776 0.00690895 2024-08-17T16:59:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:11 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2535597298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:11 smithi142 ceph-mon[40020]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T16:59:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:11 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2535597298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:11 smithi152 ceph-mon[34775]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T16:59:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:11 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2535597298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:12.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 59369 59353 142.63 92.3125 0.00764397 0.00700586 2024-08-17T16:59:12.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:59:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:59:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:12 smithi152 ceph-mon[34775]: from='client.24643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:12 smithi152 ceph-mon[34775]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:12 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:12 smithi142 ceph-mon[40020]: from='client.24643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:12 smithi142 ceph-mon[40020]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:12 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:12 smithi142 ceph-mon[31182]: from='client.24643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:12 smithi142 ceph-mon[31182]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:12 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:13.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 60927 60911 140.954 97.375 0.0149595 0.00708966 2024-08-17T16:59:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:13 smithi152 ceph-mon[34775]: pgmap v109: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T16:59:14.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:13 smithi142 ceph-mon[40020]: pgmap v109: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T16:59:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:13 smithi142 ceph-mon[31182]: pgmap v109: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T16:59:14.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 62236 62220 138.842 81.8125 0.0162058 0.00719759 2024-08-17T16:59:14.850 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:14 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/819406644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:14 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/819406644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:15.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:14 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/819406644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:15.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 63676 63660 137.157 90 0.0102731 0.00728582 2024-08-17T16:59:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:15 smithi152 ceph-mon[34775]: pgmap v110: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T16:59:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:15 smithi152 ceph-mon[34775]: from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:15 smithi142 ceph-mon[40020]: pgmap v110: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T16:59:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:15 smithi142 ceph-mon[40020]: from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:15 smithi142 ceph-mon[31182]: pgmap v110: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T16:59:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:15 smithi142 ceph-mon[31182]: from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:16.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 65158 65142 135.673 92.625 0.0146959 0.00736523 2024-08-17T16:59:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:16 smithi152 ceph-mon[34775]: from='client.24661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:16 smithi142 ceph-mon[31182]: from='client.24661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:16 smithi142 ceph-mon[40020]: from='client.24661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:17.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 66456 66440 133.913 81.125 0.00543201 0.00746224 2024-08-17T16:59:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:17 smithi152 ceph-mon[34775]: pgmap v111: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T16:59:18.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:17 smithi142 ceph-mon[31182]: pgmap v111: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T16:59:18.180 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:17 smithi142 ceph-mon[40020]: pgmap v111: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T16:59:18.428 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:18.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:59:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:18] "GET /metrics HTTP/1.1" 200 243142 "" "Prometheus/2.43.0" 2024-08-17T16:59:18.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 67705 67689 132.168 78.0625 0.003889 0.00755679 2024-08-17T16:59:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:18 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/934388416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:18 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/934388416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:19.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:18 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/934388416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:19.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 69066 69050 130.74 85.0625 0.0034948 0.00764363 2024-08-17T16:59:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:19 smithi152 ceph-mon[34775]: from='client.14889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:19 smithi152 ceph-mon[34775]: pgmap v112: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T16:59:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:19 smithi152 ceph-mon[34775]: from='client.24676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:20.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:19 smithi142 ceph-mon[31182]: from='client.14889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:20.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:19 smithi142 ceph-mon[31182]: pgmap v112: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T16:59:20.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:19 smithi142 ceph-mon[31182]: from='client.24676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:20.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:19 smithi142 ceph-mon[40020]: from='client.14889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:20.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:19 smithi142 ceph-mon[40020]: pgmap v112: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T16:59:20.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:19 smithi142 ceph-mon[40020]: from='client.24676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:20.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 70479 70463 129.492 88.3125 0.0118205 0.00771725 2024-08-17T16:59:21.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 71965 71949 128.446 92.875 0.0109876 0.00778108 2024-08-17T16:59:22.000 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:21 smithi142 ceph-mon[31182]: pgmap v113: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:22.001 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:21 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3937067862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:22.001 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:21 smithi142 ceph-mon[40020]: pgmap v113: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:22.001 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:21 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3937067862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:22.001 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:21 smithi152 ceph-mon[34775]: pgmap v113: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:21 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3937067862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:22.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 73408 73392 127.383 90.1875 0.00861355 0.00784583 2024-08-17T16:59:22.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:22 smithi152 ceph-mon[34775]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:22.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:59:22 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:59:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:22 smithi142 ceph-mon[40020]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:22 smithi142 ceph-mon[31182]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:23.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 74609 74593 125.969 75.0625 0.00323951 0.00793195 2024-08-17T16:59:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:23 smithi152 ceph-mon[34775]: from='client.14907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:23 smithi152 ceph-mon[34775]: pgmap v114: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:24.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:23 smithi142 ceph-mon[31182]: from='client.14907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:24.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:23 smithi142 ceph-mon[31182]: pgmap v114: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:23 smithi142 ceph-mon[40020]: from='client.14907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:24.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:23 smithi142 ceph-mon[40020]: pgmap v114: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:24.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 75812 75796 124.633 75.1875 0.0122709 0.00801847 2024-08-17T16:59:25.388 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:25.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 77123 77107 123.538 81.9375 0.0210172 0.00808993 2024-08-17T16:59:26.011 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:25 smithi142 ceph-mon[31182]: pgmap v115: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T16:59:26.011 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:25 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2587579551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:26.011 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:25 smithi142 ceph-mon[40020]: pgmap v115: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T16:59:26.012 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:25 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2587579551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:25 smithi152 ceph-mon[34775]: pgmap v115: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T16:59:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:25 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2587579551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:26.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T16:59:26.771964+0000 min lat: 0.00119736 max lat: 0.167154 avg lat: 0.00814207 2024-08-17T16:59:26.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T16:59:26.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 78594 78578 122.748 91.9375 0.0106223 0.00814207 2024-08-17T16:59:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:26 smithi152 ceph-mon[34775]: from='client.24791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:26 smithi152 ceph-mon[34775]: from='client.24697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:26 smithi142 ceph-mon[40020]: from='client.24791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:26 smithi142 ceph-mon[40020]: from='client.24697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:26 smithi142 ceph-mon[31182]: from='client.24791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:26 smithi142 ceph-mon[31182]: from='client.24697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:27.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 79936 79920 121.8 83.875 0.007182 0.00820501 2024-08-17T16:59:28.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: pgmap v116: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:28.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: pgmap v116: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T16:59:28.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: pgmap v116: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T16:59:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:28.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:59:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:28] "GET /metrics HTTP/1.1" 200 243208 "" "Prometheus/2.43.0" 2024-08-17T16:59:28.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 81211 81195 120.797 79.6875 0.0126684 0.00827308 2024-08-17T16:59:28.864 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:28 smithi142 ceph-mon[31182]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi152 2024-08-17T16:59:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:28 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/545357810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:28 smithi142 ceph-mon[40020]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi152 2024-08-17T16:59:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:28 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/545357810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:28 smithi152 ceph-mon[34775]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi152 2024-08-17T16:59:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:28 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/545357810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:29.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 82433 82417 119.764 76.375 0.00852788 0.00834505 2024-08-17T16:59:30.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:29 smithi152 ceph-mon[34775]: pgmap v117: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T16:59:30.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:29 smithi152 ceph-mon[34775]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:30.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:29 smithi142 ceph-mon[40020]: pgmap v117: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T16:59:30.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:29 smithi142 ceph-mon[40020]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:29 smithi142 ceph-mon[31182]: pgmap v117: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T16:59:30.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:29 smithi142 ceph-mon[31182]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:30.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 83653 83637 118.775 76.25 0.00887096 0.00841404 2024-08-17T16:59:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:30 smithi152 ceph-mon[34775]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:31.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:30 smithi142 ceph-mon[40020]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:31.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:30 smithi142 ceph-mon[31182]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:31.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 84866 84850 117.82 75.8125 0.0254032 0.00848194 2024-08-17T16:59:32.285 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:32.285 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:31 smithi142 ceph-mon[40020]: pgmap v118: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:32.285 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:31 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2787200681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:32.285 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:31 smithi142 ceph-mon[31182]: pgmap v118: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:32.285 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:31 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2787200681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:31 smithi152 ceph-mon[34775]: pgmap v118: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T16:59:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:31 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2787200681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:32.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 86052 86036 116.87 74.125 0.0144798 0.00855165 2024-08-17T16:59:32.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:59:32 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:59:33.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 87246 87230 115.972 74.625 0.0165194 0.00861794 2024-08-17T16:59:34.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:33 smithi152 ceph-mon[34775]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:33 smithi152 ceph-mon[34775]: pgmap v119: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-17T16:59:34.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:33 smithi152 ceph-mon[34775]: from='client.24827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:33 smithi142 ceph-mon[40020]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:33 smithi142 ceph-mon[40020]: pgmap v119: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-17T16:59:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:33 smithi142 ceph-mon[40020]: from='client.24827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:33 smithi142 ceph-mon[31182]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:33 smithi142 ceph-mon[31182]: pgmap v119: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-17T16:59:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:33 smithi142 ceph-mon[31182]: from='client.24827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 88400 88384 115.058 72.125 0.00950112 0.00868657 2024-08-17T16:59:35.718 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:35.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 89562 89546 114.192 72.625 0.0108064 0.00875249 2024-08-17T16:59:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:35 smithi142 ceph-mon[31182]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-17T16:59:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:35 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1349136934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:35 smithi142 ceph-mon[40020]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-17T16:59:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:35 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1349136934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:35 smithi152 ceph-mon[34775]: pgmap v120: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-17T16:59:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:35 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1349136934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:36.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 90739 90723 113.379 73.5625 0.0101061 0.00881466 2024-08-17T16:59:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:36 smithi152 ceph-mon[34775]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:37.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:36 smithi142 ceph-mon[31182]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:37.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:36 smithi142 ceph-mon[40020]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:37.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 91899 91883 112.578 72.5 0.0180947 0.00887736 2024-08-17T16:59:38.309 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:37 smithi142 ceph-mon[40020]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:38.309 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:37 smithi142 ceph-mon[40020]: pgmap v121: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T16:59:38.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:37 smithi142 ceph-mon[31182]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:38.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:37 smithi142 ceph-mon[31182]: pgmap v121: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T16:59:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:37 smithi152 ceph-mon[34775]: from='client.24842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:37 smithi152 ceph-mon[34775]: pgmap v121: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T16:59:38.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:59:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:38] "GET /metrics HTTP/1.1" 200 243208 "" "Prometheus/2.43.0" 2024-08-17T16:59:38.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 92954 92938 111.681 65.9375 0.0142383 0.00894927 2024-08-17T16:59:39.196 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:38 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/951140307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:38 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/951140307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:39.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:38 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/951140307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:39.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 94038 94022 110.852 67.75 0.00853025 0.00901586 2024-08-17T16:59:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:39 smithi152 ceph-mon[34775]: pgmap v122: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T16:59:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:39 smithi142 ceph-mon[40020]: pgmap v122: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T16:59:40.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:39 smithi142 ceph-mon[31182]: pgmap v122: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T16:59:40.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 95146 95130 110.082 69.25 0.01645 0.00907905 2024-08-17T16:59:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:40 smithi152 ceph-mon[34775]: from='client.24757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:40 smithi152 ceph-mon[34775]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:40 smithi142 ceph-mon[31182]: from='client.24757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:40 smithi142 ceph-mon[31182]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:40 smithi142 ceph-mon[40020]: from='client.24757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:40 smithi142 ceph-mon[40020]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:41.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 96248 96232 109.332 68.875 0.0182127 0.00914194 2024-08-17T16:59:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:41 smithi152 ceph-mon[34775]: pgmap v123: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T16:59:42.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:41 smithi142 ceph-mon[31182]: pgmap v123: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T16:59:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:41 smithi142 ceph-mon[40020]: pgmap v123: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T16:59:42.696 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:42.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 97346 97330 108.606 68.625 0.0182089 0.00920234 2024-08-17T16:59:42.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:59:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:59:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:42 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/490780903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:42 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:42 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/490780903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:42 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:42 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/490780903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:42 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:43.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 98513 98497 107.98 72.9375 0.010267 0.00925664 2024-08-17T16:59:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:44 smithi152 ceph-mon[34775]: pgmap v124: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T16:59:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:44 smithi152 ceph-mon[34775]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:44.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:44 smithi142 ceph-mon[40020]: pgmap v124: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T16:59:44.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:44 smithi142 ceph-mon[40020]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:44 smithi142 ceph-mon[31182]: pgmap v124: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-08-17T16:59:44.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:44 smithi142 ceph-mon[31182]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:44.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 99585 99569 107.273 67 0.019252 0.00931708 2024-08-17T16:59:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:45 smithi152 ceph-mon[34775]: from='client.24869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:45.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:45 smithi142 ceph-mon[40020]: from='client.24869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:45 smithi142 ceph-mon[31182]: from='client.24869 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:45.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 100779 100763 106.72 74.625 0.0188473 0.00936501 2024-08-17T16:59:46.170 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:46 smithi152 ceph-mon[34775]: pgmap v125: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T16:59:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:46 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/158905168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:46.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:46 smithi142 ceph-mon[40020]: pgmap v125: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T16:59:46.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:46 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/158905168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:46.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:46 smithi142 ceph-mon[31182]: pgmap v125: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T16:59:46.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:46 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/158905168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:46.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T16:59:46.773697+0000 min lat: 0.00119736 max lat: 0.167154 avg lat: 0.00941912 2024-08-17T16:59:46.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T16:59:46.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 101907 101891 106.116 70.5 0.00845887 0.00941912 2024-08-17T16:59:47.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 102963 102947 105.458 66 0.00925498 0.00947818 2024-08-17T16:59:48.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-mon[40020]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:48.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-mon[40020]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-08-17T16:59:48.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-mon[40020]: from='client.24884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:48.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-mon[31182]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:48.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-mon[31182]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-08-17T16:59:48.308 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-mon[31182]: from='client.24884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:48.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:48 smithi152 ceph-mon[34775]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:48 smithi152 ceph-mon[34775]: pgmap v126: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-08-17T16:59:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:48 smithi152 ceph-mon[34775]: from='client.24884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:48.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:59:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:48] "GET /metrics HTTP/1.1" 200 243275 "" "Prometheus/2.43.0" 2024-08-17T16:59:48.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 103993 103977 104.796 64.375 0.00779528 0.00953728 2024-08-17T16:59:49.589 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:49.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 104996 104980 104.127 62.6875 0.0193427 0.00959894 2024-08-17T16:59:50.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:50 smithi152 ceph-mon[34775]: pgmap v127: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T16:59:50.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:50 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3610088087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:50.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:50 smithi142 ceph-mon[31182]: pgmap v127: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T16:59:50.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:50 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3610088087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:50.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:50 smithi142 ceph-mon[40020]: pgmap v127: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T16:59:50.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:50 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3610088087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:50.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 106038 106022 103.518 65.125 0.00930477 0.00965459 2024-08-17T16:59:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:51 smithi152 ceph-mon[34775]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:51 smithi152 ceph-mon[34775]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:51 smithi142 ceph-mon[40020]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:51 smithi142 ceph-mon[40020]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:51 smithi142 ceph-mon[31182]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:51 smithi142 ceph-mon[31182]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:51.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 107046 107030 102.894 63 0.00950795 0.00971315 2024-08-17T16:59:52.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:52 smithi152 ceph-mon[34775]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T16:59:52.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:52 smithi142 ceph-mon[40020]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T16:59:52.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:52 smithi142 ceph-mon[31182]: pgmap v128: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T16:59:52.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 107997 107981 102.236 59.4375 0.00374298 0.00976991 2024-08-17T16:59:52.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 16:59:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:59:52.941 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:53.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:53 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2267296964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:53.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:53 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2267296964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:53.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:53 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2267296964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:53.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 108833 108817 101.49 52.25 0.0167838 0.00984842 2024-08-17T16:59:54.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:54 smithi152 ceph-mon[34775]: pgmap v129: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-08-17T16:59:54.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:54 smithi152 ceph-mon[34775]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:54.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:54 smithi142 ceph-mon[40020]: pgmap v129: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-08-17T16:59:54.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:54 smithi142 ceph-mon[40020]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:54.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:54 smithi142 ceph-mon[31182]: pgmap v129: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-08-17T16:59:54.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:54 smithi142 ceph-mon[31182]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:54.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 109746 109730 100.837 57.0625 0.0140281 0.00991215 2024-08-17T16:59:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:55 smithi152 ceph-mon[34775]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:55 smithi142 ceph-mon[40020]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:55.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:55 smithi142 ceph-mon[31182]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:55.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 15 110694 110679 100.235 59.3125 0.0148316 0.00997161 2024-08-17T16:59:56.280 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:56 smithi152 ceph-mon[34775]: pgmap v130: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T16:59:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:56 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2611854869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:56.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:56 smithi142 ceph-mon[40020]: pgmap v130: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T16:59:56.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:56 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2611854869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:56 smithi142 ceph-mon[31182]: pgmap v130: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T16:59:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:56 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2611854869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T16:59:56.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 111657 111641 99.6617 60.125 0.0137778 0.0100291 2024-08-17T16:59:57.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 112547 112531 99.0415 55.625 0.0153291 0.0100916 2024-08-17T16:59:58.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:58 smithi152 ceph-mon[34775]: from='client.15063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:58.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:58 smithi152 ceph-mon[34775]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-08-17T16:59:58.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:58 smithi152 ceph-mon[34775]: from='client.15069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:58.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 16:59:58 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:58.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:16:59:58] "GET /metrics HTTP/1.1" 200 243298 "" "Prometheus/2.43.0" 2024-08-17T16:59:58.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[40020]: from='client.15063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[40020]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[40020]: from='client.15069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[31182]: from='client.15063 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[31182]: pgmap v131: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[31182]: from='client.15069 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 16:59:58 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:58.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 113434 113418 98.4359 55.4375 0.00631934 0.0101539 2024-08-17T16:59:59.626 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T16:59:59.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 114349 114333 97.8708 57.1875 0.00852149 0.010212 2024-08-17T17:00:00.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:00 smithi152 ceph-mon[34775]: pgmap v132: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-08-17T17:00:00.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:00 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2210734480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:00.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:00 smithi152 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T17:00:00.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:00 smithi142 ceph-mon[40020]: pgmap v132: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-08-17T17:00:00.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:00 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2210734480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:00.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:00 smithi142 ceph-mon[40020]: overall HEALTH_OK 2024-08-17T17:00:00.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:00 smithi142 ceph-mon[31182]: pgmap v132: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-08-17T17:00:00.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:00 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2210734480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:00.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:00 smithi142 ceph-mon[31182]: overall HEALTH_OK 2024-08-17T17:00:00.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 115232 115216 97.294 55.1875 0.00934701 0.010273 2024-08-17T17:00:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:01 smithi142 ceph-mon[40020]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:01.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:01 smithi142 ceph-mon[40020]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:01.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:01 smithi142 ceph-mon[31182]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:01.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:01 smithi142 ceph-mon[31182]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:01.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:01 smithi152 ceph-mon[34775]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:01.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:01 smithi152 ceph-mon[34775]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:01.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 15 116144 116129 96.7575 57.0625 0.010535 0.01033 2024-08-17T17:00:02.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:02 smithi142 ceph-mon[40020]: pgmap v133: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:00:02.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:02 smithi142 ceph-mon[31182]: pgmap v133: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:00:02.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:02 smithi152 ceph-mon[34775]: pgmap v133: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:00:02.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 117161 117145 96.3199 63.5 0.0076578 0.0103772 2024-08-17T17:00:02.822 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[36843]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:00:03.073 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:03.160 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:03 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3461665540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:03.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:03 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3461665540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:03.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:03 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3461665540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:03.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 118164 118148 95.8831 62.6875 0.0156144 0.0104242 2024-08-17T17:00:04.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:04 smithi142 ceph-mon[40020]: pgmap v134: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2024-08-17T17:00:04.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:04 smithi142 ceph-mon[40020]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:04.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:04 smithi142 ceph-mon[31182]: pgmap v134: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2024-08-17T17:00:04.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:04 smithi142 ceph-mon[31182]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:04.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:04 smithi152 ceph-mon[34775]: pgmap v134: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2024-08-17T17:00:04.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:04 smithi152 ceph-mon[34775]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:04.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 119181 119165 95.4687 63.5625 0.0191472 0.0104697 2024-08-17T17:00:05.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[40020]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:05.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[31182]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:05.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:05 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:05.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:05 smithi152 ceph-mon[34775]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:05 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:05.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:05 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:00:05.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:05 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:00:05.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:05 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:05.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 120212 120196 95.0758 64.4375 0.0260779 0.0105131 2024-08-17T17:00:06.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:06 smithi152 ceph-mon[34775]: Upgrade: Updating mgr.x 2024-08-17T17:00:06.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:06 smithi152 ceph-mon[34775]: Deploying daemon mgr.x on smithi152 2024-08-17T17:00:06.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:06 smithi152 ceph-mon[34775]: pgmap v135: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:00:06.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:06 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/939044283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[40020]: Upgrade: Updating mgr.x 2024-08-17T17:00:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[40020]: Deploying daemon mgr.x on smithi152 2024-08-17T17:00:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[40020]: pgmap v135: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:00:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/939044283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[31182]: Upgrade: Updating mgr.x 2024-08-17T17:00:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[31182]: Deploying daemon mgr.x on smithi152 2024-08-17T17:00:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[31182]: pgmap v135: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:00:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:06 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/939044283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:06.492 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:06.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:06 smithi152 systemd[1]: Stopping Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:00:06.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:00:06.775475+0000 min lat: 0.00119736 max lat: 0.167154 avg lat: 0.0105634 2024-08-17T17:00:06.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:00:06.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 121156 121140 94.6248 59 0.0154822 0.0105634 2024-08-17T17:00:07.051 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:06 smithi152 podman[70813]: 2024-08-17 17:00:06.698053218 +0000 UTC m=+0.192653443 container died 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T17:00:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:07 smithi152 ceph-mon[34775]: from='client.24974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:07 smithi152 ceph-mon[34775]: pgmap v136: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2024-08-17T17:00:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:07 smithi152 ceph-mon[34775]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 podman[70813]: 2024-08-17 17:00:07.283425066 +0000 UTC m=+0.778025280 container cleanup 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T17:00:07.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 bash[70813]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:00:07.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:07 smithi142 ceph-mon[31182]: from='client.24974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:07 smithi142 ceph-mon[31182]: pgmap v136: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2024-08-17T17:00:07.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:07 smithi142 ceph-mon[31182]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:07 smithi142 ceph-mon[40020]: from='client.24974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:07 smithi142 ceph-mon[40020]: pgmap v136: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2024-08-17T17:00:07.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:07 smithi142 ceph-mon[40020]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:07.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 122171 122155 94.2398 63.4375 0.0167942 0.0106067 2024-08-17T17:00:07.862 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 podman[70830]: 2024-08-17 17:00:07.610805856 +0000 UTC m=+0.901060929 container remove 77132185e2a310965c3c31a038593f7f78c1039928e8985baae962ccc2d6e320 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:00:07.862 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:00:08.176 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T17:00:08.176 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 systemd[1]: Stopped Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:00:08.176 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:07 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Consumed 10.569s CPU time. 2024-08-17T17:00:08.519 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:08 smithi152 systemd[1]: Starting Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:00:08.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:08] "GET /metrics HTTP/1.1" 200 243298 "" "Prometheus/2.43.0" 2024-08-17T17:00:08.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 123109 123093 93.8054 58.625 0.0301051 0.0106552 2024-08-17T17:00:08.791 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:08 smithi152 podman[70945]: 2024-08-17 17:00:08.422293704 +0000 UTC m=+0.036787656 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:00:08.791 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:08 smithi152 podman[70945]: 2024-08-17 17:00:08.528676547 +0000 UTC m=+0.143170482 container create 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-cd8bdbe) 2024-08-17T17:00:09.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:09 smithi152 podman[70945]: 2024-08-17 17:00:09.007168109 +0000 UTC m=+0.621662056 container init 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:00:09.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:09 smithi152 podman[70945]: 2024-08-17 17:00:09.016705802 +0000 UTC m=+0.631199740 container start 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T17:00:09.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mgr[70979]: -- 172.21.15.152:0/859845611 <== mon.0 v2:172.21.15.142:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x558fe8799520 con 0x558fe876d400 2024-08-17T17:00:09.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mgr[70979]: -- 172.21.15.152:0/859845611 <== mon.0 v2:172.21.15.142:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558fe878f0e0 con 0x558fe876d400 2024-08-17T17:00:09.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:09 smithi152 bash[70945]: 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 2024-08-17T17:00:09.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:09 smithi152 systemd[1]: Started Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:00:09.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 123973 123957 93.3256 54 0.0236355 0.0107106 2024-08-17T17:00:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[31182]: pgmap v137: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2454482926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[40020]: pgmap v137: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:09 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2454482926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:09.963 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:10.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mon[34775]: pgmap v137: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s 2024-08-17T17:00:10.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:10.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:10.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:09 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2454482926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:10 smithi142 ceph-mon[40020]: from='client.24868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:10.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:10 smithi142 ceph-mon[31182]: from='client.24868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:10.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 124769 124753 92.8032 49.75 0.0145175 0.0107708 2024-08-17T17:00:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:10 smithi152 ceph-mon[34775]: from='client.24868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:11.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 125642 125626 92.3534 54.5625 0.0256675 0.0108226 2024-08-17T17:00:11.800 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:11 smithi142 ceph-mon[40020]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:11.800 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:11 smithi142 ceph-mon[40020]: pgmap v138: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:00:11.801 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:11 smithi142 ceph-mon[31182]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:11.801 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:11 smithi142 ceph-mon[31182]: pgmap v138: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:00:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:11 smithi152 ceph-mon[34775]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:11 smithi152 ceph-mon[34775]: pgmap v138: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:00:12.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 126509 126493 91.9096 54.1875 0.02163 0.0108752 2024-08-17T17:00:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:12 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:12 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:12 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:13.391 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:13.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 127345 127329 91.4537 52.25 0.00979299 0.0109292 2024-08-17T17:00:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:13 smithi152 ceph-mon[34775]: pgmap v139: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-08-17T17:00:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:13 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3334168177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:13 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[40020]: pgmap v139: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-08-17T17:00:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3334168177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:13.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:13.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[31182]: pgmap v139: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-08-17T17:00:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3334168177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:13 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:14.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 128181 128165 91.0082 52.25 0.00591429 0.0109828 2024-08-17T17:00:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:14 smithi152 ceph-mon[34775]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:14 smithi152 ceph-mon[34775]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:14 smithi142 ceph-mon[40020]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:14 smithi142 ceph-mon[40020]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:14 smithi142 ceph-mon[31182]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:14 smithi142 ceph-mon[31182]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:15.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 129080 129064 90.6169 56.1875 0.0151349 0.01103 2024-08-17T17:00:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:15 smithi152 ceph-mon[34775]: pgmap v140: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:00:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:15 smithi142 ceph-mon[31182]: pgmap v140: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:00:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:15 smithi142 ceph-mon[40020]: pgmap v140: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:00:16.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0118 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 129914 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 90.2062 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 43.3369 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 272.438 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 49.75 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1443 2024-08-17T17:00:16.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 693.391 2024-08-17T17:00:16.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 4359 2024-08-17T17:00:16.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 796 2024-08-17T17:00:16.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0110815 2024-08-17T17:00:16.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.00801543 2024-08-17T17:00:16.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 0.167154 2024-08-17T17:00:16.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00119736 2024-08-17T17:00:16.787 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:00:16.792 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:16 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2351400846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:16 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2351400846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:16 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2351400846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: pgmap v141: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2024-08-17T17:00:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: from='client.25013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:17] ENGINE Bus STARTING 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: CherryPy Checker: 2024-08-17T17:00:17.824 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: The Application mounted at '' has an empty config. 2024-08-17T17:00:17.825 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:17.825 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:17] ENGINE Serving on http://:::9283 2024-08-17T17:00:17.825 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:17] ENGINE Bus STARTED 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: pgmap v141: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: from='client.25013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: Standby manager daemon x restarted 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: Standby manager daemon x started 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:00:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: pgmap v141: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: from='client.25013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: Standby manager daemon x restarted 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: Standby manager daemon x started 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:00:18.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:17 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:18.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:18] "GET /metrics HTTP/1.1" 200 243321 "" "Prometheus/2.43.0" 2024-08-17T17:00:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:18 smithi152 ceph-mon[34775]: from='client.24904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:18 smithi152 ceph-mon[34775]: mgrmap e21: y(active, since 4m), standbys: x 2024-08-17T17:00:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:18 smithi142 ceph-mon[40020]: from='client.24904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:18 smithi142 ceph-mon[40020]: mgrmap e21: y(active, since 4m), standbys: x 2024-08-17T17:00:19.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:18 smithi142 ceph-mon[31182]: from='client.24904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:18 smithi142 ceph-mon[31182]: mgrmap e21: y(active, since 4m), standbys: x 2024-08-17T17:00:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:19 smithi152 ceph-mon[34775]: pgmap v142: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 812 op/s 2024-08-17T17:00:20.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:19 smithi142 ceph-mon[31182]: pgmap v142: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 812 op/s 2024-08-17T17:00:20.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:19 smithi142 ceph-mon[40020]: pgmap v142: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 812 op/s 2024-08-17T17:00:20.223 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:20 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/891002426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:20 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/891002426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:21.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:20 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/891002426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:21 smithi152 ceph-mon[34775]: from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:21 smithi152 ceph-mon[34775]: pgmap v143: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 59 MiB/s wr, 1.09k op/s 2024-08-17T17:00:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:21 smithi152 ceph-mon[34775]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:22.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:21 smithi142 ceph-mon[31182]: from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:21 smithi142 ceph-mon[31182]: pgmap v143: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 59 MiB/s wr, 1.09k op/s 2024-08-17T17:00:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:21 smithi142 ceph-mon[31182]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:21 smithi142 ceph-mon[40020]: from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:21 smithi142 ceph-mon[40020]: pgmap v143: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 59 MiB/s wr, 1.09k op/s 2024-08-17T17:00:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:21 smithi142 ceph-mon[40020]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:22.822 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:22 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:00:23.600 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:23 smithi142 ceph-mon[40020]: pgmap v144: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 731 op/s 2024-08-17T17:00:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:23 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2785649408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:23 smithi142 ceph-mon[31182]: pgmap v144: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 731 op/s 2024-08-17T17:00:23.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:23 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2785649408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:23 smithi152 ceph-mon[34775]: pgmap v144: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 731 op/s 2024-08-17T17:00:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:23 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2785649408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:24.795 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:24 smithi152 ceph-mon[34775]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:24.795 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:24 smithi152 ceph-mon[34775]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:24 smithi142 ceph-mon[40020]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:24 smithi142 ceph-mon[40020]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:24 smithi142 ceph-mon[31182]: from='client.15189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:24 smithi142 ceph-mon[31182]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:25.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:25 smithi152 ceph-mon[34775]: pgmap v145: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.14k op/s 2024-08-17T17:00:26.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:25 smithi142 ceph-mon[40020]: pgmap v145: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.14k op/s 2024-08-17T17:00:26.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:25 smithi142 ceph-mon[31182]: pgmap v145: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.14k op/s 2024-08-17T17:00:26.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:26 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/117787584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:27.013 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:27.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:26 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/117787584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:27.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:26 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/117787584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:27 smithi152 ceph-mon[34775]: pgmap v146: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 780 op/s 2024-08-17T17:00:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:27 smithi152 ceph-mon[34775]: from='client.24931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:27 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[40020]: pgmap v146: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 780 op/s 2024-08-17T17:00:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[40020]: from='client.24931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[31182]: pgmap v146: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 780 op/s 2024-08-17T17:00:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[31182]: from='client.24931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:27 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:28.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:28] "GET /metrics HTTP/1.1" 200 243384 "" "Prometheus/2.43.0" 2024-08-17T17:00:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:28 smithi152 ceph-mon[34775]: from='client.25064 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:28 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-mon[40020]: from='client.25064 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-mon[31182]: from='client.25064 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:28 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:29 smithi152 ceph-mon[34775]: pgmap v147: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 822 op/s 2024-08-17T17:00:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:29 smithi142 ceph-mon[40020]: pgmap v147: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 822 op/s 2024-08-17T17:00:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:29 smithi142 ceph-mon[31182]: pgmap v147: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 822 op/s 2024-08-17T17:00:30.506 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:00:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:30 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2821433082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:30 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2821433082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:30 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2821433082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:31 smithi152 ceph-mon[34775]: from='client.24940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:31 smithi152 ceph-mon[34775]: pgmap v148: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 1.10k op/s 2024-08-17T17:00:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:31 smithi152 ceph-mon[34775]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:31 smithi142 ceph-mon[40020]: from='client.24940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:31 smithi142 ceph-mon[40020]: pgmap v148: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 1.10k op/s 2024-08-17T17:00:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:31 smithi142 ceph-mon[40020]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:31 smithi142 ceph-mon[31182]: from='client.24940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:31 smithi142 ceph-mon[31182]: pgmap v148: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 1.10k op/s 2024-08-17T17:00:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:31 smithi142 ceph-mon[31182]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:32.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:32 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:33] ENGINE Bus STOPPING 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: pgmap v149: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1738411043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:33 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.060 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setuser ceph since I am not root 2024-08-17T17:00:34.060 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ignoring --setgroup ceph since I am not root 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:34.057+0000 7f063ec2dec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: pgmap v149: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1738411043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.061 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: pgmap v149: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2024-08-17T17:00:34.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1738411043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:00:34.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:34.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:34.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:33 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' 2024-08-17T17:00:34.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:00:34.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:34] ENGINE Bus STOPPED 2024-08-17T17:00:34.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:34.275+0000 7f063ec2dec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:00:34.728 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:34.383+0000 7f063ec2dec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:00:34.806 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENOTSUP: Warning: due to ceph-mgr restart, some PG states may not be up to date 2024-08-17T17:00:34.806 INFO:teuthology.orchestra.run.smithi142.stderr:Module 'orchestrator' is not enabled/loaded (required by command 'orch upgrade status'): use `ceph mgr module enable orchestrator` to enable it 2024-08-17T17:00:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: Activating manager daemon x 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: mgrmap e22: x(active, starting, since 0.0232839s) 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:35.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:00:35.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-mon[34775]: Manager daemon x is now available 2024-08-17T17:00:35.076 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:34.800+0000 7f61f4554640 -1 mgr.server reply reply (95) Operation not supported Warning: due to ceph-mgr restart, some PG states may not be up to date 2024-08-17T17:00:35.076 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: Module 'orchestrator' is not enabled/loaded (required by command 'orch upgrade status'): use `ceph mgr module enable orchestrator` to enable it 2024-08-17T17:00:35.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:34.774+0000 7f063ec2dec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:00:35.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:34.877+0000 7f063ec2dec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: Activating manager daemon x 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T17:00:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: mgrmap e22: x(active, starting, since 0.0232839s) 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:00:35.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[40020]: Manager daemon x is now available 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: Activating manager daemon x 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:00:35.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.14445 172.21.15.142:0/1732949141' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: mgrmap e22: x(active, starting, since 0.0232839s) 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:00:35.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:00:35.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:34 smithi142 ceph-mon[31182]: Manager daemon x is now available 2024-08-17T17:00:35.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:00:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:35 smithi152 ceph-mon[34775]: mgrmap e23: x(active, since 1.01893s) 2024-08-17T17:00:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:35 smithi152 ceph-mon[34775]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:35 smithi152 ceph-mon[34775]: pgmap v3: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-08-17T17:00:36.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:36] ENGINE Bus STARTING 2024-08-17T17:00:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:35 smithi142 ceph-mon[40020]: mgrmap e23: x(active, since 1.01893s) 2024-08-17T17:00:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:35 smithi142 ceph-mon[40020]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:35 smithi142 ceph-mon[40020]: pgmap v3: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-08-17T17:00:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:35 smithi142 ceph-mon[31182]: mgrmap e23: x(active, since 1.01893s) 2024-08-17T17:00:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:35 smithi142 ceph-mon[31182]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:35 smithi142 ceph-mon[31182]: pgmap v3: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-08-17T17:00:36.160 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:36.549 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:36.368+0000 7f063ec2dec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:00:36.549 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:36.547+0000 7f063ec2dec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:00:36.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:36] ENGINE Serving on http://:::9283 2024-08-17T17:00:36.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:00:36] ENGINE Bus STARTED 2024-08-17T17:00:36.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.295+0000 7f61cfd0b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.295+0000 7f61cfd0b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.295+0000 7f61cfd0b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.295+0000 7f61cfd0b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.295+0000 7f61cfd0b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.301+0000 7f61c81f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.301+0000 7f61c81f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.301+0000 7f61c81f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.301+0000 7f61c81f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.574 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:36.301+0000 7f61c81f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:00:36.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:36.626+0000 7f063ec2dec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: Checking for cert/key for grafana.a 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: Migrating grafana.a cert to cert store 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: Migrating grafana.a key to cert store 2024-08-17T17:00:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:00:37.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:36 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:00:37.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: Checking for cert/key for grafana.a 2024-08-17T17:00:37.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: Migrating grafana.a cert to cert store 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: Migrating grafana.a key to cert store 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:00:37.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:36.953+0000 7f063ec2dec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:37.053+0000 7f063ec2dec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:37.154+0000 7f063ec2dec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:37.353+0000 7f063ec2dec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.357 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: Checking for cert/key for grafana.a 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: Migrating grafana.a cert to cert store 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: Migrating grafana.a key to cert store 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:00:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:00:37.359 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:36 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:00:38.189 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-mon[40020]: Deploying cephadm binary to smithi142 2024-08-17T17:00:38.189 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-mon[40020]: Deploying cephadm binary to smithi152 2024-08-17T17:00:38.190 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-mon[31182]: Deploying cephadm binary to smithi142 2024-08-17T17:00:38.190 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:37 smithi142 ceph-mon[31182]: Deploying cephadm binary to smithi152 2024-08-17T17:00:38.190 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:38.007+0000 7f063ec2dec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-mon[34775]: Deploying cephadm binary to smithi142 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-mon[34775]: Deploying cephadm binary to smithi152 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:37.962+0000 7f61b4290640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:00:37] ENGINE Error in HTTPServer.serve 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: Traceback (most recent call last): 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: self._connections.run(self.expiration_interval) 2024-08-17T17:00:38.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: self._run(expiration_interval) 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: s = self.context.wrap_socket( 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: return self.sslsocket_class._create( 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: self.do_handshake() 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:00:38.324 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: self._sslobj.do_handshake() 2024-08-17T17:00:38.325 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:00:38.325 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:00:38.743 INFO:teuthology.orchestra.run.smithi142.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:00:38.743 INFO:teuthology.orchestra.run.smithi142.stdout:alertmanager.a smithi142 *:9093,9094 running (3m) 25s ago 4m 29.9M - 0.25.0 c8568f914cd2 d120aaf39f51 2024-08-17T17:00:38.743 INFO:teuthology.orchestra.run.smithi142.stdout:grafana.a smithi152 *:3000 running (3m) 11s ago 3m 91.5M - 9.4.7 954c08fa6188 e049e3632985 2024-08-17T17:00:38.743 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi142.ndmoid smithi142 running (2m) 25s ago 2m 14.4M - 17.2.7-1404-gf2c39d4d f69471c56189 bd857f554782 2024-08-17T17:00:38.743 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi152.tacamo smithi152 running (2m) 11s ago 2m 14.3M - 17.2.7-1404-gf2c39d4d f69471c56189 8913efd6a27d 2024-08-17T17:00:38.743 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.x smithi152 *:8443,9283 running (30s) 11s ago 8m 57.0M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 44af0bd5a75c 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.y smithi142 *:9283,8765 running (9m) 25s ago 9m 451M - 17.2.7-1404-gf2c39d4d f69471c56189 c1b9a1fd281f 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a smithi142 running (9m) 25s ago 9m 48.5M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 5787ca46b374 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b smithi152 running (8m) 11s ago 8m 43.3M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 b18785ced07f 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c smithi142 running (8m) 25s ago 8m 37.9M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 24ff3a433074 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.a smithi142 *:9100 running (3m) 25s ago 3m 15.7M - 1.5.0 0da6a335fe13 dde7b19121b8 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.b smithi152 *:9100 running (3m) 11s ago 3m 16.6M - 1.5.0 0da6a335fe13 83ebea2ae7c0 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 smithi142 running (7m) 25s ago 7m 335M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 cb819443ee72 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 smithi142 running (7m) 25s ago 7m 225M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 5cd1d20f8eca 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 smithi142 running (6m) 25s ago 6m 275M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 b599f0d55518 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 smithi142 running (6m) 25s ago 6m 152M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 d53780492c5b 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 smithi152 running (6m) 11s ago 6m 336M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 7a1cab3a1049 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 smithi152 running (5m) 11s ago 5m 363M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 cac09b3f3a95 2024-08-17T17:00:38.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 smithi152 running (5m) 11s ago 5m 187M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 81327b9b9e78 2024-08-17T17:00:38.745 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 smithi152 running (5m) 11s ago 5m 228M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 0a45f0507de1 2024-08-17T17:00:38.745 INFO:teuthology.orchestra.run.smithi142.stdout:prometheus.a smithi152 *:9095 running (3m) 11s ago 4m 51.8M - 2.43.0 a07b618ecd1d fa63d07ad574 2024-08-17T17:00:38.745 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:38.514+0000 7f063ec2dec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:00:38.745 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:38.627+0000 7f063ec2dec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:38.845+0000 7f063ec2dec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[40020]: [17/Aug/2024:17:00:37] ENGINE Bus STARTING 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[40020]: pgmap v5: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[40020]: mgrmap e24: x(active, since 4s) 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[40020]: [17/Aug/2024:17:00:37] ENGINE Error in HTTPServer.serve 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mon.c.smithi142.stdout: Traceback (most recent call last): 2024-08-17T17:00:39.105 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._run(expiration_interval) 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:00:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout: self.do_handshake() 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.c.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[40020]: [17/Aug/2024:17:00:37] ENGINE Serving on https://172.21.15.152:7150 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[31182]: [17/Aug/2024:17:00:37] ENGINE Bus STARTING 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[31182]: pgmap v5: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[31182]: mgrmap e24: x(active, since 4s) 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[31182]: [17/Aug/2024:17:00:37] ENGINE Error in HTTPServer.serve 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout: Traceback (most recent call last): 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:00:39.107 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._run(expiration_interval) 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:00:39.108 INFO:journalctl@ceph.mon.a.smithi142.stdout: self.do_handshake() 2024-08-17T17:00:39.109 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:00:39.109 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T17:00:39.109 INFO:journalctl@ceph.mon.a.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:00:39.109 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:38 smithi142 ceph-mon[31182]: [17/Aug/2024:17:00:37] ENGINE Serving on https://172.21.15.152:7150 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:38 smithi152 ceph-mon[34775]: [17/Aug/2024:17:00:37] ENGINE Bus STARTING 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:38 smithi152 ceph-mon[34775]: pgmap v5: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:38 smithi152 ceph-mon[34775]: mgrmap e24: x(active, since 4s) 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:38 smithi152 ceph-mon[34775]: [17/Aug/2024:17:00:37] ENGINE Error in HTTPServer.serve 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout: Traceback (most recent call last): 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:00:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._run(expiration_interval) 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: s = self.context.wrap_socket( 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: return self.sslsocket_class._create( 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: self.do_handshake() 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:00:39.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._sslobj.do_handshake() 2024-08-17T17:00:39.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:00:39.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:38 smithi152 ceph-mon[34775]: [17/Aug/2024:17:00:37] ENGINE Serving on https://172.21.15.152:7150 2024-08-17T17:00:39.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:39.296+0000 7f063ec2dec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:00:39.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:00:39.726 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:39.391+0000 7f063ec2dec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:00:39.727 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:39.723+0000 7f063ec2dec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:00:39.937 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:39.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-mon[40020]: [17/Aug/2024:17:00:38] ENGINE Serving on http://172.21.15.152:8765 2024-08-17T17:00:39.993 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-mon[40020]: [17/Aug/2024:17:00:38] ENGINE Bus STARTED 2024-08-17T17:00:39.993 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-mon[40020]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:39.994 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-mon[31182]: [17/Aug/2024:17:00:38] ENGINE Serving on http://172.21.15.152:8765 2024-08-17T17:00:39.994 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-mon[31182]: [17/Aug/2024:17:00:38] ENGINE Bus STARTED 2024-08-17T17:00:39.994 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-mon[31182]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:39.994 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:39.843+0000 7f063ec2dec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:00:39.995 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:39.989+0000 7f063ec2dec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:00:40.248 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:40 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:40.090+0000 7f063ec2dec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:00:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:39 smithi152 ceph-mon[34775]: [17/Aug/2024:17:00:38] ENGINE Serving on http://172.21.15.152:8765 2024-08-17T17:00:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:39 smithi152 ceph-mon[34775]: [17/Aug/2024:17:00:38] ENGINE Bus STARTED 2024-08-17T17:00:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:39 smithi152 ceph-mon[34775]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:41.231 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:40 smithi142 ceph-mon[40020]: pgmap v6: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 616 B/s wr, 156 op/s 2024-08-17T17:00:41.231 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:40 smithi142 ceph-mon[40020]: mgrmap e25: x(active, since 6s) 2024-08-17T17:00:41.231 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:40 smithi142 ceph-mon[31182]: pgmap v6: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 616 B/s wr, 156 op/s 2024-08-17T17:00:41.231 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:40 smithi142 ceph-mon[31182]: mgrmap e25: x(active, since 6s) 2024-08-17T17:00:41.232 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:40.997+0000 7f063ec2dec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:00:41.232 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:41.229+0000 7f063ec2dec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:00:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:40 smithi152 ceph-mon[34775]: pgmap v6: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 616 B/s wr, 156 op/s 2024-08-17T17:00:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:40 smithi152 ceph-mon[34775]: mgrmap e25: x(active, since 6s) 2024-08-17T17:00:41.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:17:00:41] ENGINE Bus STARTING 2024-08-17T17:00:41.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: CherryPy Checker: 2024-08-17T17:00:41.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: The Application mounted at '' has an empty config. 2024-08-17T17:00:41.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: 2024-08-17T17:00:41.626 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:17:00:41] ENGINE Serving on http://:::9283 2024-08-17T17:00:41.626 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: [17/Aug/2024:17:00:41] ENGINE Bus STARTED 2024-08-17T17:00:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:41 smithi152 ceph-mon[34775]: Standby manager daemon y started 2024-08-17T17:00:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:00:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:00:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:41 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[40020]: Standby manager daemon y started 2024-08-17T17:00:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:00:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:00:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:42.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[31182]: Standby manager daemon y started 2024-08-17T17:00:42.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:00:42.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:00:42.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:00:42.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:41 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2803803423' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:00:42.568 INFO:teuthology.orchestra.run.smithi142.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:00:42.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:42 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:42] "GET /metrics HTTP/1.1" 200 35864 "" "Prometheus/2.43.0" 2024-08-17T17:00:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:42 smithi152 ceph-mon[34775]: pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 KiB/s rd, 440 B/s wr, 561 op/s 2024-08-17T17:00:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:42 smithi152 ceph-mon[34775]: mgrmap e26: x(active, since 8s), standbys: y 2024-08-17T17:00:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:42 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:00:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:42 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[40020]: pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 KiB/s rd, 440 B/s wr, 561 op/s 2024-08-17T17:00:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[40020]: mgrmap e26: x(active, since 8s), standbys: y 2024-08-17T17:00:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:00:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[31182]: pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 25 KiB/s rd, 440 B/s wr, 561 op/s 2024-08-17T17:00:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[31182]: mgrmap e26: x(active, since 8s), standbys: y 2024-08-17T17:00:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:00:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:42 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:43.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:00:44.084 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:43 smithi152 ceph-mon[34775]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:43 smithi152 ceph-mon[34775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:00:44.362 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:43 smithi142 ceph-mon[40020]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:44.362 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:43 smithi142 ceph-mon[40020]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:00:44.363 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:43 smithi142 ceph-mon[31182]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:44.363 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:43 smithi142 ceph-mon[31182]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:00:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:45 smithi152 ceph-mon[34775]: pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 20 KiB/s rd, 342 B/s wr, 436 op/s 2024-08-17T17:00:45.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:45 smithi142 ceph-mon[40020]: pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 20 KiB/s rd, 342 B/s wr, 436 op/s 2024-08-17T17:00:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:45 smithi142 ceph-mon[31182]: pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 20 KiB/s rd, 342 B/s wr, 436 op/s 2024-08-17T17:00:47.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:46 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:47.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:46 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:47.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:46 smithi142 ceph-mon[40020]: pgmap v9: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 743 op/s 2024-08-17T17:00:47.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:46 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:47.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:46 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:47.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:46 smithi142 ceph-mon[31182]: pgmap v9: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 743 op/s 2024-08-17T17:00:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:46 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:46 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:46 smithi152 ceph-mon[34775]: pgmap v9: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 743 op/s 2024-08-17T17:00:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:47 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:47 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:47 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:47 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:47 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:48.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: pgmap v10: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 680 op/s 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.011 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:48 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: pgmap v10: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 680 op/s 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: pgmap v10: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 680 op/s 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:49.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:48 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:49 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi142 to 2480M 2024-08-17T17:00:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:49 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi142 to 2480M 2024-08-17T17:00:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:49 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi142 to 2480M 2024-08-17T17:00:51.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:50 smithi142 ceph-mon[40020]: pgmap v11: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 680 op/s 2024-08-17T17:00:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:50 smithi142 ceph-mon[31182]: pgmap v11: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 680 op/s 2024-08-17T17:00:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:50 smithi152 ceph-mon[34775]: pgmap v11: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 680 op/s 2024-08-17T17:00:52.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:51 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:52.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:51 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:52.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:51 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:52.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:52] "GET /metrics HTTP/1.1" 200 36275 "" "Prometheus/2.43.0" 2024-08-17T17:00:53.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: pgmap v12: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 979 op/s 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: pgmap v12: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 979 op/s 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:53.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:52 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: pgmap v12: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 979 op/s 2024-08-17T17:00:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:53.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:52 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Adjusting osd_memory_target on smithi152 to 2480M 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:54.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:53 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Adjusting osd_memory_target on smithi152 to 2480M 2024-08-17T17:00:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:00:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:00:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Adjusting osd_memory_target on smithi152 to 2480M 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:00:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:54.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:00:54.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:54.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:53 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: pgmap v13: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-08-17T17:00:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:55 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: pgmap v13: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-08-17T17:00:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: pgmap v13: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:55 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:00:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:56 smithi152 ceph-mon[34775]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:00:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:56 smithi152 ceph-mon[34775]: Deploying daemon alertmanager.a on smithi142 2024-08-17T17:00:56.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:56 smithi142 ceph-mon[40020]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:00:56.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:56 smithi142 ceph-mon[40020]: Deploying daemon alertmanager.a on smithi142 2024-08-17T17:00:56.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:56 smithi142 ceph-mon[31182]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:00:56.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:56 smithi142 ceph-mon[31182]: Deploying daemon alertmanager.a on smithi142 2024-08-17T17:00:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:57 smithi152 ceph-mon[34775]: pgmap v14: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:00:57.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:57 smithi142 ceph-mon[40020]: pgmap v14: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:00:57.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:57 smithi142 ceph-mon[31182]: pgmap v14: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:00:58.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:00:58 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:00:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:00:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:00:59 smithi152 ceph-mon[34775]: pgmap v15: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:00:59.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:00:59 smithi142 ceph-mon[40020]: pgmap v15: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:00:59.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:00:59 smithi142 ceph-mon[31182]: pgmap v15: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:01.229 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:01 smithi142 ceph-mon[40020]: pgmap v16: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:01.230 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:01 smithi142 ceph-mon[31182]: pgmap v16: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:01 smithi152 ceph-mon[34775]: pgmap v16: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:02.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:01:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:02] "GET /metrics HTTP/1.1" 200 36275 "" "Prometheus/2.43.0" 2024-08-17T17:01:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:03 smithi152 ceph-mon[34775]: pgmap v17: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:01:03.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:03 smithi142 ceph-mon[31182]: pgmap v17: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:01:03.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:03 smithi142 ceph-mon[40020]: pgmap v17: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:01:04.372 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:03 smithi142 systemd[1]: Stopping Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:04.750 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:04 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[76899]: ts=2024-08-17T17:01:04.397Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:01:04.750 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:04 smithi142 podman[90871]: 2024-08-17 17:01:04.448906585 +0000 UTC m=+0.194760427 container died d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:05.023 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:04 smithi142 podman[90871]: 2024-08-17 17:01:04.953593319 +0000 UTC m=+0.699447156 container cleanup d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:05.023 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:04 smithi142 bash[90871]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:01:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:05 smithi152 ceph-mon[34775]: pgmap v18: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-08-17T17:01:05.341 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:05 smithi142 ceph-mon[40020]: pgmap v18: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-08-17T17:01:05.341 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:05 smithi142 podman[90887]: 2024-08-17 17:01:05.247566082 +0000 UTC m=+0.794106074 container remove d120aaf39f514c75a474a0de641b3968c99b50e0a996c39dbfdb688a2da6cc41 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:05.342 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:05 smithi142 ceph-mon[31182]: pgmap v18: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-08-17T17:01:05.619 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:05 smithi142 podman[90887]: 2024-08-17 17:01:05.341164261 +0000 UTC m=+0.887704246 volume remove 85b3067985d2297f071fa90a2297808231305b11d92a6cf9ab9464f483a374dd 2024-08-17T17:01:05.937 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:05 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:01:05.937 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:05 smithi142 systemd[1]: Stopped Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:05.938 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:05 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 1.448s CPU time. 2024-08-17T17:01:06.275 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:06.276 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:05 smithi142 systemd[1]: Starting Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:06.276 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:06 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:06.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 podman[91007]: 2024-08-17 17:01:06.274965816 +0000 UTC m=+0.121275986 volume create 055833574def27864103d652d17467b929d33fa5c80859ac89617598d51c6271 2024-08-17T17:01:06.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 podman[91007]: 2024-08-17 17:01:06.189156685 +0000 UTC m=+0.035466872 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-08-17T17:01:06.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 podman[91007]: 2024-08-17 17:01:06.400749953 +0000 UTC m=+0.247060124 container create 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:07.037 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:07 smithi142 ceph-mon[31182]: pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:01:07.037 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:07 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:07.037 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:07 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:07.037 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 podman[91007]: 2024-08-17 17:01:06.771356833 +0000 UTC m=+0.617667005 container init 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 podman[91007]: 2024-08-17 17:01:06.780454991 +0000 UTC m=+0.626765157 container start 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.796Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.796Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.798Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.142 port=9094 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.802Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.844Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.845Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.848Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:06.848Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 bash[91007]: 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 2024-08-17T17:01:07.038 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:06 smithi142 systemd[1]: Started Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:07 smithi152 ceph-mon[34775]: pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:01:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:07 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:07 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:07 smithi142 ceph-mon[40020]: pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:01:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:07 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:07 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:08.053 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:07 smithi142 systemd[1]: Stopping Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:08.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:08 smithi142 ceph-mon[40020]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-17T17:01:08.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:08 smithi142 ceph-mon[40020]: Deploying daemon node-exporter.a on smithi142 2024-08-17T17:01:08.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:08 smithi142 ceph-mon[31182]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-17T17:01:08.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:08 smithi142 ceph-mon[31182]: Deploying daemon node-exporter.a on smithi142 2024-08-17T17:01:08.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:08 smithi152 ceph-mon[34775]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-17T17:01:08.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:08 smithi152 ceph-mon[34775]: Deploying daemon node-exporter.a on smithi142 2024-08-17T17:01:08.579 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:01:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:01:08.580 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:08 smithi142 podman[91202]: 2024-08-17 17:01:08.308621996 +0000 UTC m=+0.196224125 container died dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:08.872 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:08 smithi142 podman[91202]: 2024-08-17 17:01:08.685842727 +0000 UTC m=+0.573444863 container cleanup dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:08.872 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:08 smithi142 bash[91202]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T17:01:08.873 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:08.802Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000375997s 2024-08-17T17:01:09.318 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:09 smithi142 ceph-mon[40020]: pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:01:09.318 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:08 smithi142 podman[91220]: 2024-08-17 17:01:08.912816478 +0000 UTC m=+0.591405379 container remove dde7b19121b8f8b3676369f19a3e0a1df9e756896c439344dd9c5a73d72d7a1f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:09.318 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:08 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:01:09.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:09 smithi142 ceph-mon[31182]: pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:01:09.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:09 smithi152 ceph-mon[34775]: pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:01:09.623 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:09 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T17:01:09.623 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:09 smithi142 systemd[1]: Stopped Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:09.623 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:09 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 2.728s CPU time. 2024-08-17T17:01:09.999 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:09 smithi142 systemd[1]: Starting Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:10.340 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:09 smithi142 bash[91336]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T17:01:10.622 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:10 smithi142 bash[91336]: Getting image source signatures 2024-08-17T17:01:10.622 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:10 smithi142 bash[91336]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T17:01:10.622 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:10 smithi142 bash[91336]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T17:01:10.623 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:10 smithi142 bash[91336]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T17:01:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:11 smithi152 ceph-mon[34775]: pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:01:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:11 smithi142 ceph-mon[40020]: pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:01:11.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:11 smithi142 ceph-mon[31182]: pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:01:12.622 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:12 smithi142 bash[91336]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T17:01:12.622 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:12 smithi142 bash[91336]: Writing manifest to image destination 2024-08-17T17:01:12.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:01:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:12] "GET /metrics HTTP/1.1" 200 36271 "" "Prometheus/2.43.0" 2024-08-17T17:01:13.038 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:12 smithi142 podman[91336]: 2024-08-17 17:01:12.630163578 +0000 UTC m=+2.810682877 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T17:01:13.038 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:12 smithi142 podman[91336]: 2024-08-17 17:01:12.760648796 +0000 UTC m=+2.941168094 container create c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:13 smithi152 ceph-mon[34775]: pgmap v22: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:01:13.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-mon[31182]: pgmap v22: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:01:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-mon[40020]: pgmap v22: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 podman[91336]: 2024-08-17 17:01:13.038251907 +0000 UTC m=+3.218771213 container init c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 podman[91336]: 2024-08-17 17:01:13.049247262 +0000 UTC m=+3.229766560 container start c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.052Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.052Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.054Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.054Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.054Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:01:13.375 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.054Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:01:13.376 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:01:13.377 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:01:13.378 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.055Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.056Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:01:13.379 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a[91375]: ts=2024-08-17T17:01:13.056Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:01:13.380 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 bash[91336]: c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 2024-08-17T17:01:13.380 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:01:13 smithi142 systemd[1]: Started Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:14 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:14 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:14 smithi152 ceph-mon[34775]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:01:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:14 smithi152 ceph-mon[34775]: Deploying daemon prometheus.a on smithi152 2024-08-17T17:01:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:14 smithi152 ceph-mon[34775]: pgmap v23: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-08-17T17:01:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[40020]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:01:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[40020]: Deploying daemon prometheus.a on smithi152 2024-08-17T17:01:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[40020]: pgmap v23: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-08-17T17:01:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[31182]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:01:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[31182]: Deploying daemon prometheus.a on smithi152 2024-08-17T17:01:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:14 smithi142 ceph-mon[31182]: pgmap v23: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-08-17T17:01:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:16 smithi142 ceph-mon[40020]: pgmap v24: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:01:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:16 smithi142 ceph-mon[31182]: pgmap v24: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:01:17.123 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:01:16 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:01:16.804Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001705264s 2024-08-17T17:01:17.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:16 smithi152 ceph-mon[34775]: pgmap v24: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:01:18.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:01:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:01:19.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:18 smithi152 ceph-mon[34775]: pgmap v25: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-08-17T17:01:19.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:18 smithi142 ceph-mon[40020]: pgmap v25: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-08-17T17:01:19.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:18 smithi142 ceph-mon[31182]: pgmap v25: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-08-17T17:01:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:20 smithi152 ceph-mon[34775]: pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-08-17T17:01:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:20 smithi142 ceph-mon[40020]: pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-08-17T17:01:21.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:20 smithi142 ceph-mon[31182]: pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-08-17T17:01:22.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:21 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:22.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:21 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:22.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:21 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:22.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:01:22 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:22] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.43.0" 2024-08-17T17:01:23.238 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:22 smithi152 ceph-mon[34775]: pgmap v27: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:01:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:22 smithi142 ceph-mon[40020]: pgmap v27: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:01:23.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:22 smithi142 ceph-mon[31182]: pgmap v27: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:01:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:24 smithi152 ceph-mon[34775]: pgmap v28: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:01:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:24 smithi142 ceph-mon[40020]: pgmap v28: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:01:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:24 smithi142 ceph-mon[31182]: pgmap v28: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:01:27.162 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:26 smithi152 ceph-mon[34775]: pgmap v29: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:26 smithi142 ceph-mon[40020]: pgmap v29: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:27.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:26 smithi142 ceph-mon[31182]: pgmap v29: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:28.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:01:28 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[31433]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:01:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:28 smithi152 ceph-mon[34775]: pgmap v30: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-08-17T17:01:29.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 systemd[1]: Stopping Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:29.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:28 smithi142 ceph-mon[40020]: pgmap v30: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-08-17T17:01:29.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:28 smithi142 ceph-mon[31182]: pgmap v30: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-08-17T17:01:29.865 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.572Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:01:29.866 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.573Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:01:29.867 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.573Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:01:29.867 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.573Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:01:29.867 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[67986]: ts=2024-08-17T17:01:29.573Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:01:29.867 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 podman[79671]: 2024-08-17 17:01:29.580337681 +0000 UTC m=+0.149299499 container died fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:30.283 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 podman[79671]: 2024-08-17 17:01:29.963767819 +0000 UTC m=+0.532729633 container cleanup fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:30.283 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:29 smithi152 bash[79671]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:01:30.549 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:30 smithi152 podman[79686]: 2024-08-17 17:01:30.282688932 +0000 UTC m=+0.688501549 container remove fa63d07ad574b5f2bd2ca9367bdab1238782dde150b6e4d0829f65a0dc6cab40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:30.823 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:30 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:01:30.823 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:30 smithi152 systemd[1]: Stopped Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:30.823 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:30 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 2.202s CPU time. 2024-08-17T17:01:31.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:30 smithi152 ceph-mon[34775]: pgmap v31: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-08-17T17:01:31.199 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:30 smithi152 systemd[1]: Starting Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:31.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:30 smithi142 ceph-mon[40020]: pgmap v31: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-08-17T17:01:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:30 smithi142 ceph-mon[31182]: pgmap v31: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-08-17T17:01:31.533 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 podman[79801]: 2024-08-17 17:01:31.102746882 +0000 UTC m=+0.032961954 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-17T17:01:31.533 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 podman[79801]: 2024-08-17 17:01:31.234694722 +0000 UTC m=+0.164909775 container create ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:31.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 podman[79801]: 2024-08-17 17:01:31.532524062 +0000 UTC m=+0.462739131 container init ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:31.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 podman[79801]: 2024-08-17 17:01:31.543500085 +0000 UTC m=+0.473715141 container start ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:01:31.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.582Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-17T17:01:31.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.583Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-17T17:01:31.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.583Z caller=main.go:623 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi152 (none))" 2024-08-17T17:01:31.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.583Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.583Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.590Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.591Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.594Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.594Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.596Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.596Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.295µs 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.596Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.596Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-08-17T17:01:31.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.629Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.629Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.629Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=44.097µs wal_replay_duration=33.565475ms wbl_replay_duration=128ns total_replay_duration=33.697998ms 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.635Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.636Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.636Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 bash[79801]: ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 systemd[1]: Started Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.664Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=28.133181ms db_storage=1.138µs remote_storage=1.451µs web_handler=414ns query_engine=1.279µs scrape=669.689µs scrape_sd=140.068µs notify=16.481µs notify_sd=11.178µs rules=26.708814ms tracing=8.143µs 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.664Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-17T17:01:31.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:31.664Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:01:32.999 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:32 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:32.999 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:32 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:32.999 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:32 smithi152 ceph-mon[34775]: Reconfiguring grafana.a (dependencies changed)... 2024-08-17T17:01:32.999 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:32 smithi152 ceph-mon[34775]: pgmap v32: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:01:33.000 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:32 smithi152 ceph-mon[34775]: Reconfiguring daemon grafana.a on smithi152 2024-08-17T17:01:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[40020]: Reconfiguring grafana.a (dependencies changed)... 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[40020]: pgmap v32: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[40020]: Reconfiguring daemon grafana.a on smithi152 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[31182]: Reconfiguring grafana.a (dependencies changed)... 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[31182]: pgmap v32: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:01:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:32 smithi142 ceph-mon[31182]: Reconfiguring daemon grafana.a on smithi152 2024-08-17T17:01:35.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:34 smithi152 ceph-mon[34775]: pgmap v33: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-08-17T17:01:35.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:34 smithi142 ceph-mon[40020]: pgmap v33: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-08-17T17:01:35.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:34 smithi142 ceph-mon[31182]: pgmap v33: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-08-17T17:01:35.521 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:35 smithi152 systemd[1]: Stopping Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:36.285 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:35 smithi152 podman[80079]: 2024-08-17 17:01:35.983423511 +0000 UTC m=+0.162134602 container died e049e36329852db01f98c07d7d9b9b80c6288b917b7bd95e939be1bf10a03422 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.openshift.expose-services=, description=Ceph Grafana Container, distribution-scope=public) 2024-08-17T17:01:36.573 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:36 smithi152 podman[80079]: 2024-08-17 17:01:36.390342214 +0000 UTC m=+0.569053308 container cleanup e049e36329852db01f98c07d7d9b9b80c6288b917b7bd95e939be1bf10a03422 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., distribution-scope=public, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64) 2024-08-17T17:01:36.573 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:36 smithi152 bash[80079]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a 2024-08-17T17:01:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:36 smithi152 ceph-mon[34775]: pgmap v34: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:01:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:36 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:37.073 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:36 smithi152 podman[80094]: 2024-08-17 17:01:36.650591324 +0000 UTC m=+0.665936082 container remove e049e36329852db01f98c07d7d9b9b80c6288b917b7bd95e939be1bf10a03422 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, name=ubi8, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-08-17T17:01:37.073 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:36 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:01:37.174 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:36 smithi142 ceph-mon[40020]: pgmap v34: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:01:37.174 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:36 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:37.175 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:36 smithi142 ceph-mon[31182]: pgmap v34: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:01:37.175 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:36 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:37.439 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-17T17:01:37.439 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 systemd[1]: Stopped Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:37.439 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Consumed 6.243s CPU time. 2024-08-17T17:01:37.439 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 systemd[1]: Starting Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:37.756 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 podman[80175]: 2024-08-17 17:01:37.343767769 +0000 UTC m=+0.029601271 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-08-17T17:01:37.756 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 podman[80175]: 2024-08-17 17:01:37.470743105 +0000 UTC m=+0.156576595 container create d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, architecture=x86_64, io.buildah.version=1.32.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, vendor=Red Hat, Inc., vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.openshift.expose-services=) 2024-08-17T17:01:37.756 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 podman[80175]: 2024-08-17 17:01:37.755438345 +0000 UTC m=+0.441271839 container init d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, name=ubi8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, release=239.1651231664, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, architecture=x86_64, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-08-17T17:01:38.073 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 podman[80175]: 2024-08-17 17:01:37.762262885 +0000 UTC m=+0.448096385 container start d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.openshift.tags=base rhel8, name=ubi8, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-08-17T17:01:38.073 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 bash[80175]: d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb 2024-08-17T17:01:38.073 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:37 smithi152 systemd[1]: Started Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:38.493 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:38] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.51.0" 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118222234Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118612776Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118677635Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118720136Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118755507Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118788855Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118825872Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118894504Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118936413Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T17:01:38.494 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.118976996Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.11901437Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.119050224Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.119086951Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=settings t=2024-08-17T17:01:38.119122963Z level=info msg="App mode production" 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=sqlstore t=2024-08-17T17:01:38.119241681Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=sqlstore t=2024-08-17T17:01:38.119322672Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=migrator t=2024-08-17T17:01:38.144394724Z level=info msg="Starting DB migrations" 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=migrator t=2024-08-17T17:01:38.152678106Z level=info msg="migrations completed" performed=0 skipped=481 duration=695.318µs 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=sqlstore t=2024-08-17T17:01:38.153507029Z level=info msg="Created default organization" 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=plugin.loader t=2024-08-17T17:01:38.255440906Z level=info msg="Plugin registered" pluginID=input 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=plugin.loader t=2024-08-17T17:01:38.30181791Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=plugin.loader t=2024-08-17T17:01:38.302088059Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=secrets t=2024-08-17T17:01:38.302258744Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T17:01:38.495 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=query_data t=2024-08-17T17:01:38.304844073Z level=info msg="Query Service initialization" 2024-08-17T17:01:38.496 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=live.push_http t=2024-08-17T17:01:38.309151507Z level=info msg="Live Push Gateway initialization" 2024-08-17T17:01:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-mon[34775]: pgmap v35: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-08-17T17:01:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-mon[34775]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-17T17:01:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:38 smithi152 ceph-mon[34775]: Deploying daemon node-exporter.b on smithi152 2024-08-17T17:01:39.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:38 smithi152 systemd[1]: Stopping Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[40020]: pgmap v35: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-08-17T17:01:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[40020]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[40020]: Deploying daemon node-exporter.b on smithi152 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[31182]: pgmap v35: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[31182]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-17T17:01:39.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:38 smithi142 ceph-mon[31182]: Deploying daemon node-exporter.b on smithi152 2024-08-17T17:01:39.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:39 smithi152 podman[80355]: 2024-08-17 17:01:39.316303998 +0000 UTC m=+0.176174755 container died 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:01:40.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:39 smithi152 podman[80355]: 2024-08-17 17:01:39.822119899 +0000 UTC m=+0.681990649 container cleanup 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:01:40.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:39 smithi152 bash[80355]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T17:01:40.073 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=infra.usagestats.collector t=2024-08-17T17:01:39.641498102Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T17:01:40.074 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=provisioning.datasources t=2024-08-17T17:01:39.737201873Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-08-17T17:01:40.074 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=provisioning.datasources t=2024-08-17T17:01:39.737314338Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-17T17:01:40.074 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=provisioning.datasources t=2024-08-17T17:01:39.748104089Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T17:01:40.074 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=provisioning.alerting t=2024-08-17T17:01:39.855798144Z level=info msg="starting to provision alerting" 2024-08-17T17:01:40.074 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=provisioning.alerting t=2024-08-17T17:01:39.855922564Z level=info msg="finished to provision alerting" 2024-08-17T17:01:40.075 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=grafanaStorageLogger t=2024-08-17T17:01:39.856265289Z level=info msg="storage starting" 2024-08-17T17:01:40.075 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=ngalert.state.manager t=2024-08-17T17:01:39.857572302Z level=info msg="Warming state cache for startup" 2024-08-17T17:01:40.075 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=http.server t=2024-08-17T17:01:39.858520556Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T17:01:40.075 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=ngalert.state.manager t=2024-08-17T17:01:39.869010124Z level=info msg="State cache has been initialized" states=0 duration=11.436393ms 2024-08-17T17:01:40.075 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=ticker t=2024-08-17T17:01:39.869341127Z level=info msg=starting first_tick=2024-08-17T17:01:40Z 2024-08-17T17:01:40.075 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:01:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=ngalert.multiorg.alertmanager t=2024-08-17T17:01:39.869477278Z level=info msg="starting MultiOrg Alertmanager" 2024-08-17T17:01:40.484 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:40 smithi152 podman[80370]: 2024-08-17 17:01:40.224592464 +0000 UTC m=+0.906272745 container remove 83ebea2ae7c0062b8933faf5d952ce61a36bc7b8d2d3f796d9a2f50601c74270 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:01:40.484 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:40 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:01:40.798 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:40 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T17:01:40.798 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:40 smithi152 systemd[1]: Stopped Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:40.799 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:40 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 2.691s CPU time. 2024-08-17T17:01:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:40 smithi152 ceph-mon[34775]: pgmap v36: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-08-17T17:01:41.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:40 smithi152 systemd[1]: Starting Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:41.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:40 smithi142 ceph-mon[40020]: pgmap v36: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-08-17T17:01:41.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:40 smithi142 ceph-mon[31182]: pgmap v36: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-08-17T17:01:41.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:41 smithi152 bash[80482]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T17:01:42.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:41 smithi152 bash[80482]: Getting image source signatures 2024-08-17T17:01:42.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:41 smithi152 bash[80482]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T17:01:42.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:41 smithi152 bash[80482]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T17:01:42.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:41 smithi152 bash[80482]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T17:01:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:42 smithi152 ceph-mon[34775]: pgmap v37: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:01:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:42 smithi142 ceph-mon[40020]: pgmap v37: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:01:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:42 smithi142 ceph-mon[31182]: pgmap v37: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:01:44.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:43 smithi152 bash[80482]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T17:01:44.073 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:43 smithi152 bash[80482]: Writing manifest to image destination 2024-08-17T17:01:44.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:44.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:44 smithi152 podman[80482]: 2024-08-17 17:01:44.212762675 +0000 UTC m=+3.196751263 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T17:01:44.574 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:44 smithi152 podman[80482]: 2024-08-17 17:01:44.335330973 +0000 UTC m=+3.319319529 container create 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:01:45.317 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:44 smithi152 ceph-mon[34775]: pgmap v38: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:01:45.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:44 smithi142 ceph-mon[40020]: pgmap v38: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:01:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:44 smithi142 ceph-mon[31182]: pgmap v38: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:01:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 podman[80482]: 2024-08-17 17:01:45.580885152 +0000 UTC m=+4.564873714 container init 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 podman[80482]: 2024-08-17 17:01:45.588827012 +0000 UTC m=+4.572815568 container start 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.597Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.597Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:01:46.075 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.598Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:01:46.076 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:01:46.077 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:01:46.078 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:01:46.079 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b[80522]: ts=2024-08-17T17:01:45.599Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:01:46.080 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 bash[80482]: 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 2024-08-17T17:01:46.080 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:01:45 smithi152 systemd[1]: Started Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: pgmap v39: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: pgmap v39: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:01:46.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:46.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:46.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:46 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:47.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade resume' 2024-08-17T17:01:47.052 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:47.052 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:47.052 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:01:47.052 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: pgmap v39: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:47.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:47.053 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:46.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:48.043 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:48.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:01:48 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:48] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.51.0" 2024-08-17T17:01:49.147 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:48 smithi142 ceph-mon[31182]: pgmap v40: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:01:49.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:48 smithi142 ceph-mon[40020]: pgmap v40: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:01:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:48 smithi152 ceph-mon[34775]: pgmap v40: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:01:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:50 smithi142 ceph-mon[31182]: pgmap v41: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:01:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:50 smithi142 ceph-mon[40020]: pgmap v41: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:01:51.229 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:50 smithi152 ceph-mon[34775]: pgmap v41: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:01:51.937 INFO:teuthology.orchestra.run.smithi142.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:01:52.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:51 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:51 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:52.174 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:51 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:52.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for majority of mons to upgrade' 2024-08-17T17:01:53.307 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:52 smithi152 ceph-mon[34775]: pgmap v42: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:53.308 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:52 smithi152 ceph-mon[34775]: from='client.25124 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:53.308 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:52 smithi152 ceph-mon[34775]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:01:53.308 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.308 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.308 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:52 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[31182]: pgmap v42: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[31182]: from='client.25124 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[31182]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[40020]: pgmap v42: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[40020]: from='client.25124 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[40020]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:01:53.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:53.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:52 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:54.131 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:54.522 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:55.269 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:54 smithi142 ceph-mon[31182]: pgmap v43: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:55.269 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:54 smithi142 ceph-mon[40020]: pgmap v43: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:54 smithi152 ceph-mon[34775]: pgmap v43: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:01:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:56.720 INFO:teuthology.orchestra.run.smithi142.stdout:wait for majority of mons to upgrade 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: pgmap v44: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: pgmap v44: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:56 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: pgmap v44: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:01:56.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:57.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:01:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:01:56.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:01:57.772 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:01:57.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:57 smithi142 ceph-mon[40020]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:01:57.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:57 smithi142 ceph-mon[40020]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:01:57.915 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:57 smithi142 ceph-mon[31182]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:01:57.916 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:57 smithi142 ceph-mon[31182]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:01:58.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:57 smithi152 ceph-mon[34775]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:01:58.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:57 smithi152 ceph-mon[34775]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:01:58.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:01:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:01:58] "GET /metrics HTTP/1.1" 200 36272 "" "Prometheus/2.51.0" 2024-08-17T17:01:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:01:58 smithi152 ceph-mon[34775]: pgmap v45: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-08-17T17:01:59.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:01:58 smithi142 ceph-mon[31182]: pgmap v45: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-08-17T17:01:59.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:01:58 smithi142 ceph-mon[40020]: pgmap v45: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-08-17T17:01:59.580 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:02:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:00.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:00.844 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: Upgrade: Updating mgr.y 2024-08-17T17:02:00.844 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:00.844 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:00.844 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: Deploying daemon mgr.y on smithi142 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[40020]: pgmap v46: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: Upgrade: Updating mgr.y 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:00.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: Deploying daemon mgr.y on smithi142 2024-08-17T17:02:00.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:00 smithi142 ceph-mon[31182]: pgmap v46: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: Upgrade: Updating mgr.y 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: Deploying daemon mgr.y on smithi142 2024-08-17T17:02:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:00 smithi152 ceph-mon[34775]: pgmap v46: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-08-17T17:02:02.742 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:02 smithi152 ceph-mon[34775]: pgmap v47: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:02:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:02 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1750115705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:03.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:02 smithi142 systemd[1]: Stopping Ceph mgr.y for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:02:03.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:02 smithi142 ceph-mon[40020]: pgmap v47: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:02:03.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:02 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1750115705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:03.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:02 smithi142 ceph-mon[31182]: pgmap v47: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:02:03.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:02 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1750115705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:03.734 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:03 smithi142 podman[92991]: 2024-08-17 17:02:03.42857475 +0000 UTC m=+0.198879400 container died c1b9a1fd281fe5799d3c49f73ecec258e418fea7942142fd721ba5fbcc15a58e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:02:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:03 smithi152 ceph-mon[34775]: from='client.25136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:03 smithi142 ceph-mon[31182]: from='client.25136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:04.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:03 smithi142 ceph-mon[40020]: from='client.25136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:04.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:03 smithi142 podman[92991]: 2024-08-17 17:02:03.923590467 +0000 UTC m=+0.693895122 container cleanup c1b9a1fd281fe5799d3c49f73ecec258e418fea7942142fd721ba5fbcc15a58e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:02:04.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:03 smithi142 bash[92991]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y 2024-08-17T17:02:04.556 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:04 smithi142 podman[93010]: 2024-08-17 17:02:04.293428041 +0000 UTC m=+0.866064612 container remove c1b9a1fd281fe5799d3c49f73ecec258e418fea7942142fd721ba5fbcc15a58e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731) 2024-08-17T17:02:04.556 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:04 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:02:04.556 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:04 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-08-17T17:02:04.556 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:04 smithi142 systemd[1]: Stopped Ceph mgr.y for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:02:04.556 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:04 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Consumed 43.424s CPU time. 2024-08-17T17:02:04.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:04 smithi142 ceph-mon[31182]: from='client.25009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:04 smithi142 ceph-mon[31182]: pgmap v48: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-08-17T17:02:05.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:04 smithi142 systemd[1]: Starting Ceph mgr.y for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:02:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:04 smithi142 ceph-mon[40020]: from='client.25009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:04 smithi142 ceph-mon[40020]: pgmap v48: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-08-17T17:02:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:04 smithi152 ceph-mon[34775]: from='client.25009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:04 smithi152 ceph-mon[34775]: pgmap v48: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-08-17T17:02:05.541 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:05 smithi142 podman[93126]: 2024-08-17 17:02:05.094559883 +0000 UTC m=+0.030977516 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:02:05.541 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:05 smithi142 podman[93126]: 2024-08-17 17:02:05.228023587 +0000 UTC m=+0.164441201 container create 323039997e228bb34547f4fda708dc39c28eb7bcd3d831b8cb6484a68167d9bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0) 2024-08-17T17:02:05.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:05 smithi142 podman[93126]: 2024-08-17 17:02:05.539649641 +0000 UTC m=+0.476067254 container init 323039997e228bb34547f4fda708dc39c28eb7bcd3d831b8cb6484a68167d9bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:02:05.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:05 smithi142 podman[93126]: 2024-08-17 17:02:05.548396909 +0000 UTC m=+0.484814519 container start 323039997e228bb34547f4fda708dc39c28eb7bcd3d831b8cb6484a68167d9bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:02:05.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:05 smithi142 bash[93126]: 323039997e228bb34547f4fda708dc39c28eb7bcd3d831b8cb6484a68167d9bf 2024-08-17T17:02:05.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:05 smithi142 systemd[1]: Started Ceph mgr.y for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:02:06.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:06.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:05.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:06.461 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:07.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:07.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: pgmap v49: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4270876078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: pgmap v49: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:02:07.051 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/4270876078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:07.052 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:07.052 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.052 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:06 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.072 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:06.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: pgmap v49: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4270876078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:06 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:07 smithi152 ceph-mon[34775]: from='client.25021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:07 smithi152 ceph-mon[34775]: from='client.15300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:08.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:07 smithi142 ceph-mon[31182]: from='client.25021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:08.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:07 smithi142 ceph-mon[31182]: from='client.15300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:08.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:07 smithi142 ceph-mon[40020]: from='client.25021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:08.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:07 smithi142 ceph-mon[40020]: from='client.15300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:08.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:02:08] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.51.0" 2024-08-17T17:02:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:08 smithi142 ceph-mon[40020]: mgrmap e27: x(active, since 94s), standbys: y 2024-08-17T17:02:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:08 smithi142 ceph-mon[40020]: pgmap v50: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-08-17T17:02:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:08 smithi142 ceph-mon[31182]: mgrmap e27: x(active, since 94s), standbys: y 2024-08-17T17:02:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:08 smithi142 ceph-mon[31182]: pgmap v50: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-08-17T17:02:09.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:08 smithi152 ceph-mon[34775]: mgrmap e27: x(active, since 94s), standbys: y 2024-08-17T17:02:09.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:08 smithi152 ceph-mon[34775]: pgmap v50: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-08-17T17:02:10.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:09 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/817322729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:10.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:09 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/817322729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:10.187 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:10.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:09 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/817322729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:10 smithi152 ceph-mon[34775]: pgmap v51: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-08-17T17:02:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:10 smithi142 ceph-mon[31182]: pgmap v51: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-08-17T17:02:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:10 smithi142 ceph-mon[40020]: pgmap v51: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-08-17T17:02:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:11 smithi152 ceph-mon[34775]: from='client.15312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:11 smithi152 ceph-mon[34775]: from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:11 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:11 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:12.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[31182]: from='client.15312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[31182]: from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[40020]: from='client.15312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[40020]: from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:11 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[40020]: pgmap v52: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.03k op/s 2024-08-17T17:02:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[31182]: pgmap v52: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.03k op/s 2024-08-17T17:02:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:12 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:12 smithi152 ceph-mon[34775]: pgmap v52: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.03k op/s 2024-08-17T17:02:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:12 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:12 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:12 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:12 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:13.985 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:13 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1677570472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:13 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1677570472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:14.148 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:13 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1677570472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:14.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:14.611 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:14] ENGINE Bus STARTING 2024-08-17T17:02:14.611 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: CherryPy Checker: 2024-08-17T17:02:14.611 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: The Application mounted at '' has an empty config. 2024-08-17T17:02:14.611 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:14.866 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:14] ENGINE Serving on http://:::9283 2024-08-17T17:02:14.866 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:14] ENGINE Bus STARTED 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: pgmap v53: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 664 op/s 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: from='client.25039 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: Standby manager daemon y restarted 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: Standby manager daemon y started 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:02:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: pgmap v53: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 664 op/s 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: from='client.25039 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: Standby manager daemon y restarted 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: Standby manager daemon y started 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:02:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:14 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: pgmap v53: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 664 op/s 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: from='client.25039 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: Standby manager daemon y restarted 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: Standby manager daemon y started 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:02:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:14 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:15 smithi152 ceph-mon[34775]: from='client.15333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:15 smithi152 ceph-mon[34775]: mgrmap e28: x(active, since 101s), standbys: y 2024-08-17T17:02:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:16.362 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:15 smithi142 ceph-mon[31182]: from='client.15333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:16.362 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:15 smithi142 ceph-mon[31182]: mgrmap e28: x(active, since 101s), standbys: y 2024-08-17T17:02:16.363 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:15 smithi142 ceph-mon[40020]: from='client.15333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:16.363 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:15 smithi142 ceph-mon[40020]: mgrmap e28: x(active, since 101s), standbys: y 2024-08-17T17:02:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:16 smithi152 ceph-mon[34775]: pgmap v54: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 979 op/s 2024-08-17T17:02:17.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:17.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:16 smithi142 ceph-mon[40020]: pgmap v54: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 979 op/s 2024-08-17T17:02:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:16 smithi142 ceph-mon[31182]: pgmap v54: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 979 op/s 2024-08-17T17:02:17.600 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:18.177 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:17 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2510849739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:18.222 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:17 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2510849739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:18.223 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:17 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2510849739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:18.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:02:18] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.51.0" 2024-08-17T17:02:19.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:18] ENGINE Bus STOPPING 2024-08-17T17:02:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='client.25193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: pgmap v55: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 613 op/s 2024-08-17T17:02:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: Activating manager daemon y 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.24859 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: mgrmap e29: y(active, starting, since 0.00926114s) 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:02:19.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:02:19.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='client.25193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: pgmap v55: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 613 op/s 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:19.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: Activating manager daemon y 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.24859 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: mgrmap e29: y(active, starting, since 0.00926114s) 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:02:19.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:02:19.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:18 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='client.25193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: pgmap v55: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 613 op/s 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:19.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 172.21.15.152:0/294618240' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: Activating manager daemon y 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.24859 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: mgrmap e29: y(active, starting, since 0.00926114s) 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:02:19.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.tacamo"}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:02:19.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:02:18.835+0000 7f625d1d0640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ignoring --setuser ceph since I am not root 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ignoring --setgroup ceph since I am not root 2024-08-17T17:02:19.327 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:18 smithi152 ceph-mgr[70979]: -- 172.21.15.152:0/2236357336 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b4d5f910e0 con 0x55b4d5f6f000 2024-08-17T17:02:19.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:02:19.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:19] ENGINE Bus STOPPED 2024-08-17T17:02:20.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:19 smithi152 ceph-mon[34775]: Manager daemon y is now available 2024-08-17T17:02:20.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:19 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:20.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:19 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:20.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:19 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:20.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:19 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:02:20.253 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[40020]: Manager daemon y is now available 2024-08-17T17:02:20.253 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:20.253 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:20.253 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:20.253 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:19] ENGINE Bus STARTING 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:19.990+0000 7f7867156640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:19.990+0000 7f7867156640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:19.990+0000 7f7867156640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:19.990+0000 7f7867156640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:19.990+0000 7f7867156640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:20.001+0000 7f786394f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:20.001+0000 7f786394f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.254 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:20.001+0000 7f786394f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:20.001+0000 7f786394f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:20.001+0000 7f786394f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[31182]: Manager daemon y is now available 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:20.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:19 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:02:20.492 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:20.506 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:20] ENGINE Serving on http://:::9283 2024-08-17T17:02:20.506 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:02:20] ENGINE Bus STARTED 2024-08-17T17:02:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:20 smithi152 ceph-mon[34775]: mgrmap e30: y(active, since 1.10529s) 2024-08-17T17:02:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:20 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:20 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:02:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:20 smithi152 ceph-mon[34775]: pgmap v3: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:20 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3364792142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:21.324 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[31182]: mgrmap e30: y(active, since 1.10529s) 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[31182]: pgmap v3: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3364792142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[40020]: mgrmap e30: y(active, since 1.10529s) 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[40020]: pgmap v3: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:21.325 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:20 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3364792142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:21.326 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:21 smithi152 ceph-mon[34775]: pgmap v4: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:21 smithi152 ceph-mon[34775]: mgrmap e31: y(active, since 2s) 2024-08-17T17:02:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:21 smithi152 ceph-mon[34775]: from='client.25223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:22.327 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:21 smithi142 ceph-mon[40020]: pgmap v4: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:22.327 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:21 smithi142 ceph-mon[40020]: mgrmap e31: y(active, since 2s) 2024-08-17T17:02:22.327 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:21 smithi142 ceph-mon[40020]: from='client.25223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:22.328 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:21 smithi142 ceph-mon[31182]: pgmap v4: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:22.328 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:21 smithi142 ceph-mon[31182]: mgrmap e31: y(active, since 2s) 2024-08-17T17:02:22.328 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:21 smithi142 ceph-mon[31182]: from='client.25223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:22.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:22.324+0000 7f783fe85640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:02:22] ENGINE Error in HTTPServer.serve 2024-08-17T17:02:22.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: Traceback (most recent call last): 2024-08-17T17:02:22.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:02:22.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self._connections.run(self.expiration_interval) 2024-08-17T17:02:22.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:02:22.623 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self._run(expiration_interval) 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: s = self.context.wrap_socket( 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: return self.sslsocket_class._create( 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self.do_handshake() 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:02:22.624 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self._sslobj.do_handshake() 2024-08-17T17:02:22.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:02:22.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:22 smithi152 ceph-mon[34775]: [17/Aug/2024:17:02:22] ENGINE Bus STARTING 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:22 smithi152 ceph-mon[34775]: [17/Aug/2024:17:02:22] ENGINE Serving on http://172.21.15.142:8765 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:22 smithi152 ceph-mon[34775]: [17/Aug/2024:17:02:22] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:22 smithi152 ceph-mon[34775]: [17/Aug/2024:17:02:22] ENGINE Bus STARTED 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:22 smithi152 ceph-mon[34775]: [17/Aug/2024:17:02:22] ENGINE Error in HTTPServer.serve 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout: Traceback (most recent call last): 2024-08-17T17:02:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._run(expiration_interval) 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: s = self.context.wrap_socket( 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:02:23.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: return self.sslsocket_class._create( 2024-08-17T17:02:23.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:02:23.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: self.do_handshake() 2024-08-17T17:02:23.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:02:23.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._sslobj.do_handshake() 2024-08-17T17:02:23.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:02:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[40020]: [17/Aug/2024:17:02:22] ENGINE Bus STARTING 2024-08-17T17:02:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[40020]: [17/Aug/2024:17:02:22] ENGINE Serving on http://172.21.15.142:8765 2024-08-17T17:02:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[40020]: [17/Aug/2024:17:02:22] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[40020]: [17/Aug/2024:17:02:22] ENGINE Bus STARTED 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[40020]: [17/Aug/2024:17:02:22] ENGINE Error in HTTPServer.serve 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: Traceback (most recent call last): 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._run(expiration_interval) 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:02:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: self.do_handshake() 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:02:23.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[31182]: [17/Aug/2024:17:02:22] ENGINE Bus STARTING 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[31182]: [17/Aug/2024:17:02:22] ENGINE Serving on http://172.21.15.142:8765 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[31182]: [17/Aug/2024:17:02:22] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[31182]: [17/Aug/2024:17:02:22] ENGINE Bus STARTED 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:22 smithi142 ceph-mon[31182]: [17/Aug/2024:17:02:22] ENGINE Error in HTTPServer.serve 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: Traceback (most recent call last): 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._run(expiration_interval) 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:02:23.375 INFO:journalctl@ceph.mon.a.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: self.do_handshake() 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T17:02:23.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:02:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:23 smithi152 ceph-mon[34775]: pgmap v5: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:23 smithi152 ceph-mon[34775]: mgrmap e32: y(active, since 4s) 2024-08-17T17:02:24.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:23 smithi142 ceph-mon[40020]: pgmap v5: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:24.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:23 smithi142 ceph-mon[40020]: mgrmap e32: y(active, since 4s) 2024-08-17T17:02:24.132 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:23 smithi142 ceph-mon[31182]: pgmap v5: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:24.132 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:23 smithi142 ceph-mon[31182]: mgrmap e32: y(active, since 4s) 2024-08-17T17:02:24.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:24 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:24.799 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:24 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3121197391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:24 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3121197391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:24 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3121197391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:26.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:26.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:26.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: from='client.15408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:26.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: pgmap v6: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:26.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: from='client.15414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:26.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: from='client.15408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: pgmap v6: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: from='client.15414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: from='client.15408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: pgmap v6: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:02:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: from='client.15414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:26.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:26 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:27.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:27.796 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:02:27] ENGINE Bus STARTING 2024-08-17T17:02:27.796 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: CherryPy Checker: 2024-08-17T17:02:27.796 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: The Application mounted at '' has an empty config. 2024-08-17T17:02:27.797 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: 2024-08-17T17:02:27.797 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:02:27] ENGINE Serving on http://:::9283 2024-08-17T17:02:27.797 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: [17/Aug/2024:17:02:27] ENGINE Bus STARTED 2024-08-17T17:02:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: pgmap v7: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s rd, 298 B/s wr, 498 op/s 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: Standby manager daemon x started 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1928050674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: pgmap v7: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s rd, 298 B/s wr, 498 op/s 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: Standby manager daemon x started 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:27 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1928050674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: pgmap v7: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s rd, 298 B/s wr, 498 op/s 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:02:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/1788921891' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:28.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:27 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1928050674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:28.208 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:28.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:28 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[70957]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:02:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-08-17T17:02:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[31182]: mgrmap e33: y(active, since 9s), standbys: x 2024-08-17T17:02:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:02:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:29.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:29.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[31182]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:29.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[40020]: mgrmap e33: y(active, since 9s), standbys: x 2024-08-17T17:02:29.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:02:29.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:29.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:29.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:28 smithi142 ceph-mon[40020]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:28 smithi152 ceph-mon[34775]: mgrmap e33: y(active, since 9s), standbys: x 2024-08-17T17:02:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:28 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:02:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:28 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:28 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:29.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:28 smithi152 ceph-mon[34775]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:29 smithi142 ceph-mon[31182]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:29 smithi142 ceph-mon[31182]: pgmap v8: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 231 B/s wr, 386 op/s 2024-08-17T17:02:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:29 smithi142 ceph-mon[40020]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:29 smithi142 ceph-mon[40020]: pgmap v8: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 231 B/s wr, 386 op/s 2024-08-17T17:02:30.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:29 smithi152 ceph-mon[34775]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:30.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:29 smithi152 ceph-mon[34775]: pgmap v8: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 20 KiB/s rd, 231 B/s wr, 386 op/s 2024-08-17T17:02:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:31.898 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:32.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:31.641Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-08-17T17:02:32.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:31.642Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-08-17T17:02:32.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:31.648Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-08-17T17:02:32.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:31.648Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-08-17T17:02:32.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:31.648Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-08-17T17:02:32.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:31.648Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.152:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.152:8765: connect: connection refused" 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: pgmap v9: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 16 KiB/s rd, 188 B/s wr, 674 op/s 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:02:32.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:02:32.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:02:32.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:02:32.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:31 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/987011818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: pgmap v9: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 16 KiB/s rd, 188 B/s wr, 674 op/s 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:02:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/987011818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:32.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:32.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: pgmap v9: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 16 KiB/s rd, 188 B/s wr, 674 op/s 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:02:32.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:31 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/987011818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:02:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='client.25265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='client.25271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:33 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:02:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='client.25265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='client.25271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='client.25265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='client.25271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:33.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:33 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:34 smithi152 ceph-mon[34775]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:02:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:34 smithi152 ceph-mon[34775]: pgmap v10: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 609 op/s 2024-08-17T17:02:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:34 smithi152 ceph-mon[34775]: Reconfiguring daemon prometheus.a on smithi152 2024-08-17T17:02:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:34 smithi142 ceph-mon[40020]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:02:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:34 smithi142 ceph-mon[40020]: pgmap v10: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 609 op/s 2024-08-17T17:02:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:34 smithi142 ceph-mon[40020]: Reconfiguring daemon prometheus.a on smithi152 2024-08-17T17:02:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:34 smithi142 ceph-mon[31182]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:02:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:34 smithi142 ceph-mon[31182]: pgmap v10: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 609 op/s 2024-08-17T17:02:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:34 smithi142 ceph-mon[31182]: Reconfiguring daemon prometheus.a on smithi152 2024-08-17T17:02:35.502 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:35 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:35 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3632302747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:35.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:35 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:35.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:35 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3632302747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:35.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:35 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:35.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:35 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3632302747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:36.768 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:36 smithi152 ceph-mon[34775]: pgmap v11: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 609 op/s 2024-08-17T17:02:36.769 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:36 smithi152 ceph-mon[34775]: from='client.25283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:37.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:36 smithi152 systemd[1]: Stopping Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:02:37.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:36 smithi142 ceph-mon[40020]: pgmap v11: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 609 op/s 2024-08-17T17:02:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:36 smithi142 ceph-mon[40020]: from='client.25283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:36 smithi142 ceph-mon[31182]: pgmap v11: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 609 op/s 2024-08-17T17:02:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:36 smithi142 ceph-mon[31182]: from='client.25283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:37.633 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:02:37.633 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:02:37.633 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.377Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.379Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.381Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.381Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[79814]: ts=2024-08-17T17:02:37.381Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-17T17:02:37.634 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:37 smithi152 podman[83749]: 2024-08-17 17:02:37.390008797 +0000 UTC m=+0.152976626 container died ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:02:37.934 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:37 smithi152 ceph-mon[34775]: from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:38.023 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:37 smithi142 ceph-mon[40020]: from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:38.024 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:37 smithi142 ceph-mon[31182]: from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:38.294 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 podman[83749]: 2024-08-17 17:02:38.023801626 +0000 UTC m=+0.786769459 container cleanup ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:02:38.294 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 bash[83749]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:02:38.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 podman[83766]: 2024-08-17 17:02:38.293775042 +0000 UTC m=+0.895304249 container remove ea7da71ac034bea3d8fb8d34f7d2d4ee87d753d3521d97eff60e67f05e33c057 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:02:39.039 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:38 smithi142 ceph-mon[40020]: pgmap v12: 89 pgs: 89 active+clean; 836 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 956 op/s 2024-08-17T17:02:39.039 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:38 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1682909248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:39.039 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:38 smithi142 ceph-mon[31182]: pgmap v12: 89 pgs: 89 active+clean; 836 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 956 op/s 2024-08-17T17:02:39.039 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:38 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1682909248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:39.060 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:38 smithi152 ceph-mon[34775]: pgmap v12: 89 pgs: 89 active+clean; 836 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 956 op/s 2024-08-17T17:02:39.060 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:38 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1682909248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:39.060 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:02:39.060 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 systemd[1]: Stopped Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:02:39.061 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.701s CPU time. 2024-08-17T17:02:39.061 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:38 smithi152 systemd[1]: Starting Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:02:39.110 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:39.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 podman[83847]: 2024-08-17 17:02:38.961389514 +0000 UTC m=+0.030024338 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-17T17:02:39.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 podman[83847]: 2024-08-17 17:02:39.098673241 +0000 UTC m=+0.167308050 container create f12943011a9ff37b8e74596b61d25f20d48d3c951f352453bf6890828c0bc0e0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 podman[83847]: 2024-08-17 17:02:39.383269231 +0000 UTC m=+0.451904058 container init f12943011a9ff37b8e74596b61d25f20d48d3c951f352453bf6890828c0bc0e0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 podman[83847]: 2024-08-17 17:02:39.405935448 +0000 UTC m=+0.474570268 container start f12943011a9ff37b8e74596b61d25f20d48d3c951f352453bf6890828c0bc0e0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.438Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.438Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.438Z caller=main.go:623 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi152 (none))" 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.438Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.438Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.443Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.443Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.445Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:02:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.445Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.447Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.448Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.278µs 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.448Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.449Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.484Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.500Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.500Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.500Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=38.996µs wal_replay_duration=52.56524ms wbl_replay_duration=142ns total_replay_duration=52.679142ms 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 bash[83847]: f12943011a9ff37b8e74596b61d25f20d48d3c951f352453bf6890828c0bc0e0 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.507Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.507Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.507Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:02:39.825 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 systemd[1]: Started Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:02:39.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.538Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=30.845635ms db_storage=1.332µs remote_storage=1.334µs web_handler=447ns query_engine=993ns scrape=611.193µs scrape_sd=155.297µs notify=17.399µs notify_sd=12.865µs rules=29.508301ms tracing=8.963µs 2024-08-17T17:02:39.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.538Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-17T17:02:39.826 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:39.538Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:02:40.480 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:40.743 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: pgmap v13: 89 pgs: 89 active+clean; 836 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-08-17T17:02:40.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:40.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:40.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:40.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:02:40.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:02:40.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:40 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:40.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: pgmap v13: 89 pgs: 89 active+clean; 836 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: pgmap v13: 89 pgs: 89 active+clean; 836 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:02:40.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:02:40.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:40 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:41 smithi152 ceph-mon[34775]: from='client.25307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:41 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:41 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:02:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:41 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:41.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[40020]: from='client.25307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[31182]: from='client.25307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:02:41.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:41 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:42.655 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:42 smithi152 ceph-mon[34775]: pgmap v14: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:02:42.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:42 smithi152 ceph-mon[34775]: Upgrade: Updating mgr.x 2024-08-17T17:02:42.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:42 smithi152 ceph-mon[34775]: Deploying daemon mgr.x on smithi152 2024-08-17T17:02:42.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:42 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/1165507775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:42.757 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[40020]: pgmap v14: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:02:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[40020]: Upgrade: Updating mgr.x 2024-08-17T17:02:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[40020]: Deploying daemon mgr.x on smithi152 2024-08-17T17:02:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/1165507775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[31182]: pgmap v14: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:02:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[31182]: Upgrade: Updating mgr.x 2024-08-17T17:02:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[31182]: Deploying daemon mgr.x on smithi152 2024-08-17T17:02:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:42 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/1165507775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:44.323 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:43 smithi152 systemd[1]: Stopping Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:02:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:44 smithi142 ceph-mon[40020]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:44 smithi142 ceph-mon[40020]: pgmap v15: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-08-17T17:02:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:44 smithi142 ceph-mon[40020]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:44 smithi142 ceph-mon[31182]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:44 smithi142 ceph-mon[31182]: pgmap v15: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-08-17T17:02:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:44 smithi142 ceph-mon[31182]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:44 smithi152 ceph-mon[34775]: from='client.15483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:44 smithi152 ceph-mon[34775]: pgmap v15: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-08-17T17:02:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:44 smithi152 ceph-mon[34775]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:45.538 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:45 smithi152 podman[84227]: 2024-08-17 17:02:45.236031347 +0000 UTC m=+0.956656519 container died 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:02:45.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:45 smithi152 podman[84227]: 2024-08-17 17:02:45.636352921 +0000 UTC m=+1.356978082 container cleanup 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:02:45.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:45 smithi152 bash[84227]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:02:46.198 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:45 smithi152 podman[84242]: 2024-08-17 17:02:45.937692543 +0000 UTC m=+0.696511117 container remove 44af0bd5a75cbcddfadd94bbe5b041247da24e98af1163b28c0cfb3c0cd33177 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-17T17:02:46.198 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:45 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:02:46.198 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:46 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T17:02:46.347 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:46.501 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:46 smithi152 systemd[1]: Stopped Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:02:46.501 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:46 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Consumed 29.645s CPU time. 2024-08-17T17:02:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:46 smithi152 ceph-mon[34775]: pgmap v16: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-08-17T17:02:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:46 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3904580835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:46.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:46 smithi152 systemd[1]: Starting Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:02:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:46 smithi142 ceph-mon[31182]: pgmap v16: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-08-17T17:02:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:46 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3904580835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:46.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:46 smithi142 ceph-mon[40020]: pgmap v16: 89 pgs: 89 active+clean; 573 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-08-17T17:02:46.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:46 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3904580835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:47.230 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:46 smithi152 podman[84361]: 2024-08-17 17:02:46.758657866 +0000 UTC m=+0.035604815 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:02:47.230 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:47 smithi152 podman[84361]: 2024-08-17 17:02:47.089377352 +0000 UTC m=+0.366324290 container create bc654892e833d9709220e95046106a9fabceb4a247720599240482809b1e0403 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T17:02:47.231 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:47.494 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:47 smithi152 podman[84361]: 2024-08-17 17:02:47.39314181 +0000 UTC m=+0.670088748 container init bc654892e833d9709220e95046106a9fabceb4a247720599240482809b1e0403 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T17:02:47.494 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:47 smithi152 podman[84361]: 2024-08-17 17:02:47.401560225 +0000 UTC m=+0.678507156 container start bc654892e833d9709220e95046106a9fabceb4a247720599240482809b1e0403 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True) 2024-08-17T17:02:47.494 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:47 smithi152 ceph-mgr[84394]: -- 172.21.15.152:0/1533161389 <== mon.0 v2:172.21.15.142:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5561812410e0 con 0x55618121f400 2024-08-17T17:02:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:47 smithi152 ceph-mon[34775]: from='client.25331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:47.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:47 smithi152 bash[84361]: bc654892e833d9709220e95046106a9fabceb4a247720599240482809b1e0403 2024-08-17T17:02:47.823 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:47 smithi152 systemd[1]: Started Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:02:47.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:47 smithi142 ceph-mon[40020]: from='client.25331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:47 smithi142 ceph-mon[31182]: from='client.25331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[40020]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[40020]: pgmap v17: 89 pgs: 89 active+clean; 309 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:02:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[31182]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[31182]: pgmap v17: 89 pgs: 89 active+clean; 309 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:02:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:48 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:48 smithi152 ceph-mon[34775]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:48 smithi152 ceph-mon[34775]: pgmap v17: 89 pgs: 89 active+clean; 309 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:02:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:48 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:48 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:48 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:49.746 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:49 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2489173221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:49 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2489173221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:49 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/2489173221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:50.577 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:50.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:50 smithi152 ceph-mon[34775]: pgmap v18: 89 pgs: 89 active+clean; 309 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:02:50.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:50 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:51.017 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:50 smithi142 ceph-mon[40020]: pgmap v18: 89 pgs: 89 active+clean; 309 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:02:51.017 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:50 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:51.017 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:50 smithi142 ceph-mon[31182]: pgmap v18: 89 pgs: 89 active+clean; 309 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:02:51.017 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:50 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:02:51.403 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 129914 objects 2024-08-17T17:02:51.403 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :154.616 2024-08-17T17:02:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:51 smithi152 ceph-mon[34775]: from='client.25349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:51 smithi152 ceph-mon[34775]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:02:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:02:51] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-08-17T17:02:52.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:51 smithi142 ceph-mon[31182]: from='client.25349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:51 smithi142 ceph-mon[31182]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:51 smithi142 ceph-mon[40020]: from='client.25349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:51 smithi142 ceph-mon[40020]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:52.842 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:52 smithi152 ceph-mon[34775]: pgmap v19: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:02:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:52 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/463364965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:53.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:52 smithi142 ceph-mon[31182]: pgmap v19: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:02:53.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:52 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/463364965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:52 smithi142 ceph-mon[40020]: pgmap v19: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:02:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:52 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/463364965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:54.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:54.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:54.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: [17/Aug/2024:17:02:54] ENGINE Bus STARTING 2024-08-17T17:02:54.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: CherryPy Checker: 2024-08-17T17:02:54.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: The Application mounted at '' has an empty config. 2024-08-17T17:02:54.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: 2024-08-17T17:02:54.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: [17/Aug/2024:17:02:54] ENGINE Serving on http://:::9283 2024-08-17T17:02:54.573 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: [17/Aug/2024:17:02:54] ENGINE Bus STARTED 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: pgmap v20: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:02:54.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:54.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:54.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:54 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: pgmap v20: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: Standby manager daemon x restarted 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: Standby manager daemon x started 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:02:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: pgmap v20: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: Standby manager daemon x restarted 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: Standby manager daemon x started 2024-08-17T17:02:55.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:02:55.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:02:55.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:02:55.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/2245223587' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:02:55.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:54 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:55.739 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:55.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:55 smithi152 ceph-mon[34775]: mgrmap e34: y(active, since 35s), standbys: x 2024-08-17T17:02:55.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:55 smithi152 ceph-mon[34775]: pgmap v21: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-08-17T17:02:55.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:55 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3745990273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:56.045 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:56.045 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:55 smithi142 ceph-mon[40020]: mgrmap e34: y(active, since 35s), standbys: x 2024-08-17T17:02:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:55 smithi142 ceph-mon[40020]: pgmap v21: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-08-17T17:02:56.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:55 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3745990273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:55 smithi142 ceph-mon[31182]: mgrmap e34: y(active, since 35s), standbys: x 2024-08-17T17:02:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:55 smithi142 ceph-mon[31182]: pgmap v21: 89 pgs: 89 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-08-17T17:02:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:55 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3745990273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:56.842 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-08-17T17:02:56.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-08-17T17:02:56.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:56 smithi152 ceph-mon[34775]: from='client.15558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:56.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:56 smithi152 ceph-mon[34775]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:57.099 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:02:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:56 smithi142 ceph-mon[40020]: from='client.15558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:56 smithi142 ceph-mon[40020]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:57.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:56 smithi142 ceph-mon[31182]: from='client.15558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:56 smithi142 ceph-mon[31182]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:57.233 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:02:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:02:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:02:58.541 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: pgmap v22: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: Upgrade: Setting container_image for all mgr 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[40020]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: pgmap v22: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:58.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: Upgrade: Setting container_image for all mgr 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:02:58.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:58 smithi142 ceph-mon[31182]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: pgmap v22: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: Upgrade: Setting container_image for all mgr 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:02:58.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:58 smithi152 ceph-mon[34775]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:02:58.651 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:02:59.396 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_0' removed 2024-08-17T17:02:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:59 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3181773856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:02:59 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3407279011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-08-17T17:02:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:59 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3181773856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:59.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:02:59 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3407279011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-08-17T17:02:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:59 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3181773856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:02:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:02:59 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3407279011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-08-17T17:02:59.941 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:02:59.942 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:02:59.942 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-08-17T17:02:59.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-08-17T17:03:00.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[31182]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:00.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[31182]: pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 410 op/s 2024-08-17T17:03:00.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[31182]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:00.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/3407279011' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:03:00.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[31182]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:03:00.538 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:00 smithi152 ceph-mon[34775]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:00 smithi152 ceph-mon[34775]: pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 410 op/s 2024-08-17T17:03:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:00 smithi152 ceph-mon[34775]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:00 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3407279011' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:03:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:00 smithi152 ceph-mon[34775]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:03:00.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[40020]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[40020]: pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 410 op/s 2024-08-17T17:03:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[40020]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3407279011' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:03:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:00 smithi142 ceph-mon[40020]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:03:01.574 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: Upgrade: Updating mon.a 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: Deploying daemon mon.a on smithi142 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/267837823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:01.575 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: Upgrade: Updating mon.a 2024-08-17T17:03:01.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:03:01.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:03:01.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:03:01.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:01.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: Deploying daemon mon.a on smithi142 2024-08-17T17:03:01.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-mon[31182]: from='client.? 172.21.15.142:0/267837823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: Upgrade: Updating mon.a 2024-08-17T17:03:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' 2024-08-17T17:03:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:03:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:03:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:03:01.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: Deploying daemon mon.a on smithi142 2024-08-17T17:03:01.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:01 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/267837823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:01.830 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:03:01] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-08-17T17:03:02.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:02 smithi142 ceph-mon[40020]: pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:02.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:02 smithi142 ceph-mon[40020]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:02.602 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:02 smithi142 ceph-mon[31182]: pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:02.602 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:02 smithi142 ceph-mon[31182]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:02.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:02 smithi152 ceph-mon[34775]: pgmap v25: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:02.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:02 smithi152 ceph-mon[34775]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:03.363 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:02 smithi142 systemd[1]: Stopping Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:03:03.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:03 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:03] ENGINE Bus STOPPING 2024-08-17T17:03:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:03 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[31159]: 2024-08-17T17:03:03.360+0000 7f4ad1889640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:03:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:03 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[31159]: 2024-08-17T17:03:03.360+0000 7f4ad1889640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-08-17T17:03:03.921 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:03 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:03:03.921 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:03 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:03] ENGINE Bus STOPPED 2024-08-17T17:03:03.921 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:03 smithi142 podman[103350]: 2024-08-17 17:03:03.656503694 +0000 UTC m=+0.372336371 container died 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-17T17:03:04.295 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 podman[103350]: 2024-08-17 17:03:04.017829126 +0000 UTC m=+0.733661802 container cleanup 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-08-17T17:03:04.295 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 bash[103350]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:03:04.522 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:04.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:04.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 podman[103369]: 2024-08-17 17:03:04.294562453 +0000 UTC m=+0.641624267 container remove 5787ca46b374735813affb33a3771e7fe7263e2f1c7be8bde85dc19dcc2ec7f6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0) 2024-08-17T17:03:04.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T17:03:04.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 systemd[1]: Stopped Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:03:04.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service: Consumed 14.253s CPU time. 2024-08-17T17:03:05.045 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 systemd[1]: Starting Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:03:05.045 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 podman[103573]: 2024-08-17 17:03:04.802574253 +0000 UTC m=+0.017422921 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:03:05.046 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:04 smithi142 podman[103573]: 2024-08-17 17:03:04.921360573 +0000 UTC m=+0.136209301 container create 3a2a93ab0fbb9f60b0e84ff7e882aa4b32934a372b9d9a772f45dcbc4738eb53 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:03:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 podman[103573]: 2024-08-17 17:03:05.148826766 +0000 UTC m=+0.363675436 container init 3a2a93ab0fbb9f60b0e84ff7e882aa4b32934a372b9d9a772f45dcbc4738eb53 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD) 2024-08-17T17:03:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 podman[103573]: 2024-08-17 17:03:05.153402546 +0000 UTC m=+0.368251210 container start 3a2a93ab0fbb9f60b0e84ff7e882aa4b32934a372b9d9a772f45dcbc4738eb53 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-17T17:03:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:03:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: pidfile_write: ignore empty --pid-file 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: load: jerasure load: lrc 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Git sha 0 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: DB SUMMARY 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: DB Session ID: XPCMJVDU8CVTK3H4H2EU 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1179 Bytes 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-08-17T17:03:05.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 1255020 ; 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.env: 0x5594fd6a0be0 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.info_log: 0x5594fe459700 2024-08-17T17:03:05.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.statistics: (nil) 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.use_fsync: 0 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:03:05.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.db_log_dir: 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.wal_dir: 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:03:05.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.write_buffer_manager: 0x5594fe45c960 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:03:05.629 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.unordered_write: 0 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.row_cache: None 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.wal_filter: None 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:03:05.630 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.wal_compression: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:03:05.631 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:03:05.632 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_open_files: -1 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Compression algorithms supported: 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kZSTD supported: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:03:05.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.merge_operator: 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_filter: None 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:03:05.634 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5594fe459380) 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: index_type: 0 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: data_block_index_type: 0 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: index_shortening: 1 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: checksum: 4 2024-08-17T17:03:05.635 INFO:journalctl@ceph.mon.a.smithi142.stdout: no_block_cache: 0 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache: 0x5594fe44d610 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache_options: 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: capacity : 536870912 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: num_shard_bits : 4 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: strict_capacity_limit : 0 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_cache_compressed: (nil) 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: persistent_cache: (nil) 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_size: 4096 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_size_deviation: 10 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_restart_interval: 16 2024-08-17T17:03:05.636 INFO:journalctl@ceph.mon.a.smithi142.stdout: index_block_restart_interval: 1 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: metadata_block_size: 4096 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: partition_filters: 0 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: use_delta_encoding: 1 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: filter_policy: bloomfilter 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: whole_key_filtering: 1 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: verify_compression: 0 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: format_version: 5 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: enable_index_compression: 1 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: block_align: 0 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: max_auto_readahead_size: 262144 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: prepopulate_block_cache: 0 2024-08-17T17:03:05.637 INFO:journalctl@ceph.mon.a.smithi142.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression: NoCompression 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.num_levels: 7 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:03:05.638 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:03:05.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:03:05.640 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:03:05.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:03:05.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.ttl: 2592000 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:03:05.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:03:05.644 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12161, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f63ca15d-6837-4794-bb7d-ef94a8156fe8 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914185180737, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914185185185, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 1161115, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12162, "largest_seqno": 12812, "table_properties": {"data_size": 1157160, "index_size": 2083, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 837, "raw_key_size": 7622, "raw_average_key_size": 23, "raw_value_size": 1150177, "raw_average_value_size": 3594, "num_data_blocks": 94, "num_entries": 320, "num_filter_entries": 320, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914185, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f63ca15d-6837-4794-bb7d-ef94a8156fe8", "db_session_id": "XPCMJVDU8CVTK3H4H2EU", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914185185293, "job": 1, "event": "recovery_finished"} 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:03:05.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5594fe47ce00 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: DB pointer 0x5594fe57a000 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** DB Stats ** 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.646 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: L0 1/0 1.11 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 438.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: L6 1/0 7.09 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: Sum 2/0 8.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 438.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 438.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 438.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-17T17:03:05.647 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Cumulative compaction: 0.00 GB write, 161.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Interval compaction: 0.00 GB write, 161.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Block cache BinnedLRUCache@0x5594fe44d610#7 capacity: 512.00 MB usage: 3.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-17T17:03:05.648 INFO:journalctl@ceph.mon.a.smithi142.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: starting mon.a rank 0 at public addrs [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] at bind addrs [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???) e3 preinit fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).mds e8 new map 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).mds e8 print_map 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: e8 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:03:05.649 INFO:journalctl@ceph.mon.a.smithi142.stdout: legacy client fscid: 1 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: Filesystem 'foo' (1) 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: fs_name foo 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: epoch 7 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: created 2024-08-17T16:57:28.380200+0000 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: modified 2024-08-17T16:57:46.044310+0000 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: tableserver 0 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: root 0 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: session_timeout 60 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: session_autoclose 300 2024-08-17T17:03:05.650 INFO:journalctl@ceph.mon.a.smithi142.stdout: max_file_size 1099511627776 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: max_xattr_size 65536 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: required_client_features {} 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: last_failure 0 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: last_failure_osd_epoch 0 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: max_mds 1 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: in 0 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: up {0=24542} 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: failed 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: damaged 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: stopped 2024-08-17T17:03:05.651 INFO:journalctl@ceph.mon.a.smithi142.stdout: data_pools [4] 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: metadata_pool 3 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: inline_data disabled 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: balancer 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: bal_rank_mask -1 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: standby_count_wanted 1 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: [mds.foo.smithi142.ndmoid{0:24542} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: Standby daemons: 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: 2024-08-17T17:03:05.652 INFO:journalctl@ceph.mon.a.smithi142.stdout: [mds.foo.smithi152.tacamo{-1:24493} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mon[103614]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 bash[103573]: 3a2a93ab0fbb9f60b0e84ff7e882aa4b32934a372b9d9a772f45dcbc4738eb53 2024-08-17T17:03:05.653 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:05 smithi142 systemd[1]: Started Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:03:05.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:05] ENGINE Bus STARTING 2024-08-17T17:03:06.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: ignoring --setuser ceph since I am not root 2024-08-17T17:03:06.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: ignoring --setgroup ceph since I am not root 2024-08-17T17:03:06.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:05 smithi152 ceph-mgr[84394]: -- 172.21.15.152:0/1164479562 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561688a030e0 con 0x5616889e1400 2024-08-17T17:03:06.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:06.074 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:06.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ignoring --setuser ceph since I am not root 2024-08-17T17:03:06.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ignoring --setgroup ceph since I am not root 2024-08-17T17:03:06.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:05 smithi142 ceph-mgr[93185]: -- 172.21.15.142:0/411416581 <== mon.0 v2:172.21.15.142:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555bf5e670e0 con 0x555bf5e44800 2024-08-17T17:03:06.825 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_1' created 2024-08-17T17:03:07.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='client.15612 172.21.15.142:0/2117857624' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-08-17T17:03:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:03:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:03:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4038916555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='client.25451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='client.25457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: monmap epoch 3 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:07.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: last_changed 2024-08-17T16:52:21.246059+0000 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: min_mon_release 17 (quincy) 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: election_strategy: 1 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: mgrmap e34: y(active, since 46s), standbys: x 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='client.15612 ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: mgrmap e35: y(active, since 47s), standbys: x 2024-08-17T17:03:07.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:06 smithi152 ceph-mon[34775]: from='mgr.15291 ' entity='mgr.y' 2024-08-17T17:03:07.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='client.15612 172.21.15.142:0/2117857624' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-08-17T17:03:07.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:03:07.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4038916555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='client.25451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='client.25457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: mon.a calling monitor election 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:03:07.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: monmap epoch 3 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: last_changed 2024-08-17T16:52:21.246059+0000 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: min_mon_release 17 (quincy) 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: election_strategy: 1 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:03:07.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: mgrmap e34: y(active, since 46s), standbys: x 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: overall HEALTH_OK 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='client.15612 ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: mgrmap e35: y(active, since 47s), standbys: x 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[40020]: from='mgr.15291 ' entity='mgr.y' 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='client.15612 172.21.15.142:0/2117857624' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:03:07.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4038916555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='client.25451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mgr.15291 172.21.15.142:0/2798237817' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='client.25457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: pgmap v27: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: mon.a calling monitor election 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: monmap epoch 3 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: last_changed 2024-08-17T16:52:21.246059+0000 2024-08-17T17:03:07.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: min_mon_release 17 (quincy) 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: election_strategy: 1 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: mgrmap e34: y(active, since 46s), standbys: x 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: overall HEALTH_OK 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='client.15612 ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: mgrmap e35: y(active, since 47s), standbys: x 2024-08-17T17:03:07.128 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:06 smithi142 ceph-mon[103614]: from='mgr.15291 ' entity='mgr.y' 2024-08-17T17:03:07.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-08-17T17:03:08.108 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:08.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:07 smithi142 ceph-mon[40020]: from='client.15612 ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:03:08.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:07 smithi142 ceph-mon[40020]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:03:08.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:07 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/98036691' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:08.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:07 smithi142 ceph-mon[103614]: from='client.15612 ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:03:08.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:07 smithi142 ceph-mon[103614]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:03:08.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/98036691' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:08.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:07 smithi152 ceph-mon[34775]: from='client.15612 ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:03:08.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:07 smithi152 ceph-mon[34775]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:03:08.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:07 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/98036691' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:09.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:08 smithi142 ceph-mon[103614]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:03:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:08 smithi142 ceph-mon[40020]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:03:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:08 smithi152 ceph-mon[34775]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:03:10.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:09 smithi142 ceph-mon[103614]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:03:10.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:09 smithi142 ceph-mon[40020]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:03:10.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:09 smithi152 ceph-mon[34775]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:03:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:10.852 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-08-17T17:03:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3590212737' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:03:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:03:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:03:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:03:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:03:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-mon[34775]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: [17/Aug/2024:17:03:10] ENGINE Bus STARTING 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: CherryPy Checker: 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: The Application mounted at '' has an empty config. 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: [17/Aug/2024:17:03:10] ENGINE Serving on http://:::9283 2024-08-17T17:03:11.074 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:03:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x[84372]: [17/Aug/2024:17:03:10] ENGINE Bus STARTED 2024-08-17T17:03:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3590212737' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: Standby manager daemon x restarted 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: Standby manager daemon x started 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[103614]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3590212737' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:03:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:03:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: Standby manager daemon x restarted 2024-08-17T17:03:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: Standby manager daemon x started 2024-08-17T17:03:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:03:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:03:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:03:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:10 smithi142 ceph-mon[40020]: from='mgr.? 172.21.15.152:0/216700952' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.410+0000 7f2570a90640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.410+0000 7f2570a90640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.410+0000 7f2570a90640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.410+0000 7f2570a90640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.410+0000 7f2570a90640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.413+0000 7f256da8a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.413+0000 7f256da8a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.413+0000 7f256da8a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.413+0000 7f256da8a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:11.413+0000 7f256da8a640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: mgrmap e36: y(active, since 52s), standbys: x 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: Active manager daemon y restarted 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: Activating manager daemon y 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: mgrmap e37: y(active, starting, since 0.00729707s), standbys: x 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:03:11.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:03:11.852 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:03:12.079 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:12.061+0000 7f25a1585640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-17T17:03:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:12] ENGINE Bus STARTING 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: CherryPy Checker: 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: The Application mounted at '' has an empty config. 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: mgrmap e36: y(active, since 52s), standbys: x 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: Active manager daemon y restarted 2024-08-17T17:03:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: Activating manager daemon y 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: mgrmap e37: y(active, starting, since 0.00729707s), standbys: x 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi152.tacamo"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:03:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: Manager daemon y is now available 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:03:12.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi152.tacamo"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:03:12.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: Manager daemon y is now available 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:03:12.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: mgrmap e36: y(active, since 52s), standbys: x 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: Active manager daemon y restarted 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: Activating manager daemon y 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: mgrmap e37: y(active, starting, since 0.00729707s), standbys: x 2024-08-17T17:03:12.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi152.tacamo"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi142.ndmoid"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:03:12.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: Manager daemon y is now available 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:03:12.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:03:12.459 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-08-17T17:03:12.462 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:03:12.462 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:03:12.507 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:03:12.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:03:12.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_104379 2024-08-17T17:03:12.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:03:12.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:03:12.567 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:12] ENGINE Serving on http://:::9283 2024-08-17T17:03:12.567 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:03:12] ENGINE Bus STARTED 2024-08-17T17:03:12.567 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:12.171+0000 7f255a8ea640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:03:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:03:12.567 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: Traceback (most recent call last): 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self._connections.run(self.expiration_interval) 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self._run(expiration_interval) 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: s = self.context.wrap_socket( 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:03:12.568 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: return self.sslsocket_class._create( 2024-08-17T17:03:12.569 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:03:12.569 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self.do_handshake() 2024-08-17T17:03:12.569 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:03:12.569 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: self._sslobj.do_handshake() 2024-08-17T17:03:12.569 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:03:12.569 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: [17/Aug/2024:17:03:12] ENGINE Bus STARTING 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: mgrmap e38: y(active, since 1.01251s), standbys: x 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: from='client.25264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: [17/Aug/2024:17:03:12] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: [17/Aug/2024:17:03:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: Traceback (most recent call last): 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:03:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._run(expiration_interval) 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: s = self.context.wrap_socket( 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: return self.sslsocket_class._create( 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:03:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: self.do_handshake() 2024-08-17T17:03:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:03:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: self._sslobj.do_handshake() 2024-08-17T17:03:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:03:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: [17/Aug/2024:17:03:12] ENGINE Serving on http://172.21.15.142:8765 2024-08-17T17:03:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: [17/Aug/2024:17:03:12] ENGINE Bus STARTED 2024-08-17T17:03:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:13 smithi152 ceph-mon[34775]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: [17/Aug/2024:17:03:12] ENGINE Bus STARTING 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: mgrmap e38: y(active, since 1.01251s), standbys: x 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: from='client.25264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: [17/Aug/2024:17:03:12] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: [17/Aug/2024:17:03:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: Traceback (most recent call last): 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:03:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._run(expiration_interval) 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:03:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout: self.do_handshake() 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: [17/Aug/2024:17:03:12] ENGINE Serving on http://172.21.15.142:8765 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: [17/Aug/2024:17:03:12] ENGINE Bus STARTED 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[40020]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: [17/Aug/2024:17:03:12] ENGINE Bus STARTING 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: mgrmap e38: y(active, since 1.01251s), standbys: x 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: from='client.25264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:13.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: [17/Aug/2024:17:03:12] ENGINE Serving on https://172.21.15.142:7150 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: [17/Aug/2024:17:03:12] ENGINE Error in HTTPServer.serve 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: Traceback (most recent call last): 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._run(expiration_interval) 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: s = self.context.wrap_socket( 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:03:13.376 INFO:journalctl@ceph.mon.a.smithi142.stdout: return self.sslsocket_class._create( 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout: self.do_handshake() 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout: self._sslobj.do_handshake() 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: [17/Aug/2024:17:03:12] ENGINE Serving on http://172.21.15.142:8765 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: [17/Aug/2024:17:03:12] ENGINE Bus STARTED 2024-08-17T17:03:13.377 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:13 smithi142 ceph-mon[103614]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:13.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 4073 4057 253.534 253.562 0.00425689 0.0039306 2024-08-17T17:03:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:14 smithi152 ceph-mon[34775]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:14 smithi152 ceph-mon[34775]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:03:14.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:14 smithi142 ceph-mon[103614]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:14 smithi142 ceph-mon[103614]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:03:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:14 smithi142 ceph-mon[40020]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:14 smithi142 ceph-mon[40020]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:03:14.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 7602 7586 237.039 220.562 0.00593516 0.00421 2024-08-17T17:03:15.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:15 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2452843749' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:15.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2452843749' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:15.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 10716 10700 222.895 194.625 0.0034633 0.00447687 2024-08-17T17:03:15.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:15 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2452843749' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:15.684 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:16.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:16 smithi142 ceph-mon[40020]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:16.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:16 smithi142 ceph-mon[40020]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:03:16.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:16 smithi142 ceph-mon[40020]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:16.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:16 smithi142 ceph-mon[103614]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:16.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:16 smithi142 ceph-mon[103614]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:03:16.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:16 smithi142 ceph-mon[103614]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:16.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:16 smithi152 ceph-mon[34775]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:16.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:16 smithi152 ceph-mon[34775]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:03:16.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:16 smithi152 ceph-mon[34775]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:16.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 13749 13733 214.557 189.562 0.0049348 0.00465422 2024-08-17T17:03:17.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:17.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 16300 16284 203.53 159.438 0.00583349 0.00490729 2024-08-17T17:03:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:17 smithi152 ceph-mon[34775]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:17 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:17 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:17 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:17 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[103614]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:18.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[40020]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:17 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 19074 19058 198.501 173.375 0.00349224 0.0050301 2024-08-17T17:03:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[103614]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[40020]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:18.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:18.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:18 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:18 smithi152 ceph-mon[34775]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:03:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:18 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:18 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:18 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:19.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:18 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:19.483 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:19.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 15 21685 21670 193.463 163.25 0.00540687 0.00516228 2024-08-17T17:03:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:19 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/4207598081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4207598081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:19.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:19 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/4207598081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:20.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 24387 24371 190.38 168.812 0.00675389 0.00524705 2024-08-17T17:03:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:21.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:20 smithi152 ceph-mon[34775]: pgmap v7: 89 pgs: 89 active+clean; 949 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 25 KiB/s rd, 136 MiB/s wr, 2.18k op/s 2024-08-17T17:03:21.012 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:20 smithi152 ceph-mon[34775]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:21.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:20 smithi142 ceph-mon[103614]: pgmap v7: 89 pgs: 89 active+clean; 949 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 25 KiB/s rd, 136 MiB/s wr, 2.18k op/s 2024-08-17T17:03:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:20 smithi142 ceph-mon[103614]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:20 smithi142 ceph-mon[40020]: pgmap v7: 89 pgs: 89 active+clean; 949 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 25 KiB/s rd, 136 MiB/s wr, 2.18k op/s 2024-08-17T17:03:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:20 smithi142 ceph-mon[40020]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:21.508 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 27036 27020 187.621 165.562 0.00665115 0.00532502 2024-08-17T17:03:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:21 smithi152 ceph-mon[34775]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:22.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:21 smithi142 ceph-mon[103614]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:22.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:03:21] "GET /metrics HTTP/1.1" 200 33513 "" "Prometheus/2.51.0" 2024-08-17T17:03:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:21 smithi142 ceph-mon[40020]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:22.509 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 29475 29459 184.101 152.438 0.00681297 0.00542554 2024-08-17T17:03:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:22 smithi152 ceph-mon[34775]: pgmap v8: 89 pgs: 89 active+clean; 949 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 105 MiB/s wr, 1.69k op/s 2024-08-17T17:03:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:22 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/303787634' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:22 smithi142 ceph-mon[40020]: pgmap v8: 89 pgs: 89 active+clean; 949 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 105 MiB/s wr, 1.69k op/s 2024-08-17T17:03:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:22 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/303787634' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:22 smithi142 ceph-mon[103614]: pgmap v8: 89 pgs: 89 active+clean; 949 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 105 MiB/s wr, 1.69k op/s 2024-08-17T17:03:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/303787634' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:03:23.256 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:23.509 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 31220 31204 177.278 109.062 0.00667261 0.00563482 2024-08-17T17:03:24.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:24 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: pgmap v9: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 16 KiB/s rd, 162 MiB/s wr, 2.60k op/s 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:03:24.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:23 smithi152 ceph-mon[34775]: from='client.25348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: pgmap v9: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 16 KiB/s rd, 162 MiB/s wr, 2.60k op/s 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:03:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[103614]: from='client.25348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.conf 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.conf 2024-08-17T17:03:24.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: pgmap v9: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 16 KiB/s rd, 162 MiB/s wr, 2.60k op/s 2024-08-17T17:03:24.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:24.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:03:24.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:03:24.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:03:24.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:23 smithi142 ceph-mon[40020]: from='client.25348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:24.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 33164 33148 172.616 121.5 0.00891996 0.00578789 2024-08-17T17:03:25.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 35270 35254 169.462 131.625 0.00518559 0.00589605 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:25.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:03:25.743 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:03:25.743 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: Updating smithi142:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: Updating smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:03:25.779 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: Updating smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:03:25.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-08-17T17:03:26.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:26.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:26.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 37189 37173 165.924 119.938 0.0051025 0.0060215 2024-08-17T17:03:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:26 smithi152 ceph-mon[34775]: pgmap v10: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 148 MiB/s wr, 2.38k op/s 2024-08-17T17:03:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:26 smithi152 ceph-mon[34775]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:03:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:26 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:26 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/2690458449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[103614]: pgmap v10: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 148 MiB/s wr, 2.38k op/s 2024-08-17T17:03:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[103614]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:03:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2690458449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[40020]: pgmap v10: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 15 KiB/s rd, 148 MiB/s wr, 2.38k op/s 2024-08-17T17:03:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[40020]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:03:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:26 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/2690458449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:27.000 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:27.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:27.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 39079 39063 162.737 118.125 0.0080989 0.0061393 2024-08-17T17:03:28.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 41143 41127 160.628 129 0.00475594 0.00622029 2024-08-17T17:03:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: from='client.25580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: pgmap v11: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 15 KiB/s rd, 180 MiB/s wr, 2.88k op/s 2024-08-17T17:03:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: Upgrade: Updating mon.c 2024-08-17T17:03:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:03:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:03:28.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:28.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: Deploying daemon mon.c on smithi142 2024-08-17T17:03:28.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:28 smithi152 ceph-mon[34775]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: from='client.25580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: pgmap v11: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 15 KiB/s rd, 180 MiB/s wr, 2.88k op/s 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: Upgrade: Updating mon.c 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: Deploying daemon mon.c on smithi142 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[103614]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:28.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: from='client.25580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: pgmap v11: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 15 KiB/s rd, 180 MiB/s wr, 2.88k op/s 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: Upgrade: Updating mon.c 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: Deploying daemon mon.c on smithi142 2024-08-17T17:03:28.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:28 smithi142 ceph-mon[40020]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:29.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 43079 43063 158.297 121 0.00623496 0.00631192 2024-08-17T17:03:30.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 44638 44622 154.915 97.4375 0.00830194 0.00644943 2024-08-17T17:03:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:30 smithi152 ceph-mon[34775]: pgmap v12: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 199 MiB/s wr, 3.19k op/s 2024-08-17T17:03:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:30 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3810064913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:30.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:30 smithi142 ceph-mon[40020]: pgmap v12: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 199 MiB/s wr, 3.19k op/s 2024-08-17T17:03:30.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:30 smithi142 ceph-mon[40020]: from='client.? 172.21.15.142:0/3810064913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:30.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:30 smithi142 ceph-mon[103614]: pgmap v12: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 15 KiB/s rd, 199 MiB/s wr, 3.19k op/s 2024-08-17T17:03:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3810064913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:30.669 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:31.485 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:31 smithi142 ceph-mon[103614]: from='client.25595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:31.485 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:31 smithi142 ceph-mon[40020]: from='client.25595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:31.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 46289 46273 152.192 103.188 0.00334014 0.00656489 2024-08-17T17:03:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:31 smithi152 ceph-mon[34775]: from='client.25595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:31 smithi142 systemd[1]: Stopping Ceph mon.c for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:03:32.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:03:31] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:03:32.485 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c[39997]: 2024-08-17T17:03:32.214+0000 7feae4fcf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:03:32.485 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c[39997]: 2024-08-17T17:03:32.214+0000 7feae4fcf640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:03:32.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:03:32.510590+0000 min lat: 0.00132253 max lat: 0.0965206 avg lat: 0.00665824 2024-08-17T17:03:32.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:03:32.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 48050 48034 150.085 110.062 0.0129444 0.00665824 2024-08-17T17:03:32.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:32 smithi142 podman[106490]: 2024-08-17 17:03:32.485075338 +0000 UTC m=+0.418442427 container died 24ff3a4330748f88b794dad982f1e769cb8e98ca76b572f74f86ff5b497d7f0c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-08-17T17:03:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:32 smithi142 podman[106490]: 2024-08-17 17:03:32.967414487 +0000 UTC m=+0.900781568 container cleanup 24ff3a4330748f88b794dad982f1e769cb8e98ca76b572f74f86ff5b497d7f0c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-17T17:03:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:32 smithi142 bash[106490]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:03:33.510 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 49799 49783 148.143 109.312 0.0211256 0.00674327 2024-08-17T17:03:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:33 smithi142 podman[106505]: 2024-08-17 17:03:33.362969411 +0000 UTC m=+0.871925291 container remove 24ff3a4330748f88b794dad982f1e769cb8e98ca76b572f74f86ff5b497d7f0c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0) 2024-08-17T17:03:34.002 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:33 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-17T17:03:34.003 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:33 smithi142 systemd[1]: Stopped Ceph mon.c for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:03:34.003 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:33 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.c.service: Consumed 8.168s CPU time. 2024-08-17T17:03:34.362 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 systemd[1]: Starting Ceph mon.c for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:03:34.399 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:34.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 51249 51233 145.529 90.625 0.0117667 0.00686501 2024-08-17T17:03:34.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 podman[106669]: 2024-08-17 17:03:34.265534148 +0000 UTC m=+0.044425895 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:03:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 podman[106669]: 2024-08-17 17:03:34.384811187 +0000 UTC m=+0.163702913 container create e6fccaa16a72a0c24bdb8755fd86cda1f001212bbf5c03e2838dbcc5ea2ee3ac (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 podman[106669]: 2024-08-17 17:03:34.642107472 +0000 UTC m=+0.420999216 container init e6fccaa16a72a0c24bdb8755fd86cda1f001212bbf5c03e2838dbcc5ea2ee3ac (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 podman[106669]: 2024-08-17 17:03:34.652434604 +0000 UTC m=+0.431326330 container start e6fccaa16a72a0c24bdb8755fd86cda1f001212bbf5c03e2838dbcc5ea2ee3ac (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: pidfile_write: ignore empty --pid-file 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: load: jerasure load: lrc 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Git sha 0 2024-08-17T17:03:35.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: DB SUMMARY 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: DB Session ID: Q3EQQSZTBNRP3VWN621W 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1104 Bytes 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 5184194 ; 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:03:35.127 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.env: 0x55698efcdbe0 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.info_log: 0x55698fa95700 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.statistics: (nil) 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.use_fsync: 0 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:03:35.128 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.db_log_dir: 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.wal_dir: 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:03:35.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.write_buffer_manager: 0x55698fa98960 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:03:35.130 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.unordered_write: 0 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.row_cache: None 2024-08-17T17:03:35.131 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.wal_filter: None 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.wal_compression: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:03:35.132 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:03:35.133 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_open_files: -1 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Compression algorithms supported: 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kZSTD supported: 0 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:03:35.134 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.merge_operator: 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_filter: None 2024-08-17T17:03:35.135 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55698fa95380) 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: index_type: 0 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: data_block_index_type: 0 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: index_shortening: 1 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:03:35.136 INFO:journalctl@ceph.mon.c.smithi142.stdout: checksum: 4 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: no_block_cache: 0 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache: 0x55698fa89610 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache_options: 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: capacity : 536870912 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: num_shard_bits : 4 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: strict_capacity_limit : 0 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_cache_compressed: (nil) 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: persistent_cache: (nil) 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_size: 4096 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_size_deviation: 10 2024-08-17T17:03:35.137 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_restart_interval: 16 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: index_block_restart_interval: 1 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: metadata_block_size: 4096 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: partition_filters: 0 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: use_delta_encoding: 1 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: filter_policy: bloomfilter 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: whole_key_filtering: 1 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: verify_compression: 0 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: format_version: 5 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: enable_index_compression: 1 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: block_align: 0 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: max_auto_readahead_size: 262144 2024-08-17T17:03:35.138 INFO:journalctl@ceph.mon.c.smithi142.stdout: prepopulate_block_cache: 0 2024-08-17T17:03:35.139 INFO:journalctl@ceph.mon.c.smithi142.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:03:35.139 INFO:journalctl@ceph.mon.c.smithi142.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:03:35.139 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:03:35.139 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:03:35.139 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression: NoCompression 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.num_levels: 7 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:03:35.140 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:03:35.141 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:03:35.142 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:03:35.143 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:03:35.144 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.ttl: 2592000 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:03:35.145 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12786, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-17T17:03:35.146 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5bf64931-1daf-4630-987d-62e7637cb82d 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914214705075, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914214726939, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3254785, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12791, "largest_seqno": 14157, "table_properties": {"data_size": 3248178, "index_size": 4026, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 14139, "raw_average_key_size": 23, "raw_value_size": 3235341, "raw_average_value_size": 5383, "num_data_blocks": 182, "num_entries": 601, "num_filter_entries": 601, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914214, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5bf64931-1daf-4630-987d-62e7637cb82d", "db_session_id": "Q3EQQSZTBNRP3VWN621W", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914214727151, "job": 1, "event": "recovery_finished"} 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55698fab8e00 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: DB pointer 0x55698fbb6000 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:03:35.147 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** DB Stats ** 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: L0 1/0 3.10 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 308.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:03:35.148 INFO:journalctl@ceph.mon.c.smithi142.stdout: L6 1/0 7.09 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: Sum 2/0 10.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 308.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 308.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** Compaction Stats [default] ** 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 308.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-08-17T17:03:35.149 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: Cumulative compaction: 0.00 GB write, 79.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: Interval compaction: 0.00 GB write, 79.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: Block cache BinnedLRUCache@0x55698fa89610#7 capacity: 512.00 MB usage: 61.28 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: Block cache entry stats(count,size,portion): DataBlock(4,30.31 KB,0.00578165%) FilterBlock(2,10.81 KB,0.00206232%) IndexBlock(2,20.16 KB,0.0038445%) Misc(1,0.00 KB,0%) 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:03:35.150 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: starting mon.c rank 2 at public addrs [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] at bind addrs [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???) e3 preinit fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).mds e8 new map 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).mds e8 print_map 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: e8 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: legacy client fscid: 1 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: Filesystem 'foo' (1) 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: fs_name foo 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: epoch 7 2024-08-17T17:03:35.151 INFO:journalctl@ceph.mon.c.smithi142.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: created 2024-08-17T16:57:28.380200+0000 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: modified 2024-08-17T16:57:46.044310+0000 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: tableserver 0 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: root 0 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: session_timeout 60 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: session_autoclose 300 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: max_file_size 1099511627776 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: max_xattr_size 65536 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: required_client_features {} 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: last_failure 0 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: last_failure_osd_epoch 0 2024-08-17T17:03:35.152 INFO:journalctl@ceph.mon.c.smithi142.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: max_mds 1 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: in 0 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: up {0=24542} 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: failed 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: damaged 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: stopped 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: data_pools [4] 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: metadata_pool 3 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: inline_data disabled 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: balancer 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: bal_rank_mask -1 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: standby_count_wanted 1 2024-08-17T17:03:35.153 INFO:journalctl@ceph.mon.c.smithi142.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout: [mds.foo.smithi142.ndmoid{0:24542} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout: Standby daemons: 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout: 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout: [mds.foo.smithi152.tacamo{-1:24493} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 ceph-mon[106729]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 bash[106669]: e6fccaa16a72a0c24bdb8755fd86cda1f001212bbf5c03e2838dbcc5ea2ee3ac 2024-08-17T17:03:35.154 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:34 smithi142 systemd[1]: Started Ceph mon.c for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:03:35.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 52958 52942 143.845 106.812 0.00652683 0.00694734 2024-08-17T17:03:36.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:35 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:36.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:35 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:36.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 54786 54770 142.611 114.25 0.00421994 0.00700693 2024-08-17T17:03:37.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:37.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 56538 56522 141.287 109.5 0.010214 0.00707236 2024-08-17T17:03:38.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 58288 58272 140.059 109.375 0.00862485 0.00713488 2024-08-17T17:03:39.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 60034 60018 138.913 109.125 0.00982023 0.00719396 2024-08-17T17:03:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:39 smithi152 ceph-mon[34775]: pgmap v13: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-08-17T17:03:40.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:39 smithi152 ceph-mon[34775]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:40.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:40.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:39 smithi142 ceph-mon[103614]: pgmap v13: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-08-17T17:03:40.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:39 smithi142 ceph-mon[103614]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:40.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 61517 61501 137.262 92.6875 0.00567858 0.00727905 2024-08-17T17:03:41.198 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: pgmap v13: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: mon.c calling monitor election 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: mon.b calling monitor election 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: mon.a calling monitor election 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: monmap epoch 3 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:41.224 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: last_changed 2024-08-17T16:52:21.246059+0000 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: min_mon_release 17 (quincy) 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: election_strategy: 1 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: mgrmap e40: y(active, since 29s), standbys: x 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[106729]: overall HEALTH_OK 2024-08-17T17:03:41.225 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: mon.c calling monitor election 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: mon.b calling monitor election 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: mon.a calling monitor election 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: monmap epoch 3 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: last_changed 2024-08-17T16:52:21.246059+0000 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: min_mon_release 17 (quincy) 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: election_strategy: 1 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:03:41.226 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:03:41.227 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:03:41.227 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: mgrmap e40: y(active, since 29s), standbys: x 2024-08-17T17:03:41.227 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:40 smithi142 ceph-mon[103614]: overall HEALTH_OK 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: mon.c calling monitor election 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: mon.b calling monitor election 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: monmap epoch 3 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: last_changed 2024-08-17T16:52:21.246059+0000 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:03:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: min_mon_release 17 (quincy) 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: election_strategy: 1 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: mgrmap e40: y(active, since 29s), standbys: x 2024-08-17T17:03:41.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:40 smithi152 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T17:03:41.511 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 62795 62779 135.283 79.875 0.013436 0.00738664 2024-08-17T17:03:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:03:41] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-08-17T17:03:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[106729]: pgmap v18: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:03:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[106729]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[106729]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[103614]: pgmap v18: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:03:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[103614]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:42.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:42.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:42 smithi142 ceph-mon[103614]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:42 smithi152 ceph-mon[34775]: pgmap v18: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:03:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:42 smithi152 ceph-mon[34775]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:42 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:42 smithi152 ceph-mon[34775]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:42.513 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 64006 63990 133.288 75.6875 0.00836326 0.0074972 2024-08-17T17:03:43.513 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 65293 65277 131.583 80.4375 0.00814226 0.00759434 2024-08-17T17:03:44.422 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:44 smithi142 ceph-mon[103614]: pgmap v19: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-17T17:03:44.422 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:44 smithi142 ceph-mon[106729]: pgmap v19: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-17T17:03:44.460 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:03:44.514 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 66665 66649 130.147 85.75 0.00416129 0.00767809 2024-08-17T17:03:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:44 smithi152 ceph-mon[34775]: pgmap v19: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-17T17:03:44.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:45.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:44 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:03:44.757+0000 7f25a1585640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-17T17:03:45.514 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 68072 68056 128.868 87.9375 0.00795504 0.00775483 2024-08-17T17:03:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:45 smithi152 ceph-mon[34775]: from='client.? 172.21.15.142:0/3624603407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:45 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:45 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:45.614 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3624603407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:45.614 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:45.614 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:45.615 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3624603407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:03:45.615 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:45.615 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:45.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:03:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:46.180 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:46.516 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 69306 69290 127.341 77.125 0.0173566 0.0078474 2024-08-17T17:03:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:46 smithi142 ceph-mon[106729]: pgmap v20: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-08-17T17:03:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:46 smithi142 ceph-mon[103614]: pgmap v20: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-08-17T17:03:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:46 smithi152 ceph-mon[34775]: pgmap v20: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-08-17T17:03:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:46 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:46 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:47.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:47.516 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 70658 70642 126.117 84.5 0.00709634 0.00792397 2024-08-17T17:03:48.299 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:47 smithi142 ceph-mon[106729]: pgmap v21: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:03:48.299 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:47 smithi142 ceph-mon[103614]: pgmap v21: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:03:48.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:47 smithi152 ceph-mon[34775]: pgmap v21: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:03:48.517 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 72069 72053 125.059 88.1875 0.00718102 0.00799068 2024-08-17T17:03:49.518 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 73403 73387 123.932 83.375 0.0166716 0.00806318 2024-08-17T17:03:49.693 INFO:teuthology.orchestra.run.smithi142.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:alertmanager.a smithi142 *:9093,9094 running (2m) 4s ago 7m 20.3M - 0.27.0 11f11916f8cd 95562b1fc088 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:grafana.a smithi152 *:3000 running (2m) 32s ago 7m 82.5M - 9.4.7 954c08fa6188 d463717746ae 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi142.ndmoid smithi142 running (6m) 4s ago 6m 16.8M - 17.2.7-1404-gf2c39d4d f69471c56189 bd857f554782 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi152.tacamo smithi152 running (6m) 32s ago 6m 15.7M - 17.2.7-1404-gf2c39d4d f69471c56189 8913efd6a27d 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.x smithi152 *:8443,9283,8765 running (62s) 32s ago 11m 450M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb bc654892e833 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.y smithi142 *:8443,9283,8765 running (104s) 4s ago 12m 506M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 323039997e22 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a smithi142 running (44s) 4s ago 12m 40.6M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 3a2a93ab0fbb 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b smithi152 running (11m) 32s ago 11m 52.2M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 b18785ced07f 2024-08-17T17:03:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c smithi142 running (15s) 4s ago 11m 24.6M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb e6fccaa16a72 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.a smithi142 *:9100 running (2m) 4s ago 6m 12.7M - 1.7.0 72c9c2088986 c6cd5b4c28e1 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.b smithi152 *:9100 running (2m) 32s ago 6m 10.4M - 1.7.0 72c9c2088986 8e3769ed942e 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 smithi142 running (10m) 4s ago 10m 324M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 cb819443ee72 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 smithi142 running (10m) 4s ago 10m 241M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 5cd1d20f8eca 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 smithi142 running (10m) 4s ago 10m 315M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 b599f0d55518 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 smithi142 running (9m) 4s ago 9m 277M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 d53780492c5b 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 smithi152 running (9m) 32s ago 9m 288M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 7a1cab3a1049 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 smithi152 running (9m) 32s ago 9m 313M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 cac09b3f3a95 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 smithi152 running (8m) 32s ago 8m 167M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 81327b9b9e78 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 smithi152 running (8m) 32s ago 8m 211M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 0a45f0507de1 2024-08-17T17:03:49.695 INFO:teuthology.orchestra.run.smithi142.stdout:prometheus.a smithi152 *:9095 running (70s) 32s ago 7m 48.0M - 2.51.0 1d3b7f56885b f12943011a9f 2024-08-17T17:03:50.517 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 74763 74747 122.908 85 0.00971887 0.00813108 2024-08-17T17:03:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:50 smithi152 ceph-mon[34775]: pgmap v22: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:03:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:50 smithi152 ceph-mon[34775]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:50 smithi142 ceph-mon[106729]: pgmap v22: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:03:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:50 smithi142 ceph-mon[106729]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:50 smithi142 ceph-mon[103614]: pgmap v22: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:03:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:50 smithi142 ceph-mon[103614]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:51.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:03:51.518 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 76137 76121 121.956 85.875 0.00764608 0.00819368 2024-08-17T17:03:51.809 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:03:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:03:51] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-08-17T17:03:52.230 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:52.516 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:52 smithi142 ceph-mon[106729]: pgmap v23: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:03:52.516 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:52 smithi142 ceph-mon[103614]: pgmap v23: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:03:52.518 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:03:52.518477+0000 min lat: 0.00132253 max lat: 0.0965206 avg lat: 0.00827354 2024-08-17T17:03:52.518 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:03:52.518 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 77339 77323 120.785 75.125 0.00555472 0.00827354 2024-08-17T17:03:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:52 smithi152 ceph-mon[34775]: pgmap v23: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:03:53.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 78613 78597 119.781 79.625 0.0165989 0.00834367 2024-08-17T17:03:54.465 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:54 smithi142 ceph-mon[103614]: pgmap v24: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-08-17T17:03:54.465 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:54 smithi142 ceph-mon[106729]: pgmap v24: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-08-17T17:03:54.518 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 79817 79801 118.721 75.25 0.00909179 0.00841822 2024-08-17T17:03:54.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:54 smithi152 ceph-mon[34775]: pgmap v24: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-08-17T17:03:54.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:54.804 INFO:teuthology.orchestra.run.smithi142.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:03:55.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 81018 81002 117.705 75.0625 0.0296909 0.00849044 2024-08-17T17:03:55.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:03:56.044 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:03:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:55 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:03:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:56.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:03:56.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 82255 82239 116.787 77.3125 0.0107561 0.00855736 2024-08-17T17:03:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:56 smithi142 ceph-mon[103614]: pgmap v25: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:03:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:56 smithi142 ceph-mon[106729]: pgmap v25: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:03:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:56 smithi152 ceph-mon[34775]: pgmap v25: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:03:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:56 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:57.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:03:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:03:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:03:57.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 83513 83497 115.939 78.625 0.0108515 0.0086202 2024-08-17T17:03:58.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 84715 84699 115.052 75.125 0.0196922 0.00868637 2024-08-17T17:03:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:03:58 smithi152 ceph-mon[34775]: pgmap v26: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:03:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:03:58 smithi142 ceph-mon[103614]: pgmap v26: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:03:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:03:58 smithi142 ceph-mon[106729]: pgmap v26: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:03:59.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 85907 85891 114.189 74.5 0.0108532 0.00875213 2024-08-17T17:04:00.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 87179 87163 113.466 79.5 0.00838114 0.00880773 2024-08-17T17:04:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:00 smithi152 ceph-mon[34775]: pgmap v27: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:04:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:00 smithi142 ceph-mon[103614]: pgmap v27: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:04:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:00 smithi142 ceph-mon[106729]: pgmap v27: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:04:01.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 88471 88455 112.799 80.75 0.013367 0.00886063 2024-08-17T17:04:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:04:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:04:01] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:04:02.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 89722 89706 112.106 78.1875 0.0205027 0.00891533 2024-08-17T17:04:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:02 smithi152 ceph-mon[34775]: pgmap v28: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:04:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:02 smithi142 ceph-mon[103614]: pgmap v28: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:04:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:02 smithi142 ceph-mon[106729]: pgmap v28: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:04:03.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 90971 90955 111.439 78.0625 0.00521804 0.00896839 2024-08-17T17:04:04.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 92201 92185 110.774 76.875 0.0138726 0.00902222 2024-08-17T17:04:04.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:04 smithi152 ceph-mon[34775]: pgmap v29: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T17:04:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:04 smithi142 ceph-mon[103614]: pgmap v29: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T17:04:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:04 smithi142 ceph-mon[106729]: pgmap v29: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-08-17T17:04:05.519 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 93444 93428 110.15 77.6875 0.0120239 0.00907348 2024-08-17T17:04:06.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:06.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:06.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 94658 94642 109.515 75.875 0.00748857 0.00912559 2024-08-17T17:04:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:06 smithi142 ceph-mon[103614]: pgmap v30: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:04:07.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:06 smithi142 ceph-mon[106729]: pgmap v30: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:04:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:06 smithi152 ceph-mon[34775]: pgmap v30: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:04:07.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:06.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:07.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 95873 95857 108.905 75.9375 0.0309988 0.00917697 2024-08-17T17:04:08.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 97051 97035 108.274 73.625 0.0151765 0.00923021 2024-08-17T17:04:09.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:08 smithi142 ceph-mon[103614]: pgmap v31: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:04:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:08 smithi142 ceph-mon[106729]: pgmap v31: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:04:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:08 smithi152 ceph-mon[34775]: pgmap v31: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:04:09.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 98232 98216 107.67 73.8125 0.0286254 0.00928247 2024-08-17T17:04:10.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 99360 99344 107.029 70.5 0.0283213 0.00933788 2024-08-17T17:04:10.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:11.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:10 smithi142 ceph-mon[106729]: pgmap v32: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:04:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:10 smithi142 ceph-mon[103614]: pgmap v32: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:04:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:10 smithi152 ceph-mon[34775]: pgmap v32: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:04:11.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 15 100515 100500 106.439 72.25 0.0129963 0.00938978 2024-08-17T17:04:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:04:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:04:11] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-08-17T17:04:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:11 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:12.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:04:12.520335+0000 min lat: 0.00132253 max lat: 0.0965206 avg lat: 0.00944638 2024-08-17T17:04:12.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:04:12.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 101602 101586 105.797 67.875 0.0143187 0.00944638 2024-08-17T17:04:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:12 smithi142 ceph-mon[106729]: pgmap v33: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:04:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:12 smithi142 ceph-mon[103614]: pgmap v33: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:04:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:12 smithi152 ceph-mon[34775]: pgmap v33: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:04:13.520 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 102587 102571 105.072 61.5625 0.0185062 0.00951161 2024-08-17T17:04:14.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 103644 103628 104.442 66.0625 0.0273651 0.00956961 2024-08-17T17:04:14.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:14 smithi142 ceph-mon[103614]: pgmap v34: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:04:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:14 smithi142 ceph-mon[106729]: pgmap v34: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:04:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:14 smithi152 ceph-mon[34775]: pgmap v34: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:04:15.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 104696 104680 103.828 65.75 0.014175 0.00962604 2024-08-17T17:04:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:15.743Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:15.744Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:16.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 105804 105788 103.288 69.25 0.0119923 0.00967533 2024-08-17T17:04:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:16 smithi142 ceph-mon[103614]: pgmap v35: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-08-17T17:04:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:16 smithi142 ceph-mon[106729]: pgmap v35: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-08-17T17:04:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:16 smithi152 ceph-mon[34775]: pgmap v35: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-08-17T17:04:17.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:17.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 106920 106904 102.772 69.75 0.00730512 0.00972497 2024-08-17T17:04:18.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 108044 108028 102.279 70.25 0.014438 0.00977127 2024-08-17T17:04:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:18 smithi142 ceph-mon[106729]: pgmap v36: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:04:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:18 smithi142 ceph-mon[103614]: pgmap v36: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:04:19.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:18 smithi152 ceph-mon[34775]: pgmap v36: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:04:19.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 109154 109138 101.788 69.375 0.0105935 0.00981911 2024-08-17T17:04:20.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 110240 110224 101.289 67.875 0.021555 0.00986767 2024-08-17T17:04:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:20 smithi142 ceph-mon[106729]: pgmap v37: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:04:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:20 smithi142 ceph-mon[103614]: pgmap v37: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:04:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:20 smithi152 ceph-mon[34775]: pgmap v37: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:04:21.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 111320 111304 100.799 67.5 0.0140616 0.0099157 2024-08-17T17:04:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:04:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:04:21] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-08-17T17:04:22.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 112373 112357 100.3 65.8125 0.0175466 0.009965 2024-08-17T17:04:23.125 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:22 smithi152 ceph-mon[34775]: pgmap v38: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2024-08-17T17:04:23.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:22 smithi142 ceph-mon[103614]: pgmap v38: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2024-08-17T17:04:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:22 smithi142 ceph-mon[106729]: pgmap v38: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2024-08-17T17:04:23.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 113427 113411 99.8146 65.875 0.00696689 0.0100131 2024-08-17T17:04:24.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 114458 114442 99.3232 64.4375 0.0191656 0.0100631 2024-08-17T17:04:24.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:24 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:24 smithi152 ceph-mon[34775]: pgmap v39: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-08-17T17:04:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:24 smithi142 ceph-mon[106729]: pgmap v39: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-08-17T17:04:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:24 smithi142 ceph-mon[103614]: pgmap v39: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-08-17T17:04:25.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 115517 115501 98.8693 66.1875 0.0106371 0.0101097 2024-08-17T17:04:26.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:26.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:25.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:26.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 116556 116540 98.4107 64.9375 0.0118506 0.0101563 2024-08-17T17:04:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:26 smithi152 ceph-mon[34775]: pgmap v40: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:04:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:26 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:27.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:26 smithi142 ceph-mon[106729]: pgmap v40: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:04:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:27.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:26 smithi142 ceph-mon[103614]: pgmap v40: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:04:27.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:27.521 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 117597 117581 97.966 65.0625 0.0134884 0.0102022 2024-08-17T17:04:28.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 118620 118604 97.5183 63.9375 0.0133863 0.0102491 2024-08-17T17:04:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:28 smithi152 ceph-mon[34775]: pgmap v41: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-08-17T17:04:29.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:28 smithi142 ceph-mon[103614]: pgmap v41: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-08-17T17:04:29.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:28 smithi142 ceph-mon[106729]: pgmap v41: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-08-17T17:04:29.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 119630 119614 97.0716 63.125 0.00541782 0.0102954 2024-08-17T17:04:30.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 120633 120617 96.6307 62.6875 0.0138724 0.0103436 2024-08-17T17:04:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:30 smithi152 ceph-mon[34775]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:04:31.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:30 smithi142 ceph-mon[103614]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:04:31.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:30 smithi142 ceph-mon[106729]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:04:31.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 121626 121610 96.1931 62.0625 0.00726161 0.0103906 2024-08-17T17:04:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:31 smithi142 ceph-mon[103614]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-08-17T17:04:32.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:04:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:04:31] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:04:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:31 smithi142 ceph-mon[106729]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-08-17T17:04:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:31 smithi152 ceph-mon[34775]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-08-17T17:04:32.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:04:32.522130+0000 min lat: 0.00132253 max lat: 0.0965206 avg lat: 0.0104374 2024-08-17T17:04:32.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:04:32.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 122617 122601 95.7649 61.9375 0.0124707 0.0104374 2024-08-17T17:04:33.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 123618 123602 95.3549 62.5625 0.0192055 0.0104818 2024-08-17T17:04:34.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 124612 124596 94.9497 62.125 0.00761464 0.0105268 2024-08-17T17:04:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:34 smithi152 ceph-mon[34775]: pgmap v44: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:04:34.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:34.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:34 smithi142 ceph-mon[103614]: pgmap v44: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:04:34.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:34 smithi142 ceph-mon[106729]: pgmap v44: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:04:35.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 125554 125538 94.515 58.875 0.0252177 0.010575 2024-08-17T17:04:36.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:35 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:36.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:35 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:36.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 126465 126449 94.0676 56.9375 0.00730837 0.0106255 2024-08-17T17:04:36.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:36 smithi152 ceph-mon[34775]: pgmap v45: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-08-17T17:04:36.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:36 smithi142 ceph-mon[103614]: pgmap v45: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-08-17T17:04:36.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:36 smithi142 ceph-mon[106729]: pgmap v45: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-08-17T17:04:37.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:37.522 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 127419 127403 93.6624 59.625 0.00930034 0.0106717 2024-08-17T17:04:38.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 128397 128381 93.284 61.125 0.0195189 0.0107151 2024-08-17T17:04:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:38 smithi152 ceph-mon[34775]: pgmap v46: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-08-17T17:04:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:38 smithi142 ceph-mon[106729]: pgmap v46: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-08-17T17:04:38.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:38 smithi142 ceph-mon[103614]: pgmap v46: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2024-08-17T17:04:39.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 129364 129348 92.9065 60.4375 0.050413 0.0107576 2024-08-17T17:04:40.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 130316 130300 92.5268 59.5 0.0216812 0.0108019 2024-08-17T17:04:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:40 smithi152 ceph-mon[34775]: pgmap v47: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T17:04:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:40.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:40 smithi142 ceph-mon[103614]: pgmap v47: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T17:04:40.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:40 smithi142 ceph-mon[106729]: pgmap v47: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T17:04:41.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 131263 131247 92.1522 59.1875 0.00933983 0.0108464 2024-08-17T17:04:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:04:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:04:41] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:04:42.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0093 2024-08-17T17:04:42.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 132187 2024-08-17T17:04:42.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:04:42.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:04:42.523 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 91.7871 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 38.8292 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 253.562 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 56.9375 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1468 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 621.267 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 4057 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 911 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0108904 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.00706638 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 0.101151 2024-08-17T17:04:42.524 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00132253 2024-08-17T17:04:42.529 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:04:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:42 smithi152 ceph-mon[34775]: pgmap v48: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-08-17T17:04:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:42 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:42 smithi142 ceph-mon[103614]: pgmap v48: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-08-17T17:04:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:42 smithi142 ceph-mon[106729]: pgmap v48: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-08-17T17:04:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:44 smithi152 ceph-mon[34775]: pgmap v49: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:04:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:44 smithi152 ceph-mon[34775]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:04:44.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:44.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:44 smithi142 ceph-mon[106729]: pgmap v49: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:04:44.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:44 smithi142 ceph-mon[106729]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:04:44.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:44 smithi142 ceph-mon[103614]: pgmap v49: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:04:44.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:44 smithi142 ceph-mon[103614]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:04:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:46 smithi152 ceph-mon[34775]: pgmap v50: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-08-17T17:04:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:46 smithi142 ceph-mon[106729]: pgmap v50: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-08-17T17:04:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:46 smithi142 ceph-mon[103614]: pgmap v50: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-08-17T17:04:47.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:48.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:48 smithi152 ceph-mon[34775]: pgmap v51: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-08-17T17:04:48.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:48 smithi142 ceph-mon[106729]: pgmap v51: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-08-17T17:04:48.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:48 smithi142 ceph-mon[103614]: pgmap v51: 89 pgs: 89 active+clean; 8.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s 2024-08-17T17:04:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:50 smithi152 ceph-mon[34775]: pgmap v52: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 1.21k op/s 2024-08-17T17:04:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:50 smithi142 ceph-mon[106729]: pgmap v52: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 1.21k op/s 2024-08-17T17:04:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:50 smithi142 ceph-mon[103614]: pgmap v52: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 1.21k op/s 2024-08-17T17:04:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:04:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:04:51] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:04:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:52 smithi152 ceph-mon[34775]: pgmap v53: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 809 op/s 2024-08-17T17:04:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:52 smithi142 ceph-mon[106729]: pgmap v53: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 809 op/s 2024-08-17T17:04:52.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:52 smithi142 ceph-mon[103614]: pgmap v53: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 809 op/s 2024-08-17T17:04:54.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:54 smithi152 ceph-mon[34775]: pgmap v54: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.21k op/s 2024-08-17T17:04:54.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:54.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:54 smithi142 ceph-mon[106729]: pgmap v54: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.21k op/s 2024-08-17T17:04:54.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:54 smithi142 ceph-mon[103614]: pgmap v54: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.21k op/s 2024-08-17T17:04:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:56 smithi152 ceph-mon[34775]: pgmap v55: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 812 op/s 2024-08-17T17:04:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:56 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:56 smithi142 ceph-mon[103614]: pgmap v55: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 812 op/s 2024-08-17T17:04:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:56 smithi142 ceph-mon[106729]: pgmap v55: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 812 op/s 2024-08-17T17:04:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:57.236 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:57 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:57.236 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:04:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:04:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:04:57.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:57.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:04:58 smithi152 ceph-mon[34775]: pgmap v56: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 860 op/s 2024-08-17T17:04:58.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:04:58 smithi142 ceph-mon[106729]: pgmap v56: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 860 op/s 2024-08-17T17:04:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:04:58 smithi142 ceph-mon[103614]: pgmap v56: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 860 op/s 2024-08-17T17:05:00.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:00 smithi152 ceph-mon[34775]: pgmap v57: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 1.20k op/s 2024-08-17T17:05:00.464 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:00.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:00 smithi142 ceph-mon[106729]: pgmap v57: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 1.20k op/s 2024-08-17T17:05:00.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:00 smithi142 ceph-mon[103614]: pgmap v57: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 1.8 MiB/s wr, 1.20k op/s 2024-08-17T17:05:01.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade resume' 2024-08-17T17:05:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:05:01] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:05:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:01 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:01 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:01 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:02.332 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:03.160 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[106729]: pgmap v58: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:05:03.160 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:03.160 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[103614]: pgmap v58: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:03.161 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:02 smithi152 ceph-mon[34775]: pgmap v58: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:05:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:02 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:02 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:02 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:02 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:04.260 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:03 smithi142 ceph-mon[103614]: pgmap v59: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-17T17:05:04.260 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:03 smithi142 ceph-mon[106729]: pgmap v59: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-17T17:05:04.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:03 smithi152 ceph-mon[34775]: pgmap v59: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-17T17:05:04.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:05.342 INFO:teuthology.orchestra.run.smithi142.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:06.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for all mons to upgrade' 2024-08-17T17:05:06.066 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:06.066 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:06.515 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: pgmap v60: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-08-17T17:05:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[103614]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:05:06.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: pgmap v60: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-08-17T17:05:06.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:06 smithi142 ceph-mon[106729]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: pgmap v60: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:06.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:06.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:06.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-08-17T17:05:06.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-08-17T17:05:06.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-mon[34775]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:05:07.163 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:08.529 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[103614]: pgmap v61: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-08-17T17:05:08.529 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:08.529 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:08.529 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:08.529 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:08.530 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[106729]: pgmap v61: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-08-17T17:05:08.530 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:08.530 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:08.530 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:08.530 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:08.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:08 smithi152 ceph-mon[34775]: pgmap v61: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2024-08-17T17:05:08.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:08 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:08.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:08 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:08.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:08 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:08.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:08 smithi152 ceph-mon[34775]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:08.636 INFO:teuthology.orchestra.run.smithi142.stdout:wait for all mons to upgrade 2024-08-17T17:05:09.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:09 smithi152 ceph-mon[34775]: Upgrade: Updating mon.b 2024-08-17T17:05:09.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:09 smithi152 ceph-mon[34775]: Deploying daemon mon.b on smithi152 2024-08-17T17:05:09.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:09 smithi142 ceph-mon[106729]: Upgrade: Updating mon.b 2024-08-17T17:05:09.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:09 smithi142 ceph-mon[106729]: Deploying daemon mon.b on smithi152 2024-08-17T17:05:09.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:09 smithi142 ceph-mon[103614]: Upgrade: Updating mon.b 2024-08-17T17:05:09.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:09 smithi142 ceph-mon[103614]: Deploying daemon mon.b on smithi152 2024-08-17T17:05:10.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:05:10.525 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:10 smithi142 ceph-mon[103614]: pgmap v62: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:05:10.526 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:10 smithi142 ceph-mon[106729]: pgmap v62: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:05:10.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:10 smithi152 ceph-mon[34775]: pgmap v62: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:05:10.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:10 smithi152 systemd[1]: Stopping Ceph mon.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:05:10.547 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:10.617 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:11.255 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b[34738]: 2024-08-17T17:05:10.909+0000 7f9d9b49b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:05:11.255 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b[34738]: 2024-08-17T17:05:10.909+0000 7f9d9b49b640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:05:11.255 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:10 smithi152 podman[93274]: 2024-08-17 17:05:10.945913739 +0000 UTC m=+0.172045593 container died b18785ced07f4e9da13d1070fd0a2ba2eb144bef9fdcc82b1d8371f06cf7b489 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:05:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:11 smithi152 podman[93274]: 2024-08-17 17:05:11.359184329 +0000 UTC m=+0.585316181 container cleanup b18785ced07f4e9da13d1070fd0a2ba2eb144bef9fdcc82b1d8371f06cf7b489 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4) 2024-08-17T17:05:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:11 smithi152 bash[93274]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:05:11.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:05:11] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:05:11.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:11 smithi152 podman[93292]: 2024-08-17 17:05:11.594708484 +0000 UTC m=+0.651477185 container remove b18785ced07f4e9da13d1070fd0a2ba2eb144bef9fdcc82b1d8371f06cf7b489 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4) 2024-08-17T17:05:12.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:11 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-17T17:05:12.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:11 smithi152 systemd[1]: Stopped Ceph mon.b for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:05:12.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:11 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.b.service: Consumed 10.597s CPU time. 2024-08-17T17:05:12.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 systemd[1]: Starting Ceph mon.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:05:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 podman[93419]: 2024-08-17 17:05:12.416308059 +0000 UTC m=+0.026874743 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 podman[93419]: 2024-08-17 17:05:12.532992988 +0000 UTC m=+0.143559671 container create a690c34b9332eafa283ff841aa10628fe536e19f9e81d2879184cac28b61e574 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T17:05:12.885 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 podman[93419]: 2024-08-17 17:05:12.841352751 +0000 UTC m=+0.451919429 container init a690c34b9332eafa283ff841aa10628fe536e19f9e81d2879184cac28b61e574 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 podman[93419]: 2024-08-17 17:05:12.84715261 +0000 UTC m=+0.457719293 container start a690c34b9332eafa283ff841aa10628fe536e19f9e81d2879184cac28b61e574 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: pidfile_write: ignore empty --pid-file 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: load: jerasure load: lrc 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Git sha 0 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: DB SUMMARY 2024-08-17T17:05:12.886 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: DB Session ID: V819PEXTHTF8W2DZOCTQ 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1104 Bytes 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 7108411 ; 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.env: 0x562a106b2be0 2024-08-17T17:05:12.887 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:05:12.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.info_log: 0x562a11109700 2024-08-17T17:05:12.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:05:12.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.statistics: (nil) 2024-08-17T17:05:12.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.use_fsync: 0 2024-08-17T17:05:12.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:05:12.888 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.db_log_dir: 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.wal_dir: 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:05:12.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.write_buffer_manager: 0x562a1110c960 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:05:12.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:05:12.893 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:05:12.893 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:05:12.893 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.unordered_write: 0 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.row_cache: None 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.wal_filter: None 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:05:12.894 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.wal_compression: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:05:12.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:05:12.896 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_open_files: -1 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Compression algorithms supported: 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kZSTD supported: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:05:12.897 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:05:12.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:05:12.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:05:12.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:05:12.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:05:12.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:05:12.898 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:05:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-08-17T17:05:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:05:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:05:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.merge_operator: 2024-08-17T17:05:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_filter: None 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562a11109380) 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_type: 0 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_block_index_type: 0 2024-08-17T17:05:13.324 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_shortening: 1 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: checksum: 4 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: no_block_cache: 0 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache: 0x562a110fd610 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_options: 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: capacity : 536870912 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: num_shard_bits : 4 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: strict_capacity_limit : 0 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_compressed: (nil) 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: persistent_cache: (nil) 2024-08-17T17:05:13.325 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_size: 4096 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_size_deviation: 10 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_restart_interval: 16 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_block_restart_interval: 1 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: metadata_block_size: 4096 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: partition_filters: 0 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: use_delta_encoding: 1 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: filter_policy: bloomfilter 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: whole_key_filtering: 1 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: verify_compression: 0 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: format_version: 5 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: enable_index_compression: 1 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_align: 0 2024-08-17T17:05:13.326 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_auto_readahead_size: 262144 2024-08-17T17:05:13.327 INFO:journalctl@ceph.mon.b.smithi152.stdout: prepopulate_block_cache: 0 2024-08-17T17:05:13.327 INFO:journalctl@ceph.mon.b.smithi152.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:05:13.327 INFO:journalctl@ceph.mon.b.smithi152.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:05:13.327 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:05:13.327 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:05:13.327 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression: NoCompression 2024-08-17T17:05:13.328 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:05:13.328 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:05:13.328 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:05:13.328 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.num_levels: 7 2024-08-17T17:05:13.328 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:05:13.328 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:05:13.329 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:05:13.330 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:05:13.331 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:05:13.332 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.ttl: 2592000 2024-08-17T17:05:13.333 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:05:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12863, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d74ccb31-a67a-435e-b5c6-d7e466b146f8 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914312883368, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914312907662, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4436550, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12868, "largest_seqno": 15562, "table_properties": {"data_size": 4425592, "index_size": 6839, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3077, "raw_key_size": 26137, "raw_average_key_size": 21, "raw_value_size": 4401720, "raw_average_value_size": 3622, "num_data_blocks": 309, "num_entries": 1215, "num_filter_entries": 1215, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914312, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d74ccb31-a67a-435e-b5c6-d7e466b146f8", "db_session_id": "V819PEXTHTF8W2DZOCTQ", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914312907889, "job": 1, "event": "recovery_finished"} 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562a1112ce00 2024-08-17T17:05:13.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: DB pointer 0x562a1122a000 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** DB Stats ** 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** Compaction Stats [default] ** 2024-08-17T17:05:13.336 INFO:journalctl@ceph.mon.b.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: L0 1/0 4.23 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 411.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: L6 1/0 7.09 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: Sum 2/0 11.32 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 411.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 411.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** Compaction Stats [default] ** 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 411.9 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.337 INFO:journalctl@ceph.mon.b.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative compaction: 0.00 GB write, 144.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval compaction: 0.00 GB write, 144.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: Block cache BinnedLRUCache@0x562a110fd610#7 capacity: 512.00 MB usage: 11.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,3.28 KB,0.000625849%) IndexBlock(1,8.08 KB,0.00154078%) Misc(1,0.00 KB,0%) 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:05:13.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: starting mon.b rank 1 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???) e3 preinit fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).mds e8 new map 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).mds e8 print_map 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: e8 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: legacy client fscid: 1 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: Filesystem 'foo' (1) 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: fs_name foo 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: epoch 7 2024-08-17T17:05:13.339 INFO:journalctl@ceph.mon.b.smithi152.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: created 2024-08-17T16:57:28.380200+0000 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: modified 2024-08-17T16:57:46.044310+0000 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: tableserver 0 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: root 0 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: session_timeout 60 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: session_autoclose 300 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_file_size 1099511627776 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_xattr_size 65536 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: required_client_features {} 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: last_failure 0 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: last_failure_osd_epoch 0 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_mds 1 2024-08-17T17:05:13.340 INFO:journalctl@ceph.mon.b.smithi152.stdout: in 0 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: up {0=24542} 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: failed 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: damaged 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: stopped 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_pools [4] 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: metadata_pool 3 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: inline_data disabled 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: balancer 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: bal_rank_mask -1 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: standby_count_wanted 1 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: [mds.foo.smithi142.ndmoid{0:24542} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.142:6834/2273628262,v1:172.21.15.142:6835/2273628262] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:05:13.341 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout: Standby daemons: 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout: [mds.foo.smithi152.tacamo{-1:24493} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.152:6832/755466509,v1:172.21.15.152:6833/755466509] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 ceph-mon[93456]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 bash[93419]: a690c34b9332eafa283ff841aa10628fe536e19f9e81d2879184cac28b61e574 2024-08-17T17:05:13.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:12 smithi152 systemd[1]: Started Ceph mon.b for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:05:13.665 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:05:14.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:15.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:17.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:18.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:05:17.929+0000 7f25a1585640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-17T17:05:20.052 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:05:20.556 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:20.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mon.b calling monitor election 2024-08-17T17:05:20.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mon.c calling monitor election 2024-08-17T17:05:20.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: pgmap v67: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mon.b calling monitor election 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: monmap epoch 4 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: last_changed 2024-08-17T17:05:14.388445+0000 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: min_mon_release 19 (squid) 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: election_strategy: 1 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:05:20.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: overall HEALTH_OK 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mon.a calling monitor election 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: monmap epoch 4 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: last_changed 2024-08-17T17:05:14.388445+0000 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: min_mon_release 19 (squid) 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: election_strategy: 1 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:05:20.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: overall HEALTH_OK 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:20.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4266025197' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:05:20.699 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mon.b calling monitor election 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mon.c calling monitor election 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: pgmap v67: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mon.b calling monitor election 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: monmap epoch 4 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: last_changed 2024-08-17T17:05:14.388445+0000 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: min_mon_release 19 (squid) 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: election_strategy: 1 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:05:20.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: overall HEALTH_OK 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mon.a calling monitor election 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: monmap epoch 4 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: last_changed 2024-08-17T17:05:14.388445+0000 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: min_mon_release 19 (squid) 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: election_strategy: 1 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:05:20.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: overall HEALTH_OK 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:20.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4266025197' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mon.b calling monitor election 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mon.c calling monitor election 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: pgmap v67: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mon.b calling monitor election 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: monmap epoch 4 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: last_changed 2024-08-17T17:05:14.388445+0000 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: min_mon_release 19 (squid) 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: election_strategy: 1 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:05:20.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: overall HEALTH_OK 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mon.a calling monitor election 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: monmap epoch 4 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: last_changed 2024-08-17T17:05:14.388445+0000 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: created 2024-08-17T16:50:49.283901+0000 2024-08-17T17:05:20.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: min_mon_release 19 (squid) 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: election_strategy: 1 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: 0: [v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] mon.a 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: 1: [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon.b 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: 2: [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0] mon.c 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: fsmap foo:1 {0=foo.smithi142.ndmoid=up:active} 1 up:standby 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: mgrmap e41: y(active, since 2m), standbys: x 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: overall HEALTH_OK 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:20.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4266025197' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:05:20.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:05:21.217 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:05:21] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:05:22.570 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:22 smithi152 ceph-mon[93456]: pgmap v68: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-08-17T17:05:22.759 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:22 smithi142 ceph-mon[103614]: pgmap v68: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-08-17T17:05:22.759 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:22 smithi142 ceph-mon[106729]: pgmap v68: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-08-17T17:05:24.204 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:23 smithi142 ceph-mon[103614]: pgmap v69: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:24.204 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:23 smithi142 ceph-mon[106729]: pgmap v69: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:24.205 INFO:teuthology.orchestra.run.smithi142.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:05:24.205 INFO:teuthology.orchestra.run.smithi142.stdout:alertmanager.a smithi142 *:9093,9094 running (4m) 22s ago 8m 20.3M - 0.27.0 11f11916f8cd 95562b1fc088 2024-08-17T17:05:24.205 INFO:teuthology.orchestra.run.smithi142.stdout:grafana.a smithi152 *:3000 running (3m) 23s ago 8m 91.3M - 9.4.7 954c08fa6188 d463717746ae 2024-08-17T17:05:24.205 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi142.ndmoid smithi142 running (7m) 22s ago 7m 17.4M - 17.2.7-1404-gf2c39d4d f69471c56189 bd857f554782 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi152.tacamo smithi152 running (7m) 23s ago 7m 16.1M - 17.2.7-1404-gf2c39d4d f69471c56189 8913efd6a27d 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.x smithi152 *:8443,9283,8765 running (2m) 23s ago 12m 453M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb bc654892e833 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.y smithi142 *:8443,9283,8765 running (3m) 22s ago 14m 509M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 323039997e22 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a smithi142 running (2m) 22s ago 14m 43.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 3a2a93ab0fbb 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b smithi152 starting - - - 2048M 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c smithi142 running (109s) 22s ago 13m 35.0M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb e6fccaa16a72 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.a smithi142 *:9100 running (4m) 22s ago 8m 12.4M - 1.7.0 72c9c2088986 c6cd5b4c28e1 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.b smithi152 *:9100 running (3m) 23s ago 8m 11.1M - 1.7.0 72c9c2088986 8e3769ed942e 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 smithi142 running (12m) 22s ago 12m 381M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 cb819443ee72 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 smithi142 running (12m) 22s ago 12m 331M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 5cd1d20f8eca 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 smithi142 running (11m) 22s ago 11m 457M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 b599f0d55518 2024-08-17T17:05:24.206 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 smithi142 running (11m) 22s ago 11m 479M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 d53780492c5b 2024-08-17T17:05:24.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 smithi152 running (10m) 23s ago 10m 407M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 7a1cab3a1049 2024-08-17T17:05:24.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 smithi152 running (10m) 23s ago 10m 440M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 cac09b3f3a95 2024-08-17T17:05:24.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 smithi152 running (10m) 23s ago 10m 299M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 81327b9b9e78 2024-08-17T17:05:24.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 smithi152 running (9m) 23s ago 9m 382M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 0a45f0507de1 2024-08-17T17:05:24.207 INFO:teuthology.orchestra.run.smithi142.stdout:prometheus.a smithi152 *:9095 running (2m) 23s ago 9m 49.2M - 2.51.0 1d3b7f56885b f12943011a9f 2024-08-17T17:05:24.251 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:23 smithi152 ceph-mon[93456]: pgmap v69: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:24.251 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:24 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:24.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:05:25.335 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[103614]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[106729]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-mon[93456]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:26.075 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:26.075 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:25 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:26.785 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:26 smithi142 ceph-mon[103614]: pgmap v70: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-08-17T17:05:26.786 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:26.788 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:26 smithi142 ceph-mon[106729]: pgmap v70: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-08-17T17:05:26.788 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:26.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:26 smithi152 ceph-mon[93456]: pgmap v70: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-08-17T17:05:26.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:27.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:27.870 INFO:teuthology.orchestra.run.smithi142.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:28.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[106729]: pgmap v71: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[106729]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[106729]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[103614]: pgmap v71: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[103614]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[103614]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:28 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:29.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:28 smithi152 ceph-mon[93456]: pgmap v71: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-08-17T17:05:29.158 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:28 smithi152 ceph-mon[93456]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:29.158 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:28 smithi152 ceph-mon[93456]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:29.158 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:28 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:29.368 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:30.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: pgmap v72: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:05:30.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:30.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: pgmap v72: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:30.917 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: pgmap v72: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:31.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:32.062 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:31 smithi142 ceph-mon[106729]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:05:32.062 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:31 smithi142 ceph-mon[106729]: Reconfiguring daemon mon.a on smithi142 2024-08-17T17:05:32.062 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:05:31] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-08-17T17:05:32.063 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:31 smithi142 ceph-mon[103614]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:05:32.063 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:31 smithi142 ceph-mon[103614]: Reconfiguring daemon mon.a on smithi142 2024-08-17T17:05:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:31 smithi152 ceph-mon[93456]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:05:32.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:31 smithi152 ceph-mon[93456]: Reconfiguring daemon mon.a on smithi142 2024-08-17T17:05:33.222 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:32 smithi142 ceph-mon[103614]: pgmap v73: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-08-17T17:05:33.222 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:32 smithi142 ceph-mon[106729]: pgmap v73: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-08-17T17:05:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:32 smithi152 ceph-mon[93456]: pgmap v73: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-08-17T17:05:34.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: pgmap v74: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:34.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:34.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:05:34.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:05:34.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:05:34.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:34.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:34 smithi152 ceph-mon[93456]: Reconfiguring daemon mgr.y on smithi142 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: pgmap v74: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[103614]: Reconfiguring daemon mgr.y on smithi142 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: pgmap v74: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:34.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:34 smithi142 ceph-mon[106729]: Reconfiguring daemon mgr.y on smithi142 2024-08-17T17:05:36.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:35 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:36.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:35 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: pgmap v75: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:36.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[106729]: Reconfiguring daemon mon.c on smithi142 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: pgmap v75: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:36 smithi142 ceph-mon[103614]: Reconfiguring daemon mon.c on smithi142 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: pgmap v75: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:36.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-mon[93456]: Reconfiguring daemon mon.c on smithi142 2024-08-17T17:05:37.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:36 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: pgmap v76: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:38 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.0 on smithi142 2024-08-17T17:05:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: pgmap v76: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.0 on smithi142 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: pgmap v76: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:38.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:05:38.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:05:38.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:38.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:38 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.0 on smithi142 2024-08-17T17:05:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: pgmap v77: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:05:40.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:05:40.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:40.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:40 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.1 on smithi142 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: pgmap v77: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.1 on smithi142 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: pgmap v77: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:40.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:40 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.1 on smithi142 2024-08-17T17:05:42.024 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:05:41] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.51.0" 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: pgmap v78: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:42.282 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.2 on smithi142 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: pgmap v78: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.2 on smithi142 2024-08-17T17:05:42.283 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: pgmap v78: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-08-17T17:05:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:05:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:05:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.2 on smithi142 2024-08-17T17:05:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:44.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:45.168 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[106729]: pgmap v79: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:05:45.168 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:45.169 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:45.169 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:05:45.169 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:45.169 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[103614]: pgmap v79: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:05:45.169 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:45.170 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:45.170 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:05:45.170 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:44 smithi152 ceph-mon[93456]: pgmap v79: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:05:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:05:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:46.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:45 smithi152 ceph-mon[93456]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:05:46.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:45 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.3 on smithi142 2024-08-17T17:05:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:46.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:46.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:45 smithi142 ceph-mon[106729]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:05:46.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:45 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.3 on smithi142 2024-08-17T17:05:46.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:45 smithi142 ceph-mon[103614]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:05:46.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:45 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.3 on smithi142 2024-08-17T17:05:47.229 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[106729]: pgmap v80: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[103614]: pgmap v80: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:47.230 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:47.231 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:05:47.231 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:46 smithi152 ceph-mon[93456]: pgmap v80: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-08-17T17:05:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi142.ndmoid", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:05:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:47.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:47 smithi152 ceph-mon[93456]: Reconfiguring mds.foo.smithi142.ndmoid (monmap changed)... 2024-08-17T17:05:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:47 smithi152 ceph-mon[93456]: Reconfiguring daemon mds.foo.smithi142.ndmoid on smithi142 2024-08-17T17:05:48.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:47 smithi142 ceph-mon[106729]: Reconfiguring mds.foo.smithi142.ndmoid (monmap changed)... 2024-08-17T17:05:48.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:47 smithi142 ceph-mon[106729]: Reconfiguring daemon mds.foo.smithi142.ndmoid on smithi142 2024-08-17T17:05:48.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:47 smithi142 ceph-mon[103614]: Reconfiguring mds.foo.smithi142.ndmoid (monmap changed)... 2024-08-17T17:05:48.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:47 smithi142 ceph-mon[103614]: Reconfiguring daemon mds.foo.smithi142.ndmoid on smithi142 2024-08-17T17:05:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: pgmap v81: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-08-17T17:05:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:05:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:49.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:49.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:48 smithi152 ceph-mon[93456]: Reconfiguring daemon mon.b on smithi152 2024-08-17T17:05:49.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: pgmap v81: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[103614]: Reconfiguring daemon mon.b on smithi152 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: pgmap v81: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:49.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:05:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:05:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:05:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:48 smithi142 ceph-mon[106729]: Reconfiguring daemon mon.b on smithi152 2024-08-17T17:05:50.529 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: pgmap v82: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:05:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:05:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:05:51.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:05:51.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:51.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:50 smithi152 ceph-mon[93456]: Reconfiguring daemon mgr.x on smithi152 2024-08-17T17:05:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: pgmap v82: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[103614]: Reconfiguring daemon mgr.x on smithi152 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: pgmap v82: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:51.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:05:51.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:05:51.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:05:51.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:51.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:50 smithi142 ceph-mon[106729]: Reconfiguring daemon mgr.x on smithi152 2024-08-17T17:05:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:05:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:05:51] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.51.0" 2024-08-17T17:05:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: pgmap v83: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:05:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:05:52.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:05:52.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:52.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:52 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.4 on smithi152 2024-08-17T17:05:53.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: pgmap v83: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.4 on smithi152 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: pgmap v83: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:05:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:05:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:53.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:52 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.4 on smithi152 2024-08-17T17:05:54.338 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: pgmap v84: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:54.942 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:54 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.5 on smithi152 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: pgmap v84: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.5 on smithi152 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: pgmap v84: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:55.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:54 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.5 on smithi152 2024-08-17T17:05:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:56.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:55.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: pgmap v85: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.6 on smithi152 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: pgmap v85: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-08-17T17:05:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:05:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:05:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.6 on smithi152 2024-08-17T17:05:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:56.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: pgmap v85: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-08-17T17:05:56.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:56.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:56.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:05:56.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:05:56.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:56.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.6 on smithi152 2024-08-17T17:05:56.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:57.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:05:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:05:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:05:59.048 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:58 smithi152 ceph-mon[93456]: pgmap v86: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-08-17T17:05:59.048 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:59.048 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:59.048 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:05:59.049 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:59.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[103614]: pgmap v86: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[106729]: pgmap v86: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:05:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:00.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:59 smithi152 ceph-mon[93456]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:06:00.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:05:59 smithi152 ceph-mon[93456]: Reconfiguring daemon osd.7 on smithi152 2024-08-17T17:06:00.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:59 smithi142 ceph-mon[106729]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:06:00.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:05:59 smithi142 ceph-mon[106729]: Reconfiguring daemon osd.7 on smithi152 2024-08-17T17:06:00.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:59 smithi142 ceph-mon[103614]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:06:00.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:05:59 smithi142 ceph-mon[103614]: Reconfiguring daemon osd.7 on smithi152 2024-08-17T17:06:00.528 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:00 smithi152 ceph-mon[93456]: pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:06:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:00 smithi142 ceph-mon[103614]: pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:06:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:00 smithi142 ceph-mon[106729]: pgmap v87: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:06:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:01 smithi152 ceph-mon[93456]: Reconfiguring mds.foo.smithi152.tacamo (monmap changed)... 2024-08-17T17:06:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:06:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:01 smithi152 ceph-mon[93456]: Reconfiguring daemon mds.foo.smithi152.tacamo on smithi152 2024-08-17T17:06:02.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[103614]: Reconfiguring mds.foo.smithi152.tacamo (monmap changed)... 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[103614]: Reconfiguring daemon mds.foo.smithi152.tacamo on smithi152 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[106729]: Reconfiguring mds.foo.smithi152.tacamo (monmap changed)... 2024-08-17T17:06:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.tacamo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:06:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-mon[106729]: Reconfiguring daemon mds.foo.smithi152.tacamo on smithi152 2024-08-17T17:06:02.124 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:06:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:06:01] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.51.0" 2024-08-17T17:06:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:02 smithi152 ceph-mon[93456]: pgmap v88: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:06:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:03.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[103614]: pgmap v88: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:06:03.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:03.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:03.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:03.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[106729]: pgmap v88: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:06:03.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:03.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:03.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:04.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:04.939 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:04 smithi142 ceph-mon[103614]: pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:06:04.939 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:04 smithi142 ceph-mon[106729]: pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:06:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:04 smithi152 ceph-mon[93456]: pgmap v89: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:06:06.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:06.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:05.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:06 smithi152 ceph-mon[93456]: pgmap v90: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T17:06:06.824 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:06 smithi142 ceph-mon[103614]: pgmap v90: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T17:06:06.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:06 smithi142 ceph-mon[106729]: pgmap v90: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T17:06:07.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:08 smithi152 ceph-mon[93456]: pgmap v91: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T17:06:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:08 smithi142 ceph-mon[106729]: pgmap v91: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T17:06:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:08 smithi142 ceph-mon[103614]: pgmap v91: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T17:06:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:10.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:10.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:10.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:10.025 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:10.025 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:10.025 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:10 smithi152 ceph-mon[93456]: pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:06:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:10 smithi142 ceph-mon[103614]: pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:06:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:10 smithi142 ceph-mon[106729]: pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:06:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:12.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:06:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:06:11] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-08-17T17:06:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:12 smithi152 ceph-mon[93456]: pgmap v93: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-08-17T17:06:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:12 smithi142 ceph-mon[106729]: pgmap v93: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-08-17T17:06:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:12 smithi142 ceph-mon[103614]: pgmap v93: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-08-17T17:06:14.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.142\", device_class=\"ssd\", hostname=\"smithi142\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.142\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:14 smithi152 ceph-mon[93456]: pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:06:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:14 smithi142 ceph-mon[106729]: pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:06:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:14 smithi142 ceph-mon[103614]: pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:06:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:16.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:16 smithi152 ceph-mon[93456]: pgmap v95: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-17T17:06:17.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi152\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi152\", job=\"node\", machine=\"x86_64\", nodename=\"smithi152\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:16 smithi142 ceph-mon[106729]: pgmap v95: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-17T17:06:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:16 smithi142 ceph-mon[103614]: pgmap v95: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-17T17:06:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:18 smithi152 ceph-mon[93456]: pgmap v96: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-17T17:06:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:18 smithi142 ceph-mon[106729]: pgmap v96: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-17T17:06:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:18 smithi142 ceph-mon[103614]: pgmap v96: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-17T17:06:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:20 smithi152 ceph-mon[93456]: pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:06:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:20 smithi142 ceph-mon[106729]: pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:06:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:20 smithi142 ceph-mon[103614]: pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:06:22.047 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:06:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:06:21] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-08-17T17:06:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:22 smithi152 ceph-mon[93456]: pgmap v98: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-08-17T17:06:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:22 smithi142 ceph-mon[106729]: pgmap v98: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-08-17T17:06:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:22 smithi142 ceph-mon[103614]: pgmap v98: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-08-17T17:06:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:24 smithi152 ceph-mon[93456]: pgmap v99: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:06:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:24 smithi142 ceph-mon[106729]: pgmap v99: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:06:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:24 smithi142 ceph-mon[103614]: pgmap v99: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:06:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:26 smithi152 ceph-mon[93456]: pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:06:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:27.073 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi142\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi142\", job=\"node\", machine=\"x86_64\", nodename=\"smithi142\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi142\", job=\"node\", machine=\"x86_64\", nodename=\"smithi142\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:26 smithi142 ceph-mon[106729]: pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:06:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:26 smithi142 ceph-mon[103614]: pgmap v100: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:06:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:28 smithi152 ceph-mon[93456]: pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:06:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:28 smithi142 ceph-mon[106729]: pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:06:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:28 smithi142 ceph-mon[103614]: pgmap v101: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:06:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:30 smithi152 ceph-mon[93456]: pgmap v102: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-08-17T17:06:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:30 smithi142 ceph-mon[106729]: pgmap v102: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-08-17T17:06:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:30 smithi142 ceph-mon[103614]: pgmap v102: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-08-17T17:06:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:06:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:06:31] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-08-17T17:06:32.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade resume' 2024-08-17T17:06:32.865 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:33.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:32 smithi142 ceph-mon[103614]: pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:06:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:32 smithi142 ceph-mon[106729]: pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:06:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:32 smithi152 ceph-mon[93456]: pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:06:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:34 smithi142 ceph-mon[103614]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-08-17T17:06:34.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:34 smithi142 ceph-mon[106729]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-08-17T17:06:35.315 INFO:teuthology.orchestra.run.smithi142.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:35.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:34 smithi152 ceph-mon[93456]: pgmap v104: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-08-17T17:06:36.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for half of osds to upgrade' 2024-08-17T17:06:36.507 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:36 smithi152 ceph-mon[93456]: pgmap v105: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-08-17T17:06:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:36 smithi152 ceph-mon[93456]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:36 smithi152 ceph-mon[93456]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:36.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[103614]: pgmap v105: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[103614]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[103614]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[106729]: pgmap v105: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[106729]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[106729]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:36.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:38 smithi152 ceph-mon[93456]: pgmap v106: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-08-17T17:06:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:38 smithi142 ceph-mon[106729]: pgmap v106: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-08-17T17:06:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:38 smithi142 ceph-mon[103614]: pgmap v106: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-08-17T17:06:38.669 INFO:teuthology.orchestra.run.smithi142.stdout:wait for half of osds to upgrade 2024-08-17T17:06:39.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:06:40.060 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:06:40.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:40 smithi152 ceph-mon[93456]: pgmap v107: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:06:40.416 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:40.529 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:40 smithi142 ceph-mon[103614]: pgmap v107: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:06:40.529 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:40 smithi142 ceph-mon[106729]: pgmap v107: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:06:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:06:41.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:06:41.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:06:41.875 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:06:41] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:06:41.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:06:41.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:06:41.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:06:41.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:06:41.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:06:41.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:41.877 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:06:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:06:42.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:06:42.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:42.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:42.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:06:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[103614]: pgmap v108: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-08-17T17:06:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[103614]: Upgrade: Setting container_image for all mon 2024-08-17T17:06:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[103614]: Upgrade: Setting container_image for all crash 2024-08-17T17:06:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[103614]: Upgrade: osd.0 is safe to restart 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3225451842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[106729]: pgmap v108: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[106729]: Upgrade: Setting container_image for all mon 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[106729]: Upgrade: Setting container_image for all crash 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[106729]: Upgrade: osd.0 is safe to restart 2024-08-17T17:06:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3225451842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:42 smithi152 ceph-mon[93456]: pgmap v108: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-08-17T17:06:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:42 smithi152 ceph-mon[93456]: Upgrade: Setting container_image for all mon 2024-08-17T17:06:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:42 smithi152 ceph-mon[93456]: Upgrade: Setting container_image for all crash 2024-08-17T17:06:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:42 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:06:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:42 smithi152 ceph-mon[93456]: Upgrade: osd.0 is safe to restart 2024-08-17T17:06:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3225451842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:43.076 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:06:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: pgmap v109: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:06:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: Upgrade: Updating osd.0 2024-08-17T17:06:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:06:45.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:45.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:44 smithi152 ceph-mon[93456]: Deploying daemon osd.0 on smithi142 2024-08-17T17:06:45.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: pgmap v109: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: Upgrade: Updating osd.0 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[103614]: Deploying daemon osd.0 on smithi142 2024-08-17T17:06:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: pgmap v109: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: Upgrade: Updating osd.0 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:44 smithi142 ceph-mon[106729]: Deploying daemon osd.0 on smithi142 2024-08-17T17:06:46.732 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:06:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:46 smithi152 ceph-mon[93456]: pgmap v110: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:06:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1357163554' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:46 smithi142 ceph-mon[106729]: pgmap v110: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:06:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1357163554' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:46 smithi142 ceph-mon[103614]: pgmap v110: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:06:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1357163554' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:47 smithi152 ceph-mon[93456]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:47 smithi142 ceph-mon[103614]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:48.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:47 smithi142 ceph-mon[106729]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:48.124 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:47 smithi142 systemd[1]: Stopping Ceph osd.0 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:06:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-mon[106729]: pgmap v111: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:06:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-mon[106729]: from='client.54130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-mon[106729]: osd.0 marked itself down and dead 2024-08-17T17:06:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-mon[103614]: pgmap v111: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:06:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-mon[103614]: from='client.54130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-mon[103614]: osd.0 marked itself down and dead 2024-08-17T17:06:48.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[44296]: 2024-08-17T17:06:48.495+0000 7faa442bd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:06:48.874 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[44296]: 2024-08-17T17:06:48.495+0000 7faa442bd640 -1 osd.0 85 *** Got signal Terminated *** 2024-08-17T17:06:48.874 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:48 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[44296]: 2024-08-17T17:06:48.495+0000 7faa442bd640 -1 osd.0 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:06:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:48 smithi152 ceph-mon[93456]: pgmap v111: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-08-17T17:06:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:48 smithi152 ceph-mon[93456]: from='client.54130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:48 smithi152 ceph-mon[93456]: osd.0 marked itself down and dead 2024-08-17T17:06:49.622 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:49 smithi142 podman[115808]: 2024-08-17 17:06:49.256737559 +0000 UTC m=+1.214090454 container died cb819443ee722a381e60533238687d951cda3dbffefdb8310e71467f586a752f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:06:49.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:06:49.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[103614]: osdmap e86: 8 total, 7 up, 8 in 2024-08-17T17:06:49.946 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2856650229' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:49.947 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[103614]: osdmap e87: 8 total, 7 up, 8 in 2024-08-17T17:06:49.947 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:06:49.947 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[106729]: osdmap e86: 8 total, 7 up, 8 in 2024-08-17T17:06:49.947 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2856650229' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:49.947 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:49 smithi142 ceph-mon[106729]: osdmap e87: 8 total, 7 up, 8 in 2024-08-17T17:06:49.947 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:49 smithi142 podman[115808]: 2024-08-17 17:06:49.687704683 +0000 UTC m=+1.645057576 container cleanup cb819443ee722a381e60533238687d951cda3dbffefdb8310e71467f586a752f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2) 2024-08-17T17:06:49.948 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:49 smithi142 bash[115808]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:06:49.948 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:49 smithi142 podman[115828]: 2024-08-17 17:06:49.931784048 +0000 UTC m=+0.673651604 container remove cb819443ee722a381e60533238687d951cda3dbffefdb8310e71467f586a752f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:06:49.948 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:06:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:49 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:06:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:49 smithi152 ceph-mon[93456]: osdmap e86: 8 total, 7 up, 8 in 2024-08-17T17:06:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2856650229' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:49 smithi152 ceph-mon[93456]: osdmap e87: 8 total, 7 up, 8 in 2024-08-17T17:06:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:06:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:06:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:06:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:50 smithi142 ceph-mon[106729]: pgmap v113: 89 pgs: 13 stale+active+clean, 76 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T17:06:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:50 smithi142 ceph-mon[106729]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:50.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:50 smithi142 podman[115980]: 2024-08-17 17:06:50.519490773 +0000 UTC m=+0.038665617 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:50.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:50 smithi142 podman[115980]: 2024-08-17 17:06:50.64135164 +0000 UTC m=+0.160526456 container create b1bff5c114a5b062fc943b4d8c4ae5b0e191694f5c6237efef5ac304e976666a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:06:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:50 smithi142 ceph-mon[103614]: pgmap v113: 89 pgs: 13 stale+active+clean, 76 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T17:06:50.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:50 smithi142 ceph-mon[103614]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:50 smithi152 ceph-mon[93456]: pgmap v113: 89 pgs: 13 stale+active+clean, 76 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T17:06:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:50 smithi152 ceph-mon[93456]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:51.220 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:50 smithi142 podman[115980]: 2024-08-17 17:06:50.919898873 +0000 UTC m=+0.439073700 container init b1bff5c114a5b062fc943b4d8c4ae5b0e191694f5c6237efef5ac304e976666a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-08-17T17:06:51.220 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:50 smithi142 podman[115980]: 2024-08-17 17:06:50.926811966 +0000 UTC m=+0.445986793 container start b1bff5c114a5b062fc943b4d8c4ae5b0e191694f5c6237efef5ac304e976666a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-17T17:06:51.220 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:51 smithi142 podman[115980]: 2024-08-17 17:06:51.028257656 +0000 UTC m=+0.547432479 container attach b1bff5c114a5b062fc943b4d8c4ae5b0e191694f5c6237efef5ac304e976666a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe) 2024-08-17T17:06:51.220 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:51 smithi142 podman[115980]: 2024-08-17 17:06:51.219255867 +0000 UTC m=+0.738430695 container died b1bff5c114a5b062fc943b4d8c4ae5b0e191694f5c6237efef5ac304e976666a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:06:51.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:51 smithi142 ceph-mon[103614]: from='client.54148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:51.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:06:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:06:51] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-08-17T17:06:51.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:51 smithi142 ceph-mon[106729]: from='client.54148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:51 smithi152 ceph-mon[93456]: from='client.54148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.215 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:51 smithi142 podman[116021]: 2024-08-17 17:06:51.875181252 +0000 UTC m=+0.646639738 container remove b1bff5c114a5b062fc943b4d8c4ae5b0e191694f5c6237efef5ac304e976666a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-08-17T17:06:52.215 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:51 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-17T17:06:52.216 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:51 smithi142 systemd[1]: Stopped Ceph osd.0 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:06:52.216 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:51 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.0.service: Consumed 12min 16.895s CPU time. 2024-08-17T17:06:52.555 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:52 smithi142 systemd[1]: Starting Ceph osd.0 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:06:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:52 smithi142 ceph-mon[106729]: pgmap v115: 89 pgs: 13 stale+active+clean, 76 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 472 op/s 2024-08-17T17:06:52.872 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:52 smithi142 podman[116117]: 2024-08-17 17:06:52.459170606 +0000 UTC m=+0.036352633 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:52.872 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:52 smithi142 podman[116117]: 2024-08-17 17:06:52.584821752 +0000 UTC m=+0.162003766 container create b5a0c227207e392b4603700d141a77c4682ab4ed0ae86359fe01c9a0a7e89db7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:06:52.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:52 smithi142 ceph-mon[103614]: pgmap v115: 89 pgs: 13 stale+active+clean, 76 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 472 op/s 2024-08-17T17:06:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:52 smithi152 ceph-mon[93456]: pgmap v115: 89 pgs: 13 stale+active+clean, 76 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 472 op/s 2024-08-17T17:06:53.149 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:52 smithi142 podman[116117]: 2024-08-17 17:06:52.871449899 +0000 UTC m=+0.448631930 container init b5a0c227207e392b4603700d141a77c4682ab4ed0ae86359fe01c9a0a7e89db7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:06:53.149 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:52 smithi142 podman[116117]: 2024-08-17 17:06:52.880191711 +0000 UTC m=+0.457373725 container start b5a0c227207e392b4603700d141a77c4682ab4ed0ae86359fe01c9a0a7e89db7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:06:53.149 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:52 smithi142 podman[116117]: 2024-08-17 17:06:52.996720588 +0000 UTC m=+0.573902609 container attach b5a0c227207e392b4603700d141a77c4682ab4ed0ae86359fe01c9a0a7e89db7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True) 2024-08-17T17:06:53.150 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.150 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.150 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.150 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.573 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:06:53.573 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:06:53.573 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:06:53.573 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.573 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.574 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.574 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:53.574 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:06:53.574 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:06:53.574 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:06:53.574 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:06:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3558891329' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:06:53.873 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:06:53.874 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:06:53.874 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate[116150]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:06:53.874 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 bash[116117]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:06:53.874 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:53 smithi142 podman[116117]: 2024-08-17 17:06:53.645476694 +0000 UTC m=+1.222658712 container died b5a0c227207e392b4603700d141a77c4682ab4ed0ae86359fe01c9a0a7e89db7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-08-17T17:06:53.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3558891329' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3558891329' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:54.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:54 smithi142 ceph-mon[103614]: pgmap v116: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:54.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:54 smithi142 ceph-mon[103614]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:54.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:54 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 4869/57924 objects degraded (8.406%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:54.952 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:54 smithi142 ceph-mon[106729]: pgmap v116: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:54.952 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:54 smithi142 ceph-mon[106729]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:54.952 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:54 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 4869/57924 objects degraded (8.406%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:54.953 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:54 smithi142 podman[116323]: 2024-08-17 17:06:54.62811431 +0000 UTC m=+0.971936911 container remove b5a0c227207e392b4603700d141a77c4682ab4ed0ae86359fe01c9a0a7e89db7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-08-17T17:06:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:54 smithi152 ceph-mon[93456]: pgmap v116: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:54 smithi152 ceph-mon[93456]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:54 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 4869/57924 objects degraded (8.406%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:55.267 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:54 smithi142 podman[116408]: 2024-08-17 17:06:54.860751551 +0000 UTC m=+0.039673192 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:55.267 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:54 smithi142 podman[116408]: 2024-08-17 17:06:54.987612552 +0000 UTC m=+0.166534176 container create 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.36.0) 2024-08-17T17:06:55.267 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:55 smithi142 podman[116408]: 2024-08-17 17:06:55.266280335 +0000 UTC m=+0.445201957 container init 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe) 2024-08-17T17:06:55.623 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:55 smithi142 podman[116408]: 2024-08-17 17:06:55.273803493 +0000 UTC m=+0.452725119 container start 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:06:55.623 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:55 smithi142 bash[116408]: 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca 2024-08-17T17:06:55.623 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:55 smithi142 systemd[1]: Started Ceph osd.0 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:06:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:55 smithi152 ceph-mon[93456]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[106729]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:56.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:56.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:56.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:56.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[103614]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:56.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:56.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:56.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:56 smithi152 ceph-mon[93456]: pgmap v117: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2261436236' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:57.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[103614]: pgmap v117: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:57.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:57.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:57.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2261436236' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:57.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[106729]: pgmap v117: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:57.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:06:57.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:57.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2261436236' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:06:57.125 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:06:56 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[116420]: 2024-08-17T17:06:56.943+0000 7f65ca534640 -1 Falling back to public interface 2024-08-17T17:06:57.175 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:06:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:58 smithi152 ceph-mon[93456]: pgmap v118: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:58 smithi152 ceph-mon[93456]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:58 smithi152 ceph-mon[93456]: from='client.54172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:59.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:58 smithi142 ceph-mon[106729]: pgmap v118: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:58 smithi142 ceph-mon[106729]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:58 smithi142 ceph-mon[106729]: from='client.54172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:58 smithi142 ceph-mon[103614]: pgmap v118: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 4869/57924 objects degraded (8.406%) 2024-08-17T17:06:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:58 smithi142 ceph-mon[103614]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:58 smithi142 ceph-mon[103614]: from='client.54172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:00.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:06:59 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3728/43851 objects degraded (8.502%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:00.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:06:59 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3728/43851 objects degraded (8.502%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:00.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:06:59 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3728/43851 objects degraded (8.502%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:07:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:07:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:07:00.787 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:01.038 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:00 smithi142 ceph-mon[103614]: pgmap v119: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 3728/43851 objects degraded (8.502%) 2024-08-17T17:07:01.038 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1683776235' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:01.038 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:00 smithi142 ceph-mon[106729]: pgmap v119: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 3728/43851 objects degraded (8.502%) 2024-08-17T17:07:01.038 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1683776235' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:00 smithi152 ceph-mon[93456]: pgmap v119: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 3728/43851 objects degraded (8.502%) 2024-08-17T17:07:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1683776235' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:02.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:07:01] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.51.0" 2024-08-17T17:07:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-mon[106729]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-mon[103614]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:02.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:01 smithi152 ceph-mon[93456]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[106729]: pgmap v120: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3728/43851 objects degraded (8.502%) 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[106729]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[103614]: pgmap v120: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3728/43851 objects degraded (8.502%) 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[103614]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:02 smithi152 ceph-mon[93456]: pgmap v120: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 3728/43851 objects degraded (8.502%) 2024-08-17T17:07:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:02 smithi152 ceph-mon[93456]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:04.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3994198594' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:04.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3994198594' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:04.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3994198594' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:04.458 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:04.622 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[116420]: 2024-08-17T17:07:04.404+0000 7f65ca534640 -1 osd.0 85 log_to_monitors true 2024-08-17T17:07:05.118 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-mon[103614]: pgmap v121: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:05.118 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-mon[103614]: from='osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:07:05.118 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-mon[103614]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:07:05.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-mon[106729]: pgmap v121: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:05.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-mon[106729]: from='osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:07:05.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:04 smithi142 ceph-mon[106729]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:07:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:04 smithi152 ceph-mon[93456]: pgmap v121: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:04 smithi152 ceph-mon[93456]: from='osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:07:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:04 smithi152 ceph-mon[93456]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:07:05.872 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[116420]: 2024-08-17T17:07:05.407+0000 7f65be2d6640 -1 osd.0 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:07:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:07:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: from='osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: osdmap e88: 8 total, 7 up, 8 in 2024-08-17T17:07:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2655/30753 objects degraded (8.633%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:06.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:05 smithi152 ceph-mon[93456]: from='osd.0 ' entity='osd.0' 2024-08-17T17:07:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: from='osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: osdmap e88: 8 total, 7 up, 8 in 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2655/30753 objects degraded (8.633%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[106729]: from='osd.0 ' entity='osd.0' 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:07:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: from='osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:06.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: osdmap e88: 8 total, 7 up, 8 in 2024-08-17T17:07:06.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:06.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2655/30753 objects degraded (8.633%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:06.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:05 smithi142 ceph-mon[103614]: from='osd.0 ' entity='osd.0' 2024-08-17T17:07:07.074 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[106729]: from='client.44240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[106729]: pgmap v123: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[106729]: osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512] boot 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[106729]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[103614]: from='client.44240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[103614]: pgmap v123: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:07.075 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:07.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[103614]: osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512] boot 2024-08-17T17:07:07.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[103614]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:07:07.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:07:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:06 smithi152 ceph-mon[93456]: from='client.44240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:06 smithi152 ceph-mon[93456]: pgmap v123: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:06 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:06 smithi152 ceph-mon[93456]: osd.0 [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512] boot 2024-08-17T17:07:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:06 smithi152 ceph-mon[93456]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:07:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:06 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:07:07.570 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:07.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:07:07.641+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3580729052' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:07:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3580729052' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3580729052' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:08.375 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:07:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:08 smithi152 ceph-mon[93456]: pgmap v125: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:08 smithi152 ceph-mon[93456]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:08 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:07:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:08 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:08 smithi152 ceph-mon[93456]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:08 smithi152 ceph-mon[93456]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T17:07:09.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[103614]: pgmap v125: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[103614]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[103614]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[103614]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[106729]: pgmap v125: 89 pgs: 19 active+undersized, 11 active+undersized+degraded, 59 active+clean; 640 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 2655/30753 objects degraded (8.633%) 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[106729]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[106729]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:09.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:08 smithi142 ceph-mon[106729]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T17:07:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:07:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:07:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:07:10.890 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:11.175 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:10 smithi142 ceph-mon[106729]: pgmap v128: 89 pgs: 2 remapped+peering, 11 peering, 9 active+undersized, 2 active+undersized+degraded, 65 active+clean; 432 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 3/20775 objects degraded (0.014%) 2024-08-17T17:07:11.175 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3/20775 objects degraded (0.014%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:11.175 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2257465183' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:11.175 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:10 smithi142 ceph-mon[103614]: pgmap v128: 89 pgs: 2 remapped+peering, 11 peering, 9 active+undersized, 2 active+undersized+degraded, 65 active+clean; 432 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 3/20775 objects degraded (0.014%) 2024-08-17T17:07:11.175 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3/20775 objects degraded (0.014%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:11.175 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2257465183' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:10 smithi152 ceph-mon[93456]: pgmap v128: 89 pgs: 2 remapped+peering, 11 peering, 9 active+undersized, 2 active+undersized+degraded, 65 active+clean; 432 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 3/20775 objects degraded (0.014%) 2024-08-17T17:07:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3/20775 objects degraded (0.014%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2257465183' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:12.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-mon[103614]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:07:11] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.51.0" 2024-08-17T17:07:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-mon[106729]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:11 smithi152 ceph-mon[93456]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[106729]: pgmap v129: 89 pgs: 2 remapped+peering, 11 peering, 76 active+clean; 432 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[106729]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/20775 objects degraded (0.014%), 2 pgs degraded) 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[103614]: pgmap v129: 89 pgs: 2 remapped+peering, 11 peering, 76 active+clean; 432 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[103614]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/20775 objects degraded (0.014%), 2 pgs degraded) 2024-08-17T17:07:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:12 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:07:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:12 smithi152 ceph-mon[93456]: pgmap v129: 89 pgs: 2 remapped+peering, 11 peering, 76 active+clean; 432 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2024-08-17T17:07:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:12 smithi152 ceph-mon[93456]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:12 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/20775 objects degraded (0.014%), 2 pgs degraded) 2024-08-17T17:07:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:12 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:07:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:13 smithi152 ceph-mon[93456]: pgmap v130: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 39 objects/s recovering 2024-08-17T17:07:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:13 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 2785/11349 objects degraded (24.540%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/770983174' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:13 smithi142 ceph-mon[103614]: pgmap v130: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 39 objects/s recovering 2024-08-17T17:07:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:13 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 2785/11349 objects degraded (24.540%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/770983174' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:13 smithi142 ceph-mon[106729]: pgmap v130: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 39 objects/s recovering 2024-08-17T17:07:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:13 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 2785/11349 objects degraded (24.540%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/770983174' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:14.520 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:14 smithi152 ceph-mon[93456]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:15.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:14 smithi142 ceph-mon[103614]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:15.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:15 smithi142 ceph-mon[106729]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:16 smithi152 ceph-mon[93456]: from='client.54235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:16 smithi152 ceph-mon[93456]: pgmap v131: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 33 objects/s recovering 2024-08-17T17:07:16.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:16 smithi142 ceph-mon[103614]: from='client.54235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:16.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:16 smithi142 ceph-mon[103614]: pgmap v131: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 33 objects/s recovering 2024-08-17T17:07:16.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:16 smithi142 ceph-mon[106729]: from='client.54235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:16.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:16 smithi142 ceph-mon[106729]: pgmap v131: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 33 objects/s recovering 2024-08-17T17:07:17.165 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 132187 objects 2024-08-17T17:07:17.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :154.636 2024-08-17T17:07:17.679 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:18.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:18 smithi152 ceph-mon[93456]: pgmap v132: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 27 objects/s recovering 2024-08-17T17:07:18.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1716216591' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:18.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:18 smithi152 ceph-mon[93456]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:18.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:18 smithi142 ceph-mon[106729]: pgmap v132: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 27 objects/s recovering 2024-08-17T17:07:18.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1716216591' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:18.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:18 smithi142 ceph-mon[106729]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:18.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:18 smithi142 ceph-mon[103614]: pgmap v132: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 235 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 2785/11349 objects degraded (24.540%); 0 B/s, 27 objects/s recovering 2024-08-17T17:07:18.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1716216591' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:18.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:18 smithi142 ceph-mon[103614]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:19 smithi152 ceph-mon[93456]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:19 smithi152 ceph-mon[93456]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T17:07:19.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:19 smithi142 ceph-mon[106729]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:19.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:19 smithi142 ceph-mon[106729]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T17:07:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:19 smithi142 ceph-mon[103614]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:19 smithi142 ceph-mon[103614]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T17:07:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:20 smithi152 ceph-mon[93456]: pgmap v134: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2785/78 objects degraded (3570.513%); 0 B/s, 65 objects/s recovering 2024-08-17T17:07:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2785/78 objects degraded (3570.513%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:20 smithi152 ceph-mon[93456]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T17:07:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:07:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:07:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:07:20.606 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:20.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:20 smithi142 ceph-mon[106729]: pgmap v134: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2785/78 objects degraded (3570.513%); 0 B/s, 65 objects/s recovering 2024-08-17T17:07:20.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2785/78 objects degraded (3570.513%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:20.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:20 smithi142 ceph-mon[106729]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T17:07:20.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:20 smithi142 ceph-mon[103614]: pgmap v134: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 2785/78 objects degraded (3570.513%); 0 B/s, 65 objects/s recovering 2024-08-17T17:07:20.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2785/78 objects degraded (3570.513%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:20.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:20 smithi142 ceph-mon[103614]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T17:07:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:21 smithi152 ceph-mon[93456]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:07:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3431271899' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:21 smithi152 ceph-mon[93456]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:21 smithi152 ceph-mon[93456]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[103614]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:07:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3431271899' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[103614]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[103614]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[106729]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:07:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3431271899' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[106729]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-mon[106729]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:07:21] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.51.0" 2024-08-17T17:07:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:22 smithi152 ceph-mon[93456]: pgmap v137: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 2785/78 objects degraded (3570.513%); 0 B/s, 64 objects/s recovering 2024-08-17T17:07:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:22 smithi152 ceph-mon[93456]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:07:22.622 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-08-17T17:07:22.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-08-17T17:07:22.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:22 smithi142 ceph-mon[103614]: pgmap v137: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 2785/78 objects degraded (3570.513%); 0 B/s, 64 objects/s recovering 2024-08-17T17:07:22.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:22 smithi142 ceph-mon[103614]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:07:22.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:22 smithi142 ceph-mon[106729]: pgmap v137: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 2785/78 objects degraded (3570.513%); 0 B/s, 64 objects/s recovering 2024-08-17T17:07:22.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:22 smithi142 ceph-mon[106729]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:07:22.957 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[106729]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[106729]: Upgrade: osd.1 is safe to restart 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[103614]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:07:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:23 smithi142 ceph-mon[103614]: Upgrade: osd.1 is safe to restart 2024-08-17T17:07:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:23 smithi152 ceph-mon[93456]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T17:07:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:23 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:07:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:23 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:07:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:23 smithi152 ceph-mon[93456]: Upgrade: osd.1 is safe to restart 2024-08-17T17:07:23.628 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[106729]: pgmap v140: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 955/78 objects degraded (1224.359%); 0 B/s, 359 objects/s recovering 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2838284512' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[106729]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[106729]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[103614]: pgmap v140: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 955/78 objects degraded (1224.359%); 0 B/s, 359 objects/s recovering 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2838284512' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[103614]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T17:07:24.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:24 smithi142 ceph-mon[103614]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:24 smithi152 ceph-mon[93456]: pgmap v140: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 955/78 objects degraded (1224.359%); 0 B/s, 359 objects/s recovering 2024-08-17T17:07:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2838284512' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:24 smithi152 ceph-mon[93456]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T17:07:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:24 smithi152 ceph-mon[93456]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:25 smithi152 ceph-mon[93456]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:07:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 955/78 objects degraded (1224.359%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:07:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:25 smithi152 ceph-mon[93456]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[106729]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 955/78 objects degraded (1224.359%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[106729]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[103614]: from='client.34503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:25.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:07:25.613 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:25.613 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 955/78 objects degraded (1224.359%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:07:25.613 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:25 smithi142 ceph-mon[103614]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T17:07:26.508 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[103614]: Upgrade: Updating osd.1 2024-08-17T17:07:26.508 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[103614]: Deploying daemon osd.1 on smithi142 2024-08-17T17:07:26.508 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[103614]: pgmap v142: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 87 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 485 objects/s recovering 2024-08-17T17:07:26.508 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 955/78 objects degraded (1224.359%), 1 pg degraded) 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[103614]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[106729]: Upgrade: Updating osd.1 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[106729]: Deploying daemon osd.1 on smithi142 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[106729]: pgmap v142: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 87 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 485 objects/s recovering 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 955/78 objects degraded (1224.359%), 1 pg degraded) 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:07:26.509 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:26 smithi142 ceph-mon[106729]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:07:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:26 smithi152 ceph-mon[93456]: Upgrade: Updating osd.1 2024-08-17T17:07:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:26 smithi152 ceph-mon[93456]: Deploying daemon osd.1 on smithi142 2024-08-17T17:07:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:26 smithi152 ceph-mon[93456]: pgmap v142: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 87 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 485 objects/s recovering 2024-08-17T17:07:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:26 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 955/78 objects degraded (1224.359%), 1 pg degraded) 2024-08-17T17:07:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:26 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:07:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:26 smithi152 ceph-mon[93456]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:07:26.609 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:27.392 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_1' removed 2024-08-17T17:07:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1790653007' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/491646196' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:07:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:07:27.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:27.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:27 smithi152 ceph-mon[93456]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1790653007' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/491646196' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[103614]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1790653007' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/491646196' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:07:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:07:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:27 smithi142 ceph-mon[106729]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:28 smithi152 ceph-mon[93456]: pgmap v145: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 87 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 242 objects/s recovering 2024-08-17T17:07:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:07:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:28 smithi152 ceph-mon[93456]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:07:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:28 smithi142 ceph-mon[103614]: pgmap v145: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 87 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 242 objects/s recovering 2024-08-17T17:07:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:07:28.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:28 smithi142 ceph-mon[103614]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:07:28.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:28 smithi142 ceph-mon[106729]: pgmap v145: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 87 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 242 objects/s recovering 2024-08-17T17:07:28.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:07:28.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:28 smithi142 ceph-mon[106729]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:07:29.482 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/894207338' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:29.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/894207338' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:29.736 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:07:29.737 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:07:29.737 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-08-17T17:07:29.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-08-17T17:07:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/894207338' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:30.293 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:07:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:30 smithi152 ceph-mon[93456]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:30 smithi152 ceph-mon[93456]: from='client.44372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:30 smithi152 ceph-mon[93456]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:07:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:07:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:07:30.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:30 smithi142 ceph-mon[106729]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:30 smithi142 ceph-mon[106729]: from='client.44372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:30 smithi142 ceph-mon[106729]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:30 smithi142 ceph-mon[103614]: pgmap v147: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:30 smithi142 ceph-mon[103614]: from='client.44372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:30 smithi142 ceph-mon[103614]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:32.061 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:07:31] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-08-17T17:07:32.347 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:32 smithi142 systemd[1]: Stopping Ceph osd.1 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:07:32.354 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:32.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:32 smithi142 ceph-mon[106729]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:32.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3157775021' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:32.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:32 smithi142 ceph-mon[103614]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:32.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3157775021' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:32 smithi152 ceph-mon[93456]: pgmap v148: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3157775021' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:33.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[50321]: 2024-08-17T17:07:33.103+0000 7f4854af8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:07:33.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[50321]: 2024-08-17T17:07:33.103+0000 7f4854af8640 -1 osd.1 100 *** Got signal Terminated *** 2024-08-17T17:07:33.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[50321]: 2024-08-17T17:07:33.103+0000 7f4854af8640 -1 osd.1 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:07:33.407 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_2' created 2024-08-17T17:07:33.712 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[106729]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:33.712 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[106729]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:33.712 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[106729]: osd.1 marked itself down and dead 2024-08-17T17:07:33.712 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/421581250' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:07:33.712 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[103614]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:33.712 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[103614]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:33.713 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[103614]: osd.1 marked itself down and dead 2024-08-17T17:07:33.713 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:33 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/421581250' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:07:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:33 smithi152 ceph-mon[93456]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:33 smithi152 ceph-mon[93456]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:33 smithi152 ceph-mon[93456]: osd.1 marked itself down and dead 2024-08-17T17:07:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:33 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/421581250' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:07:33.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-08-17T17:07:34.291 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:33 smithi142 podman[122126]: 2024-08-17 17:07:33.982667806 +0000 UTC m=+0.977057749 container died 5cd1d20f8eca672efa47af2e87170ceac934841d4bc632775706b1ee02ee626e (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:07:34.600 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[106729]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/421581250' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[106729]: osdmap e101: 8 total, 7 up, 8 in 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[103614]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/421581250' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:07:34.601 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:34 smithi142 ceph-mon[103614]: osdmap e101: 8 total, 7 up, 8 in 2024-08-17T17:07:34.602 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:34 smithi142 podman[122126]: 2024-08-17 17:07:34.39819633 +0000 UTC m=+1.392586280 container cleanup 5cd1d20f8eca672efa47af2e87170ceac934841d4bc632775706b1ee02ee626e (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:07:34.602 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:34 smithi142 bash[122126]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:07:34.684 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:07:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:34 smithi152 ceph-mon[93456]: pgmap v149: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:07:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:34 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/421581250' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:07:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:34 smithi152 ceph-mon[93456]: osdmap e101: 8 total, 7 up, 8 in 2024-08-17T17:07:34.872 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:34 smithi142 podman[122172]: 2024-08-17 17:07:34.599935286 +0000 UTC m=+0.695835648 container remove 5cd1d20f8eca672efa47af2e87170ceac934841d4bc632775706b1ee02ee626e (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:07:35.240 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:35.525 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:35 smithi142 ceph-mon[106729]: osdmap e102: 8 total, 7 up, 8 in 2024-08-17T17:07:35.525 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2903146381' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:35.525 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:35 smithi142 ceph-mon[103614]: osdmap e102: 8 total, 7 up, 8 in 2024-08-17T17:07:35.525 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2903146381' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:35 smithi152 ceph-mon[93456]: osdmap e102: 8 total, 7 up, 8 in 2024-08-17T17:07:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2903146381' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:36.122 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:36 smithi142 podman[122474]: 2024-08-17 17:07:35.964471969 +0000 UTC m=+0.019991198 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:36.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[106729]: pgmap v152: 89 pgs: 5 active+undersized, 12 unknown, 8 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:07:36.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[106729]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:36.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:07:36.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:07:36.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[106729]: osdmap e103: 8 total, 7 up, 8 in 2024-08-17T17:07:36.499 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[106729]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:36.499 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:36 smithi142 podman[122474]: 2024-08-17 17:07:36.327639787 +0000 UTC m=+0.383159058 container create 25b9ead6341e02810e429abf36eac5bd911b3c94cd1fe9587b95282c0ae05982 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T17:07:36.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[103614]: pgmap v152: 89 pgs: 5 active+undersized, 12 unknown, 8 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:07:36.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[103614]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:36.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:07:36.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:07:36.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[103614]: osdmap e103: 8 total, 7 up, 8 in 2024-08-17T17:07:36.500 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:36 smithi142 ceph-mon[103614]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:36 smithi152 ceph-mon[93456]: pgmap v152: 89 pgs: 5 active+undersized, 12 unknown, 8 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:07:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:36 smithi152 ceph-mon[93456]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:36 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:07:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:36 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:07:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:36 smithi152 ceph-mon[93456]: osdmap e103: 8 total, 7 up, 8 in 2024-08-17T17:07:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:36 smithi152 ceph-mon[93456]: from='client.44420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:36.807 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:36 smithi142 podman[122474]: 2024-08-17 17:07:36.714661651 +0000 UTC m=+0.770180891 container init 25b9ead6341e02810e429abf36eac5bd911b3c94cd1fe9587b95282c0ae05982 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-17T17:07:36.807 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:36 smithi142 podman[122474]: 2024-08-17 17:07:36.721744849 +0000 UTC m=+0.777264079 container start 25b9ead6341e02810e429abf36eac5bd911b3c94cd1fe9587b95282c0ae05982 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:07:36.807 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:36 smithi142 podman[122474]: 2024-08-17 17:07:36.806205432 +0000 UTC m=+0.861724664 container attach 25b9ead6341e02810e429abf36eac5bd911b3c94cd1fe9587b95282c0ae05982 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:07:37.122 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:36 smithi142 podman[122474]: 2024-08-17 17:07:36.907912866 +0000 UTC m=+0.963432096 container died 25b9ead6341e02810e429abf36eac5bd911b3c94cd1fe9587b95282c0ae05982 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD) 2024-08-17T17:07:37.462 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-08-17T17:07:37.519 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:37.520 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1877943669' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:07:37.520 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:37.520 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1877943669' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:07:37.779 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:37 smithi142 podman[122570]: 2024-08-17 17:07:37.519053453 +0000 UTC m=+0.605981291 container remove 25b9ead6341e02810e429abf36eac5bd911b3c94cd1fe9587b95282c0ae05982 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:07:37.779 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:37 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.1.service: Deactivated successfully. 2024-08-17T17:07:37.779 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:37 smithi142 systemd[1]: Stopped Ceph osd.1 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:07:37.779 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:37 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.1.service: Consumed 10min 24.961s CPU time. 2024-08-17T17:07:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1877943669' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:07:38.122 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:37 smithi142 systemd[1]: Starting Ceph osd.1 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:07:38.140 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:38.451 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 podman[122749]: 2024-08-17 17:07:38.235468764 +0000 UTC m=+0.020118500 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:38.452 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 podman[122749]: 2024-08-17 17:07:38.340447687 +0000 UTC m=+0.125097408 container create ffb7d3fc8729334dc4c0a7857a68ae810ea12ae1697e0382488c2198aad2fb75 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:07:38.709 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[106729]: pgmap v154: 89 pgs: 5 active+undersized, 12 unknown, 8 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1877943669' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[106729]: osdmap e104: 8 total, 7 up, 8 in 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3995499362' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[103614]: pgmap v154: 89 pgs: 5 active+undersized, 12 unknown, 8 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1877943669' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[103614]: osdmap e104: 8 total, 7 up, 8 in 2024-08-17T17:07:38.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3995499362' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 podman[122749]: 2024-08-17 17:07:38.560063576 +0000 UTC m=+0.344713314 container init ffb7d3fc8729334dc4c0a7857a68ae810ea12ae1697e0382488c2198aad2fb75 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 podman[122749]: 2024-08-17 17:07:38.564804756 +0000 UTC m=+0.349454478 container start ffb7d3fc8729334dc4c0a7857a68ae810ea12ae1697e0382488c2198aad2fb75 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS) 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 podman[122749]: 2024-08-17 17:07:38.643142266 +0000 UTC m=+0.427792003 container attach ffb7d3fc8729334dc4c0a7857a68ae810ea12ae1697e0382488c2198aad2fb75 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:38.711 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:38 smithi152 ceph-mon[93456]: pgmap v154: 89 pgs: 5 active+undersized, 12 unknown, 8 stale+active+clean, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:07:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1877943669' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:07:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:38 smithi152 ceph-mon[93456]: osdmap e104: 8 total, 7 up, 8 in 2024-08-17T17:07:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3995499362' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:39.086 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:07:39.086 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:07:39.086 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:39.086 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:39.086 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:39.087 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:39.087 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:07:39.087 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:07:39.087 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:07:39.087 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:38 smithi142 bash[122749]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:07:39.165 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-08-17T17:07:39.167 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:07:39.168 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:07:39.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:07:39.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:07:39.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_122928 2024-08-17T17:07:39.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:07:39.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:07:39.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:07:39.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 bash[122749]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:07:39.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:07:39.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 bash[122749]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 bash[122749]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 bash[122749]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate[122785]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 bash[122749]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:07:39.374 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 podman[122749]: 2024-08-17 17:07:39.11393504 +0000 UTC m=+0.898584765 container died ffb7d3fc8729334dc4c0a7857a68ae810ea12ae1697e0382488c2198aad2fb75 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-08-17T17:07:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:39 smithi152 ceph-mon[93456]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:39 smithi152 ceph-mon[93456]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:39.841 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-mon[106729]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:39.842 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-mon[106729]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:39.842 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-mon[103614]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:39.842 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:39 smithi142 ceph-mon[103614]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:40.123 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:39 smithi142 podman[122917]: 2024-08-17 17:07:39.840600284 +0000 UTC m=+0.720511514 container remove ffb7d3fc8729334dc4c0a7857a68ae810ea12ae1697e0382488c2198aad2fb75 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-activate, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-08-17T17:07:40.216 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 4512 4496 280.984 281 0.00606503 0.00355342 2024-08-17T17:07:40.474 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:07:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:07:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:07:40.475 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 podman[123014]: 2024-08-17 17:07:40.06396017 +0000 UTC m=+0.038741458 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:40.475 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 podman[123014]: 2024-08-17 17:07:40.208392842 +0000 UTC m=+0.183174108 container create d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:07:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:40 smithi152 ceph-mon[93456]: pgmap v156: 89 pgs: 34 active+undersized, 4 active+undersized+degraded, 51 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:07:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:40 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:07:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:40 smithi142 ceph-mon[103614]: pgmap v156: 89 pgs: 34 active+undersized, 4 active+undersized+degraded, 51 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:07:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:40 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:07:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:40 smithi142 ceph-mon[106729]: pgmap v156: 89 pgs: 34 active+undersized, 4 active+undersized+degraded, 51 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:07:40.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:40 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:07:40.874 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 podman[123014]: 2024-08-17 17:07:40.611722224 +0000 UTC m=+0.586503495 container init d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-17T17:07:40.874 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 podman[123014]: 2024-08-17 17:07:40.620374101 +0000 UTC m=+0.595155375 container start d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:07:40.874 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 ceph-osd[123072]: -- 172.21.15.142:0/2000162613 <== mon.0 v2:172.21.15.142:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56371c5f3a40 con 0x56371d3c6400 2024-08-17T17:07:40.874 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 bash[123014]: d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 2024-08-17T17:07:40.874 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:40 smithi142 systemd[1]: Started Ceph osd.1 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:07:41.214 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 8091 8075 252.326 223.688 0.00290001 0.00395216 2024-08-17T17:07:41.581 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:41.787 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:07:41] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.51.0" 2024-08-17T17:07:41.788 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[123031]: 2024-08-17T17:07:41.528+0000 7f1a79726640 -1 Falling back to public interface 2024-08-17T17:07:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3853306094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3853306094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3853306094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:42.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 11078 11062 230.441 186.688 0.00156535 0.00433203 2024-08-17T17:07:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:42 smithi152 ceph-mon[93456]: pgmap v157: 89 pgs: 32 active+undersized, 6 active+undersized+degraded, 51 active+clean; 56 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s wr, 131 op/s; 442/2733 objects degraded (16.173%) 2024-08-17T17:07:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:42 smithi152 ceph-mon[93456]: from='client.54382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:42 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 442/2733 objects degraded (16.173%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:43.106 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:42 smithi142 ceph-mon[106729]: pgmap v157: 89 pgs: 32 active+undersized, 6 active+undersized+degraded, 51 active+clean; 56 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s wr, 131 op/s; 442/2733 objects degraded (16.173%) 2024-08-17T17:07:43.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:42 smithi142 ceph-mon[106729]: from='client.54382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:43.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:42 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 442/2733 objects degraded (16.173%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:43.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:42 smithi142 ceph-mon[103614]: pgmap v157: 89 pgs: 32 active+undersized, 6 active+undersized+degraded, 51 active+clean; 56 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s wr, 131 op/s; 442/2733 objects degraded (16.173%) 2024-08-17T17:07:43.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:42 smithi142 ceph-mon[103614]: from='client.54382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:43.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:42 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 442/2733 objects degraded (16.173%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:43.215 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 15 13983 13968 218.232 181.625 0.00292899 0.00457465 2024-08-17T17:07:44.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:43 smithi152 ceph-mon[93456]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:44.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:43 smithi142 ceph-mon[106729]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:44.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:43 smithi142 ceph-mon[103614]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:44.217 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 16888 16872 210.838 181.5 0.00761118 0.00473472 2024-08-17T17:07:44.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:44 smithi142 ceph-mon[103614]: pgmap v158: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 597 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 3316/28692 objects degraded (11.557%) 2024-08-17T17:07:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:44 smithi152 ceph-mon[93456]: pgmap v158: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 597 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 3316/28692 objects degraded (11.557%) 2024-08-17T17:07:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/768352663' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:44 smithi142 ceph-mon[106729]: pgmap v158: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 597 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 3316/28692 objects degraded (11.557%) 2024-08-17T17:07:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/768352663' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/768352663' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:45.219 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 19389 19373 201.629 156.312 0.00802944 0.00495318 2024-08-17T17:07:45.279 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:46.219 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 21818 21802 194.515 151.812 0.00251352 0.00513538 2024-08-17T17:07:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:46 smithi152 ceph-mon[93456]: pgmap v159: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 833 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s; 5195/40032 objects degraded (12.977%) 2024-08-17T17:07:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:46 smithi152 ceph-mon[93456]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:46 smithi152 ceph-mon[93456]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:47.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[103614]: pgmap v159: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 833 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s; 5195/40032 objects degraded (12.977%) 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[103614]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[103614]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[106729]: pgmap v159: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 833 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s; 5195/40032 objects degraded (12.977%) 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[106729]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:46 smithi142 ceph-mon[106729]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:47.220 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 15 24146 24131 188.398 145.562 0.00418852 0.00530091 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:47.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:48.220 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 15 26542 26527 184.105 149.75 0.00473797 0.00542608 2024-08-17T17:07:48.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:47 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[123031]: 2024-08-17T17:07:47.922+0000 7f1a79726640 -1 osd.1 100 log_to_monitors true 2024-08-17T17:07:48.930 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:49.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:48 smithi142 ceph-mon[106729]: pgmap v160: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 833 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 5195/40032 objects degraded (12.977%) 2024-08-17T17:07:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:48 smithi142 ceph-mon[106729]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:07:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1758764645' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:48 smithi142 ceph-mon[103614]: pgmap v160: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 833 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 5195/40032 objects degraded (12.977%) 2024-08-17T17:07:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:48 smithi142 ceph-mon[103614]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:07:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1758764645' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:49.220 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 28422 28406 177.44 117.438 0.0389528 0.00561459 2024-08-17T17:07:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:48 smithi152 ceph-mon[93456]: pgmap v160: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 833 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 5195/40032 objects degraded (12.977%) 2024-08-17T17:07:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:48 smithi152 ceph-mon[93456]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:07:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1758764645' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:49.872 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[123031]: 2024-08-17T17:07:49.463+0000 7f1a6d4c8640 -1 osd.1 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:07:50.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:49 smithi152 ceph-mon[93456]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:49 smithi152 ceph-mon[93456]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:07:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:49 smithi152 ceph-mon[93456]: osdmap e105: 8 total, 7 up, 8 in 2024-08-17T17:07:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:49 smithi152 ceph-mon[93456]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:49 smithi152 ceph-mon[93456]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' 2024-08-17T17:07:50.221 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 30788 30772 174.734 147.875 0.00790548 0.00571707 2024-08-17T17:07:50.245 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[106729]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:50.245 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[106729]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:07:50.245 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[106729]: osdmap e105: 8 total, 7 up, 8 in 2024-08-17T17:07:50.245 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[106729]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:50.245 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[106729]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' 2024-08-17T17:07:50.246 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[103614]: from='client.34662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:50.246 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[103614]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:07:50.246 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[103614]: osdmap e105: 8 total, 7 up, 8 in 2024-08-17T17:07:50.246 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[103614]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:07:50.246 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:49 smithi142 ceph-mon[103614]: from='osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993]' entity='osd.1' 2024-08-17T17:07:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:07:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:07:51.221 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 31255 31239 162.611 29.1875 0.00161504 0.00576929 2024-08-17T17:07:51.293 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: pgmap v162: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 8405/69228 objects degraded (12.141%) 2024-08-17T17:07:51.293 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:51.293 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8405/69228 objects degraded (12.141%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:51.293 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:51.293 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993] boot 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: pgmap v162: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 8405/69228 objects degraded (12.141%) 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8405/69228 objects degraded (12.141%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993] boot 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T17:07:51.294 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: pgmap v162: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s; 8405/69228 objects degraded (12.141%) 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8405/69228 objects degraded (12.141%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: osd.1 [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993] boot 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T17:07:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:07:51.913 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:07:51] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.51.0" 2024-08-17T17:07:52.221 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 31255 31239 150.108 0 - 0.00576929 2024-08-17T17:07:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:52 smithi152 ceph-mon[93456]: pgmap v164: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 9836/77688 objects degraded (12.661%) 2024-08-17T17:07:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:52 smithi152 ceph-mon[93456]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T17:07:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1231678343' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:52 smithi152 ceph-mon[93456]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:52.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[103614]: pgmap v164: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 9836/77688 objects degraded (12.661%) 2024-08-17T17:07:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[103614]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T17:07:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1231678343' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[103614]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[106729]: pgmap v164: 89 pgs: 28 active+undersized, 10 active+undersized+degraded, 51 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 9836/77688 objects degraded (12.661%) 2024-08-17T17:07:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[106729]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T17:07:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1231678343' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:52.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:52 smithi142 ceph-mon[106729]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:53.221 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 32147 32131 143.37 27.875 0.0101461 0.00696882 2024-08-17T17:07:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:53 smithi152 ceph-mon[93456]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:53 smithi152 ceph-mon[93456]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T17:07:53.844 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:53 smithi142 ceph-mon[103614]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:53.845 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:53 smithi142 ceph-mon[103614]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T17:07:53.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:53 smithi142 ceph-mon[106729]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:53.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:53 smithi142 ceph-mon[106729]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T17:07:54.221 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 33659 33643 140.113 94.5 0.00689622 0.00713202 2024-08-17T17:07:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:54 smithi152 ceph-mon[93456]: pgmap v167: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 11 active+undersized, 3 active+undersized+degraded, 71 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 5768/93012 objects degraded (6.201%) 2024-08-17T17:07:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:07:54.560+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-08-17T17:07:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[106729]: pgmap v167: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 11 active+undersized, 3 active+undersized+degraded, 71 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 5768/93012 objects degraded (6.201%) 2024-08-17T17:07:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[103614]: pgmap v167: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 11 active+undersized, 3 active+undersized+degraded, 71 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 5768/93012 objects degraded (6.201%) 2024-08-17T17:07:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:55.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 35017 35001 136.661 84.875 0.0103522 0.00731064 2024-08-17T17:07:55.330 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3893897555' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5768/93012 objects degraded (6.201%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3893897555' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:55.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5768/93012 objects degraded (6.201%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:07:55.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:07:55.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-08-17T17:07:55.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3893897555' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:55.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5768/93012 objects degraded (6.201%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:56.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 36486 36470 134.024 91.8125 0.0138855 0.00745526 2024-08-17T17:07:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:56 smithi152 ceph-mon[93456]: pgmap v168: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5777/96342 objects degraded (5.996%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:07:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:56 smithi152 ceph-mon[93456]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:56 smithi152 ceph-mon[93456]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[106729]: pgmap v168: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5777/96342 objects degraded (5.996%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[106729]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[106729]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[103614]: pgmap v168: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5777/96342 objects degraded (5.996%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[103614]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[103614]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:07:56.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:57.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 38021 38005 131.908 95.9375 0.00856028 0.00757558 2024-08-17T17:07:58.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 39458 39442 129.692 89.8125 0.0118595 0.00770334 2024-08-17T17:07:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:58 smithi152 ceph-mon[93456]: pgmap v169: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 5777/96342 objects degraded (5.996%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:07:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:07:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1752901861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:58 smithi142 ceph-mon[103614]: pgmap v169: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 5777/96342 objects degraded (5.996%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:07:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:07:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1752901861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:58 smithi142 ceph-mon[106729]: pgmap v169: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 5777/96342 objects degraded (5.996%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:07:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:07:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1752901861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:07:58.993 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:07:59.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:07:59.221946+0000 min lat: 0.00116924 max lat: 2.18324 avg lat: 0.00784231 2024-08-17T17:07:59.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:07:59.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 40801 40785 127.405 83.9375 0.0100313 0.00784231 2024-08-17T17:08:00.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 42158 42142 125.377 84.8125 0.00956182 0.00796944 2024-08-17T17:08:00.561 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:08:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:08:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:08:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:00 smithi152 ceph-mon[93456]: from='client.34701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:00 smithi152 ceph-mon[93456]: pgmap v170: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 11539/112179 objects degraded (10.286%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:08:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:00 smithi152 ceph-mon[93456]: from='client.54442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11539/112179 objects degraded (10.286%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[106729]: from='client.34701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[106729]: pgmap v170: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 11539/112179 objects degraded (10.286%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[106729]: from='client.54442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11539/112179 objects degraded (10.286%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[103614]: from='client.34701 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[103614]: pgmap v170: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 11539/112179 objects degraded (10.286%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[103614]: from='client.54442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11539/112179 objects degraded (10.286%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:01.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 15 43646 43631 123.908 93.0625 0.00665898 0.00806445 2024-08-17T17:08:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:08:01] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.51.0" 2024-08-17T17:08:02.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 44977 44961 122.135 83.125 0.0071699 0.00818145 2024-08-17T17:08:02.619 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:02 smithi152 ceph-mon[93456]: pgmap v171: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 11284/117573 objects degraded (9.597%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:08:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1355635469' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:02 smithi142 ceph-mon[106729]: pgmap v171: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 11284/117573 objects degraded (9.597%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:08:02.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1355635469' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:02.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:02 smithi142 ceph-mon[103614]: pgmap v171: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 11284/117573 objects degraded (9.597%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:08:02.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1355635469' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:03.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 46266 46250 120.403 80.5625 0.00868755 0.00829833 2024-08-17T17:08:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:03 smithi142 ceph-mon[106729]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:03 smithi142 ceph-mon[103614]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:03 smithi152 ceph-mon[93456]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:04.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 47699 47683 119.169 89.5625 0.00952357 0.00838454 2024-08-17T17:08:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:04 smithi142 ceph-mon[106729]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:04.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:04 smithi142 ceph-mon[106729]: pgmap v172: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 11284/132999 objects degraded (8.484%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:08:04.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:04 smithi142 ceph-mon[103614]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:04.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:04 smithi142 ceph-mon[103614]: pgmap v172: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 11284/132999 objects degraded (8.484%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:08:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:04 smithi152 ceph-mon[93456]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:04 smithi152 ceph-mon[93456]: pgmap v172: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 11284/132999 objects degraded (8.484%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:08:05.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 49027 49011 117.778 83 0.0601506 0.00848029 2024-08-17T17:08:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11284/132999 objects degraded (8.484%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11284/132999 objects degraded (8.484%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11284/132999 objects degraded (8.484%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:06.222 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 50254 50238 116.256 76.6875 0.00954842 0.00859616 2024-08-17T17:08:06.247 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:06 smithi142 ceph-mon[103614]: pgmap v173: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 11000/138213 objects degraded (7.959%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:08:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1042836708' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:06 smithi142 ceph-mon[106729]: pgmap v173: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 11000/138213 objects degraded (7.959%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:08:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1042836708' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:06 smithi152 ceph-mon[93456]: pgmap v173: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 11000/138213 objects degraded (7.959%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:08:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1042836708' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:07.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 51507 51491 114.901 78.3125 0.00716716 0.00869777 2024-08-17T17:08:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:07 smithi152 ceph-mon[93456]: from='client.54463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:07 smithi152 ceph-mon[93456]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:07 smithi142 ceph-mon[103614]: from='client.54463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:07 smithi142 ceph-mon[103614]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:07.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:07 smithi142 ceph-mon[106729]: from='client.54463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:07.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:07 smithi142 ceph-mon[106729]: from='client.34737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:08.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 52862 52846 113.859 84.6875 0.0202084 0.00877623 2024-08-17T17:08:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:08 smithi142 ceph-mon[106729]: pgmap v174: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 11000/138213 objects degraded (7.959%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:08:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:08 smithi142 ceph-mon[103614]: pgmap v174: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 11000/138213 objects degraded (7.959%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:08:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:08 smithi152 ceph-mon[93456]: pgmap v174: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 11000/138213 objects degraded (7.959%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:08:09.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 54125 54109 112.695 78.9375 0.00979672 0.00886644 2024-08-17T17:08:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3619426387' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:09.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:08:09.579+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3619426387' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:09.893 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3619426387' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:10.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 55274 55258 111.376 71.8125 0.0176505 0.00897242 2024-08-17T17:08:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:08:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:08:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:08:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:10 smithi152 ceph-mon[93456]: pgmap v175: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 11000/153033 objects degraded (7.188%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:08:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:10 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:10 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:10 smithi152 ceph-mon[93456]: from='client.54475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:11.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11000/153033 objects degraded (7.188%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[106729]: pgmap v175: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 11000/153033 objects degraded (7.188%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:08:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[106729]: from='client.54475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:11.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11000/153033 objects degraded (7.188%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:11.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[103614]: pgmap v175: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 11000/153033 objects degraded (7.188%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:08:11.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:11.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:11.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[103614]: from='client.54475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:11.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11000/153033 objects degraded (7.188%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:11.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 15 56517 56502 110.325 77.75 0.0127431 0.00905874 2024-08-17T17:08:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:11 smithi152 ceph-mon[93456]: from='client.34752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:11 smithi142 ceph-mon[106729]: from='client.34752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:08:11] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:08:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:11 smithi142 ceph-mon[103614]: from='client.34752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:12.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 57705 57689 109.23 74.1875 0.00688646 0.00914876 2024-08-17T17:08:12.858 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:12 smithi142 ceph-mon[106729]: pgmap v176: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 10695/157560 objects degraded (6.788%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:08:12.858 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:12 smithi142 ceph-mon[103614]: pgmap v176: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 10695/157560 objects degraded (6.788%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:08:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:12 smithi152 ceph-mon[93456]: pgmap v176: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 10695/157560 objects degraded (6.788%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:08:13.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 58680 58664 107.81 60.9375 0.0228919 0.00926892 2024-08-17T17:08:13.555 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2351938860' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2351938860' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2351938860' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:14.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 59644 59628 106.451 60.25 0.0174172 0.00938834 2024-08-17T17:08:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:14 smithi152 ceph-mon[93456]: pgmap v177: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.50k op/s; 10695/171447 objects degraded (6.238%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:08:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:14 smithi152 ceph-mon[93456]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:14 smithi142 ceph-mon[106729]: pgmap v177: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.50k op/s; 10695/171447 objects degraded (6.238%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:08:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:14 smithi142 ceph-mon[106729]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:14 smithi142 ceph-mon[103614]: pgmap v177: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.50k op/s; 10695/171447 objects degraded (6.238%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:08:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:14 smithi142 ceph-mon[103614]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:15.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 60904 60888 105.682 78.75 0.00441578 0.00945619 2024-08-17T17:08:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:15 smithi152 ceph-mon[93456]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10695/171447 objects degraded (6.238%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:15 smithi142 ceph-mon[106729]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10695/171447 objects degraded (6.238%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:15 smithi142 ceph-mon[103614]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10695/171447 objects degraded (6.238%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:16.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 62124 62108 104.886 76.25 0.00597709 0.00952686 2024-08-17T17:08:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:16 smithi152 ceph-mon[93456]: pgmap v178: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 10377/175659 objects degraded (5.907%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:08:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1511911203' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:16 smithi142 ceph-mon[106729]: pgmap v178: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 10377/175659 objects degraded (5.907%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:08:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1511911203' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:16 smithi142 ceph-mon[103614]: pgmap v178: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 10377/175659 objects degraded (5.907%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:08:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1511911203' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:17.214 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:17.223 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 63194 63178 103.886 66.875 0.0115689 0.00961997 2024-08-17T17:08:18.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 64323 64307 103.031 70.5625 0.00851722 0.00969952 2024-08-17T17:08:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:18 smithi152 ceph-mon[93456]: pgmap v179: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10377/175659 objects degraded (5.907%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:08:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:18 smithi152 ceph-mon[93456]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:18 smithi152 ceph-mon[93456]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:18 smithi142 ceph-mon[106729]: pgmap v179: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10377/175659 objects degraded (5.907%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:08:19.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:18 smithi142 ceph-mon[106729]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:19.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:18 smithi142 ceph-mon[106729]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:18 smithi142 ceph-mon[103614]: pgmap v179: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10377/175659 objects degraded (5.907%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:08:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:18 smithi142 ceph-mon[103614]: from='client.34773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:18 smithi142 ceph-mon[103614]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:19.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:08:19.223825+0000 min lat: 0.00116924 max lat: 2.18324 avg lat: 0.00976382 2024-08-17T17:08:19.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:08:19.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 65538 65522 102.354 75.9375 0.0110589 0.00976382 2024-08-17T17:08:20.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 66671 66655 101.585 70.8125 0.00821331 0.00983845 2024-08-17T17:08:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:08:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:08:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:08:20.893 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:20 smithi152 ceph-mon[93456]: pgmap v180: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 10377/187947 objects degraded (5.521%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:08:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10377/187947 objects degraded (5.521%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2306571582' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:20 smithi142 ceph-mon[103614]: pgmap v180: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 10377/187947 objects degraded (5.521%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:08:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10377/187947 objects degraded (5.521%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2306571582' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:20 smithi142 ceph-mon[106729]: pgmap v180: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 10377/187947 objects degraded (5.521%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:08:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10377/187947 objects degraded (5.521%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2306571582' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:21.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 67700 67684 100.697 64.3125 0.0194569 0.00992409 2024-08-17T17:08:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:21 smithi152 ceph-mon[93456]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:21 smithi142 ceph-mon[103614]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:22.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:08:21] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:08:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:21 smithi142 ceph-mon[106729]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:22.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 68846 68830 100.021 71.625 0.01229 0.00999237 2024-08-17T17:08:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:22 smithi152 ceph-mon[93456]: pgmap v181: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10018/192483 objects degraded (5.205%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:08:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:22 smithi152 ceph-mon[93456]: from='client.34791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:22 smithi142 ceph-mon[106729]: pgmap v181: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10018/192483 objects degraded (5.205%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:08:23.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:22 smithi142 ceph-mon[106729]: from='client.34791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:22 smithi142 ceph-mon[103614]: pgmap v181: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 10018/192483 objects degraded (5.205%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:08:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:22 smithi142 ceph-mon[103614]: from='client.34791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:23.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 70032 70016 99.4325 74.125 0.0113061 0.0100512 2024-08-17T17:08:24.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 70760 70744 98.234 45.5 0.00280369 0.0100844 2024-08-17T17:08:24.258 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:24.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:08:24.596+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[106729]: pgmap v182: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 10018/205275 objects degraded (4.880%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:08:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[106729]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:08:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/430650556' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:24.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[103614]: pgmap v182: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 10018/205275 objects degraded (4.880%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:08:24.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[103614]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:08:24.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/430650556' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:24.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:24 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:24 smithi152 ceph-mon[93456]: pgmap v182: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 10018/205275 objects degraded (4.880%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:08:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:24 smithi152 ceph-mon[93456]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:08:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/430650556' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:24 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:25.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 71342 71326 96.8894 36.375 0.0167215 0.010315 2024-08-17T17:08:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:25 smithi152 ceph-mon[93456]: from='client.44609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:25 smithi152 ceph-mon[93456]: from='client.44615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:25 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:25 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:25 smithi152 ceph-mon[93456]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:08:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10018/205275 objects degraded (4.880%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[106729]: from='client.44609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[106729]: from='client.44615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[106729]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10018/205275 objects degraded (4.880%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[103614]: from='client.44609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[103614]: from='client.44615 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:26.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[103614]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:08:26.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10018/205275 objects degraded (4.880%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:26.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 72515 72499 96.3877 73.3125 0.0117371 0.0103688 2024-08-17T17:08:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:26 smithi152 ceph-mon[93456]: pgmap v185: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 9748/208662 objects degraded (4.672%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:08:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:26 smithi142 ceph-mon[106729]: pgmap v185: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 9748/208662 objects degraded (4.672%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:08:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:26 smithi142 ceph-mon[103614]: pgmap v185: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 9748/208662 objects degraded (4.672%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:08:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:27.224 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 73571 73555 95.7545 66 0.0116687 0.010437 2024-08-17T17:08:27.630 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:28.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/936608890' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/936608890' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/936608890' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:28.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 15 74555 74540 95.0567 61.5625 0.0176072 0.0105137 2024-08-17T17:08:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:28 smithi152 ceph-mon[93456]: pgmap v186: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 9748/208662 objects degraded (4.672%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:08:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:28 smithi152 ceph-mon[93456]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:28 smithi142 ceph-mon[106729]: pgmap v186: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 9748/208662 objects degraded (4.672%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:08:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:28 smithi142 ceph-mon[106729]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:28 smithi142 ceph-mon[103614]: pgmap v186: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 9748/208662 objects degraded (4.672%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:08:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:28 smithi142 ceph-mon[103614]: from='client.44621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:29.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 15 75705 75690 94.593 71.875 0.0165859 0.0105655 2024-08-17T17:08:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:29 smithi152 ceph-mon[93456]: from='client.54532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:29 smithi142 ceph-mon[106729]: from='client.54532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:29 smithi142 ceph-mon[103614]: from='client.54532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:30.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 76821 76805 94.1046 69.6875 0.0185405 0.0106209 2024-08-17T17:08:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:08:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:08:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:08:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:30 smithi152 ceph-mon[93456]: pgmap v187: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 9395/219366 objects degraded (4.283%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 9395/219366 objects degraded (4.283%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3511649057' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:30 smithi142 ceph-mon[103614]: pgmap v187: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 9395/219366 objects degraded (4.283%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 9395/219366 objects degraded (4.283%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3511649057' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:30 smithi142 ceph-mon[106729]: pgmap v187: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 9395/219366 objects degraded (4.283%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 9395/219366 objects degraded (4.283%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3511649057' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:31.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 77816 77800 93.4908 62.1875 0.0109303 0.0106908 2024-08-17T17:08:31.264 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:08:31] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.51.0" 2024-08-17T17:08:32.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 78846 78830 92.9414 64.375 0.0190053 0.0107542 2024-08-17T17:08:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:32 smithi152 ceph-mon[93456]: pgmap v188: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 9395/223422 objects degraded (4.205%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:32 smithi152 ceph-mon[93456]: from='client.54541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:32 smithi152 ceph-mon[93456]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:32 smithi142 ceph-mon[106729]: pgmap v188: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 9395/223422 objects degraded (4.205%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:32 smithi142 ceph-mon[106729]: from='client.54541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:32 smithi142 ceph-mon[106729]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:32 smithi142 ceph-mon[103614]: pgmap v188: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 9395/223422 objects degraded (4.205%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:32 smithi142 ceph-mon[103614]: from='client.54541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:32 smithi142 ceph-mon[103614]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:33.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 79961 79945 92.5107 69.6875 0.00564758 0.0108043 2024-08-17T17:08:34.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 80992 80976 92.0002 64.4375 0.00834691 0.0108629 2024-08-17T17:08:34.885 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:34 smithi152 ceph-mon[93456]: pgmap v189: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 9050/235398 objects degraded (3.845%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:08:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1606613151' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:34 smithi142 ceph-mon[106729]: pgmap v189: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 9050/235398 objects degraded (3.845%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:08:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1606613151' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:34 smithi142 ceph-mon[103614]: pgmap v189: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 9050/235398 objects degraded (3.845%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:08:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1606613151' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:35.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 81943 81927 91.4187 59.4375 0.0165734 0.0109322 2024-08-17T17:08:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:35 smithi152 ceph-mon[93456]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 9050/235398 objects degraded (3.845%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:35 smithi142 ceph-mon[106729]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 9050/235398 objects degraded (3.845%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:35 smithi142 ceph-mon[103614]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 9050/235398 objects degraded (3.845%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:36.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 83006 82990 90.9803 66.4375 0.00998533 0.0109858 2024-08-17T17:08:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:36 smithi152 ceph-mon[93456]: pgmap v190: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 9050/239193 objects degraded (3.784%); 5.8 MiB/s, 92 objects/s recovering 2024-08-17T17:08:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:36 smithi152 ceph-mon[93456]: from='client.44642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:36 smithi142 ceph-mon[106729]: pgmap v190: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 9050/239193 objects degraded (3.784%); 5.8 MiB/s, 92 objects/s recovering 2024-08-17T17:08:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:36 smithi142 ceph-mon[106729]: from='client.44642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:36 smithi142 ceph-mon[103614]: pgmap v190: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 9050/239193 objects degraded (3.784%); 5.8 MiB/s, 92 objects/s recovering 2024-08-17T17:08:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:36 smithi142 ceph-mon[103614]: from='client.44642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:37.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 84080 84064 90.569 67.125 0.0135949 0.0110357 2024-08-17T17:08:38.225 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 85025 85009 90.0349 59.0625 0.0150514 0.0111012 2024-08-17T17:08:38.510 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:38 smithi152 ceph-mon[93456]: pgmap v191: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9050/239193 objects degraded (3.784%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:08:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1602690843' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:38 smithi142 ceph-mon[106729]: pgmap v191: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9050/239193 objects degraded (3.784%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:08:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1602690843' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:38 smithi142 ceph-mon[103614]: pgmap v191: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9050/239193 objects degraded (3.784%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:08:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1602690843' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:39.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:08:39.225654+0000 min lat: 0.00116924 max lat: 2.18324 avg lat: 0.0111646 2024-08-17T17:08:39.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:08:39.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 85974 85958 89.5228 59.3125 0.0116199 0.0111646 2024-08-17T17:08:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:39 smithi142 ceph-mon[103614]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:39 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:39.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:08:39.613+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:39.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:39 smithi142 ceph-mon[106729]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:39.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:39 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:39 smithi152 ceph-mon[93456]: from='client.54571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:39 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:08:40.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 87009 86993 89.1156 64.6875 0.01303 0.0112143 2024-08-17T17:08:40.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:08:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:08:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:08:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:40 smithi152 ceph-mon[93456]: from='client.54577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:40 smithi152 ceph-mon[93456]: pgmap v192: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s; 8690/250977 objects degraded (3.462%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:08:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:40 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:40 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8690/250977 objects degraded (3.462%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[103614]: from='client.54577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[103614]: pgmap v192: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s; 8690/250977 objects degraded (3.462%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:08:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8690/250977 objects degraded (3.462%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:41.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[106729]: from='client.54577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:41.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[106729]: pgmap v192: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s; 8690/250977 objects degraded (3.462%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:08:41.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:08:41.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:08:41.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8690/250977 objects degraded (3.462%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:41.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 87998 87982 88.6751 61.8125 0.0232576 0.0112709 2024-08-17T17:08:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1541597204' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:42.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:08:41] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:08:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:42.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1541597204' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:42.161 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:42.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 88860 88844 88.1227 53.875 0.0538803 0.0113406 2024-08-17T17:08:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1541597204' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:42.846 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:42 smithi142 ceph-mon[106729]: pgmap v193: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 8690/254673 objects degraded (3.412%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:08:42.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:42 smithi142 ceph-mon[103614]: pgmap v193: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 8690/254673 objects degraded (3.412%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:08:43.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 89686 89670 87.5524 51.625 0.0199685 0.011416 2024-08-17T17:08:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:42 smithi152 ceph-mon[93456]: pgmap v193: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 8690/254673 objects degraded (3.412%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:08:44.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:43 smithi142 ceph-mon[106729]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:44.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:43 smithi142 ceph-mon[106729]: from='client.34878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:44.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:43 smithi142 ceph-mon[103614]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:44.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:43 smithi142 ceph-mon[103614]: from='client.34878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:44.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 90583 90567 87.0679 56.0625 0.00752096 0.0114786 2024-08-17T17:08:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:43 smithi152 ceph-mon[93456]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:44.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:43 smithi152 ceph-mon[93456]: from='client.34878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:44 smithi142 ceph-mon[103614]: pgmap v194: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8311/265404 objects degraded (3.131%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:08:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:44 smithi142 ceph-mon[106729]: pgmap v194: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8311/265404 objects degraded (3.131%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:08:45.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 91531 91515 86.6464 59.25 0.00734705 0.0115345 2024-08-17T17:08:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:44 smithi152 ceph-mon[93456]: pgmap v194: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8311/265404 objects degraded (3.131%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:08:45.850 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3811778025' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8311/265404 objects degraded (3.131%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:46.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3811778025' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:46.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8311/265404 objects degraded (3.131%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:46.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 92401 92385 86.1647 54.375 0.00681491 0.0115992 2024-08-17T17:08:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3811778025' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8311/265404 objects degraded (3.131%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:47.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 93400 93384 85.8157 62.4375 0.0141082 0.0116469 2024-08-17T17:08:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:46 smithi152 ceph-mon[93456]: pgmap v195: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 8311/268788 objects degraded (3.092%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:08:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:46 smithi152 ceph-mon[93456]: from='client.54595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:46 smithi142 ceph-mon[103614]: pgmap v195: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 8311/268788 objects degraded (3.092%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:08:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:46 smithi142 ceph-mon[103614]: from='client.54595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:47.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:46 smithi142 ceph-mon[106729]: pgmap v195: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 8311/268788 objects degraded (3.092%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:08:47.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:46 smithi142 ceph-mon[106729]: from='client.54595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:48.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 94389 94373 85.4678 61.8125 0.0133406 0.0116937 2024-08-17T17:08:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:47 smithi152 ceph-mon[93456]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:48.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:47 smithi142 ceph-mon[106729]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:48.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:47 smithi142 ceph-mon[103614]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:49.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 95290 95274 85.0513 56.3125 0.00440364 0.0117271 2024-08-17T17:08:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:48 smithi152 ceph-mon[93456]: pgmap v196: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 8311/268788 objects degraded (3.092%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:08:49.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:48 smithi142 ceph-mon[106729]: pgmap v196: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 8311/268788 objects degraded (3.092%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:08:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:48 smithi142 ceph-mon[103614]: pgmap v196: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 8311/268788 objects degraded (3.092%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:08:49.379 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:49 smithi152 ceph-mon[93456]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:08:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3418657999' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:50.226 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 95639 95623 84.1606 21.8125 0.0128964 0.011876 2024-08-17T17:08:50.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:49 smithi142 ceph-mon[106729]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:08:50.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3418657999' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:50.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:49 smithi142 ceph-mon[103614]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:08:50.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3418657999' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:08:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:08:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:08:51.227 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 96627 96611 83.8493 61.75 0.0103916 0.0119206 2024-08-17T17:08:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:50 smithi152 ceph-mon[93456]: pgmap v198: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 7912/279249 objects degraded (2.833%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:50 smithi152 ceph-mon[93456]: from='client.54613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:50 smithi152 ceph-mon[93456]: from='client.34902 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:50 smithi152 ceph-mon[93456]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:08:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7912/279249 objects degraded (2.833%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[106729]: pgmap v198: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 7912/279249 objects degraded (2.833%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[106729]: from='client.54613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[106729]: from='client.34902 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[106729]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7912/279249 objects degraded (2.833%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[103614]: pgmap v198: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 7912/279249 objects degraded (2.833%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[103614]: from='client.54613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[103614]: from='client.34902 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[103614]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:08:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7912/279249 objects degraded (2.833%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:08:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:08:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:08:51] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:08:52.227 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 15 97548 97533 83.49 57.625 0.0180612 0.0119718 2024-08-17T17:08:52.803 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:53.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:52 smithi142 ceph-mon[106729]: pgmap v200: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 7912/282234 objects degraded (2.803%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:08:53.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/503193461' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:52 smithi142 ceph-mon[103614]: pgmap v200: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 7912/282234 objects degraded (2.803%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:08:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/503193461' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:53.227 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 98382 98366 83.0654 52.0625 0.0127722 0.0120326 2024-08-17T17:08:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:52 smithi152 ceph-mon[93456]: pgmap v200: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 7912/282234 objects degraded (2.803%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:08:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/503193461' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:54.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:53 smithi152 ceph-mon[93456]: from='client.54628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:54.227 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 99290 99274 82.7144 56.75 0.0179321 0.0120837 2024-08-17T17:08:54.310 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:53 smithi142 ceph-mon[106729]: from='client.54628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:54.310 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:53 smithi142 ceph-mon[103614]: from='client.54628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:55.227 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 100172 100156 82.3514 55.125 0.021957 0.0121368 2024-08-17T17:08:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:54 smithi152 ceph-mon[93456]: pgmap v201: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 7496/291423 objects degraded (2.572%); 6.4 MiB/s, 101 objects/s recovering 2024-08-17T17:08:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:54 smithi152 ceph-mon[93456]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:08:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:54 smithi142 ceph-mon[106729]: pgmap v201: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 7496/291423 objects degraded (2.572%); 6.4 MiB/s, 101 objects/s recovering 2024-08-17T17:08:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:54 smithi142 ceph-mon[106729]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:08:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:54 smithi142 ceph-mon[103614]: pgmap v201: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 7496/291423 objects degraded (2.572%); 6.4 MiB/s, 101 objects/s recovering 2024-08-17T17:08:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:54 smithi142 ceph-mon[103614]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:08:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7496/291423 objects degraded (2.572%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:08:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1572276132' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:56.227 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 100926 100910 81.8939 47.125 0.00969144 0.012205 2024-08-17T17:08:56.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7496/291423 objects degraded (2.572%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:08:56.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1572276132' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7496/291423 objects degraded (2.572%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:08:56.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1572276132' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:08:56.377 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:08:57.228 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 101737 101721 81.4931 50.6875 0.00611151 0.012265 2024-08-17T17:08:57.274 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:56 smithi142 ceph-mon[106729]: pgmap v202: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 7496/294861 objects degraded (2.542%); 6.4 MiB/s, 101 objects/s recovering 2024-08-17T17:08:57.275 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:57.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:56 smithi142 ceph-mon[103614]: pgmap v202: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 7496/294861 objects degraded (2.542%); 6.4 MiB/s, 101 objects/s recovering 2024-08-17T17:08:57.275 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:56 smithi152 ceph-mon[93456]: pgmap v202: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 7496/294861 objects degraded (2.542%); 6.4 MiB/s, 101 objects/s recovering 2024-08-17T17:08:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:58.228 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 102574 102558 81.1236 52.3125 0.0299981 0.012321 2024-08-17T17:08:58.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:57 smithi152 ceph-mon[93456]: from='client.54637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:57 smithi152 ceph-mon[93456]: from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:57 smithi142 ceph-mon[106729]: from='client.54637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:57 smithi142 ceph-mon[106729]: from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:57 smithi142 ceph-mon[103614]: from='client.54637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:58.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:57 smithi142 ceph-mon[103614]: from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:08:59.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:08:59.231230+0000 min lat: 0.00116924 max lat: 2.18324 avg lat: 0.0123745 2024-08-17T17:08:59.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:08:59.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 103418 103402 80.7658 52.75 0.0214764 0.0123745 2024-08-17T17:08:59.318 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:58 smithi152 ceph-mon[93456]: pgmap v203: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 7496/294861 objects degraded (2.542%); 3.1 MiB/s, 50 objects/s recovering 2024-08-17T17:08:59.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:58 smithi142 ceph-mon[103614]: pgmap v203: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 7496/294861 objects degraded (2.542%); 3.1 MiB/s, 50 objects/s recovering 2024-08-17T17:08:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:58 smithi142 ceph-mon[106729]: pgmap v203: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s; 7496/294861 objects degraded (2.542%); 3.1 MiB/s, 50 objects/s recovering 2024-08-17T17:08:59.963 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:00.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3516373906' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:00.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:00.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:08:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:00.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3516373906' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:00.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:00.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:08:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:00.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 104179 104163 80.3559 47.5625 0.0290071 0.0124388 2024-08-17T17:09:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3516373906' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:08:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:09:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:09:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:09:01.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 104990 104974 79.9941 50.6875 0.0149182 0.0124946 2024-08-17T17:09:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:00 smithi152 ceph-mon[93456]: pgmap v204: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 7185/304488 objects degraded (2.360%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:09:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:00 smithi152 ceph-mon[93456]: from='client.44717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7185/304488 objects degraded (2.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:00 smithi152 ceph-mon[93456]: from='client.54646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:01.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[103614]: pgmap v204: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 7185/304488 objects degraded (2.360%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[103614]: from='client.44717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7185/304488 objects degraded (2.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[103614]: from='client.54646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[106729]: pgmap v204: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 7185/304488 objects degraded (2.360%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[106729]: from='client.44717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7185/304488 objects degraded (2.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:00 smithi142 ceph-mon[106729]: from='client.54646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:02.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:09:01] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-08-17T17:09:02.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 105797 105781 79.6379 50.4375 0.0136771 0.0125509 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: pgmap v205: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 7185/307401 objects degraded (2.337%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: pgmap v205: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 7185/307401 objects degraded (2.337%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:02.625 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:02 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:09:02.355+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: pgmap v205: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 7185/307401 objects degraded (2.337%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:09:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:09:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:09:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:09:02.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:09:02.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:09:02.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:03.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 106631 106615 79.3104 52.125 0.0133977 0.0126027 2024-08-17T17:09:03.603 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:03 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:03 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3871063794' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:03 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:03.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:03 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:03.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3871063794' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:03 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:03 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3871063794' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:04.231 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 107428 107412 78.9633 49.8125 0.0256539 0.0126585 2024-08-17T17:09:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:04 smithi152 ceph-mon[93456]: pgmap v206: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 6875/316443 objects degraded (2.173%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:09:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:04 smithi152 ceph-mon[93456]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:04 smithi142 ceph-mon[106729]: pgmap v206: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 6875/316443 objects degraded (2.173%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:09:04.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:04 smithi142 ceph-mon[106729]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:04.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:04 smithi142 ceph-mon[103614]: pgmap v206: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 6875/316443 objects degraded (2.173%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:09:04.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:04 smithi142 ceph-mon[103614]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:05.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 108319 108303 78.6927 55.6875 0.0172831 0.0127012 2024-08-17T17:09:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:05 smithi152 ceph-mon[93456]: from='client.34932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6875/316443 objects degraded (2.173%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:05 smithi142 ceph-mon[103614]: from='client.34932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6875/316443 objects degraded (2.173%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:05 smithi142 ceph-mon[106729]: from='client.34932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6875/316443 objects degraded (2.173%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:06.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 109218 109202 78.434 56.1875 0.0191444 0.0127435 2024-08-17T17:09:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:06 smithi152 ceph-mon[93456]: pgmap v207: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 6875/319683 objects degraded (2.151%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:06 smithi142 ceph-mon[103614]: pgmap v207: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 6875/319683 objects degraded (2.151%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:06 smithi142 ceph-mon[106729]: pgmap v207: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 6875/319683 objects degraded (2.151%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:07.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 109998 109982 78.0966 48.75 0.0169434 0.0127989 2024-08-17T17:09:07.258 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2625287874' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2625287874' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:07.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2625287874' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:08.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 110825 110809 77.7999 51.6875 0.0223899 0.0128477 2024-08-17T17:09:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:08 smithi152 ceph-mon[93456]: pgmap v208: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 6875/319683 objects degraded (2.151%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:08 smithi152 ceph-mon[93456]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:08 smithi152 ceph-mon[93456]: from='client.44747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:08 smithi142 ceph-mon[103614]: pgmap v208: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 6875/319683 objects degraded (2.151%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:08 smithi142 ceph-mon[103614]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:08 smithi142 ceph-mon[103614]: from='client.44747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:08 smithi142 ceph-mon[106729]: pgmap v208: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s; 6875/319683 objects degraded (2.151%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:08 smithi142 ceph-mon[106729]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:08 smithi142 ceph-mon[106729]: from='client.44747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:09.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0109 2024-08-17T17:09:09.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 111691 2024-08-17T17:09:09.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:09:09.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:09:09.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 77.5538 2024-08-17T17:09:09.232 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 42.8198 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 281 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1240 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 685.117 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 4496 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 0 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0128891 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0311056 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 2.18324 2024-08-17T17:09:09.233 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00116924 2024-08-17T17:09:09.238 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:09:10.531 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:09:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:09:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:09:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:10 smithi152 ceph-mon[93456]: pgmap v209: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 6564/329049 objects degraded (1.995%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:09:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6564/329049 objects degraded (1.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/744506493' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:10 smithi142 ceph-mon[106729]: pgmap v209: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 6564/329049 objects degraded (1.995%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:09:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6564/329049 objects degraded (1.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/744506493' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:10 smithi142 ceph-mon[103614]: pgmap v209: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 6564/329049 objects degraded (1.995%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:09:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6564/329049 objects degraded (1.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/744506493' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:10.897 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:11.774 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:11 smithi142 ceph-mon[106729]: from='client.44753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:11.774 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:11.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:11 smithi142 ceph-mon[103614]: from='client.44753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:11.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:11 smithi152 ceph-mon[93456]: from='client.44753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:12.065 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:09:11] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-08-17T17:09:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:12 smithi152 ceph-mon[93456]: pgmap v210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 772 op/s; 6564/330864 objects degraded (1.984%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:12 smithi152 ceph-mon[93456]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:12 smithi142 ceph-mon[106729]: pgmap v210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 772 op/s; 6564/330864 objects degraded (1.984%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:12.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:12 smithi142 ceph-mon[106729]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:12 smithi142 ceph-mon[103614]: pgmap v210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 772 op/s; 6564/330864 objects degraded (1.984%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:09:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:12 smithi142 ceph-mon[103614]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:14.507 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:14 smithi152 ceph-mon[93456]: pgmap v211: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 951 op/s; 6223/328635 objects degraded (1.894%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:09:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3174366510' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:14 smithi142 ceph-mon[103614]: pgmap v211: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 951 op/s; 6223/328635 objects degraded (1.894%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:09:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3174366510' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:14.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:14 smithi142 ceph-mon[106729]: pgmap v211: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 951 op/s; 6223/328635 objects degraded (1.894%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:09:14.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3174366510' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:15 smithi152 ceph-mon[93456]: from='client.54679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6223/328635 objects degraded (1.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:15 smithi142 ceph-mon[103614]: from='client.54679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6223/328635 objects degraded (1.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:15.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:15 smithi142 ceph-mon[106729]: from='client.54679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:15.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6223/328635 objects degraded (1.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:16 smithi152 ceph-mon[93456]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:16 smithi152 ceph-mon[93456]: pgmap v212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 700 op/s; 6223/328635 objects degraded (1.894%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:09:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:16 smithi142 ceph-mon[103614]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:16 smithi142 ceph-mon[103614]: pgmap v212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 700 op/s; 6223/328635 objects degraded (1.894%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:09:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:16 smithi142 ceph-mon[106729]: from='client.34971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:16 smithi142 ceph-mon[106729]: pgmap v212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 700 op/s; 6223/328635 objects degraded (1.894%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:09:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3133329721' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:17.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:09:17.371+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:17.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:17 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:17.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3133329721' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:18.044 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:17 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3133329721' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[106729]: pgmap v213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 717 op/s; 6223/324792 objects degraded (1.916%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:09:18.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:18.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:18.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[106729]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:09:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[103614]: pgmap v213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 717 op/s; 6223/324792 objects degraded (1.916%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:09:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:18.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:18 smithi142 ceph-mon[103614]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:09:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:18 smithi152 ceph-mon[93456]: pgmap v213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 717 op/s; 6223/324792 objects degraded (1.916%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:09:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:18 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:18 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:18 smithi152 ceph-mon[93456]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:09:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:19 smithi142 ceph-mon[106729]: from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:19.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:19 smithi142 ceph-mon[106729]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:19.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:19 smithi142 ceph-mon[106729]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:09:19.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:19 smithi142 ceph-mon[103614]: from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:19.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:19 smithi142 ceph-mon[103614]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:19.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:19 smithi142 ceph-mon[103614]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:09:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:19 smithi152 ceph-mon[93456]: from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:19 smithi152 ceph-mon[93456]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:19 smithi152 ceph-mon[93456]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:09:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:09:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:09:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:09:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:20 smithi142 ceph-mon[106729]: pgmap v216: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 9.3 MiB/s wr, 909 op/s; 5887/316158 objects degraded (1.862%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:09:20.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5887/316158 objects degraded (1.862%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:20 smithi142 ceph-mon[103614]: pgmap v216: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 9.3 MiB/s wr, 909 op/s; 5887/316158 objects degraded (1.862%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:09:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5887/316158 objects degraded (1.862%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:20 smithi152 ceph-mon[93456]: pgmap v216: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 9.3 MiB/s wr, 909 op/s; 5887/316158 objects degraded (1.862%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:09:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5887/316158 objects degraded (1.862%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:21.436 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:21.773 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3289335443' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:21.773 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3289335443' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3289335443' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:09:21] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-08-17T17:09:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:22 smithi142 ceph-mon[106729]: pgmap v217: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 5887/313323 objects degraded (1.879%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:09:22.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:22 smithi142 ceph-mon[106729]: from='client.44792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:22.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:22 smithi142 ceph-mon[106729]: from='client.54715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:22 smithi142 ceph-mon[103614]: pgmap v217: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 5887/313323 objects degraded (1.879%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:09:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:22 smithi142 ceph-mon[103614]: from='client.44792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:22 smithi142 ceph-mon[103614]: from='client.54715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:22 smithi152 ceph-mon[93456]: pgmap v217: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 5887/313323 objects degraded (1.879%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:09:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:22 smithi152 ceph-mon[93456]: from='client.44792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:22 smithi152 ceph-mon[93456]: from='client.54715 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:24.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:24 smithi142 ceph-mon[106729]: pgmap v218: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 5676/305691 objects degraded (1.857%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:09:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/421105559' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:24.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:24 smithi142 ceph-mon[103614]: pgmap v218: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 5676/305691 objects degraded (1.857%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:09:24.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/421105559' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:24.875 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:24 smithi152 ceph-mon[93456]: pgmap v218: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 5676/305691 objects degraded (1.857%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:09:24.875 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/421105559' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:25.029 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5676/305691 objects degraded (1.857%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5676/305691 objects degraded (1.857%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5676/305691 objects degraded (1.857%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[106729]: from='client.44801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[106729]: pgmap v219: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 5676/304977 objects degraded (1.861%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[106729]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[103614]: from='client.44801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[103614]: pgmap v219: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 5676/304977 objects degraded (1.861%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[103614]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:26 smithi152 ceph-mon[93456]: from='client.44801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:26 smithi152 ceph-mon[93456]: pgmap v219: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 5676/304977 objects degraded (1.861%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:09:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:26 smithi152 ceph-mon[93456]: from='client.54730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:28.625 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:28.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:28 smithi142 ceph-mon[106729]: pgmap v220: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5676/301335 objects degraded (1.884%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:09:28.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/528233920' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:28.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:28 smithi142 ceph-mon[103614]: pgmap v220: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5676/301335 objects degraded (1.884%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:09:28.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/528233920' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:28 smithi152 ceph-mon[93456]: pgmap v220: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5676/301335 objects degraded (1.884%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:09:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/528233920' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:29 smithi152 ceph-mon[93456]: from='client.35016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:29 smithi142 ceph-mon[106729]: from='client.35016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:29 smithi142 ceph-mon[103614]: from='client.35016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:09:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:09:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:09:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:30 smithi152 ceph-mon[93456]: from='client.35019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:30 smithi152 ceph-mon[93456]: pgmap v221: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 5541/292713 objects degraded (1.893%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:09:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5541/292713 objects degraded (1.893%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:30 smithi142 ceph-mon[106729]: from='client.35019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:30 smithi142 ceph-mon[106729]: pgmap v221: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 5541/292713 objects degraded (1.893%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:09:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5541/292713 objects degraded (1.893%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:30 smithi142 ceph-mon[103614]: from='client.35019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:30 smithi142 ceph-mon[103614]: pgmap v221: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 5541/292713 objects degraded (1.893%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:09:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5541/292713 objects degraded (1.893%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:32.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:09:31] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-08-17T17:09:32.231 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:32.622 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:09:32.394+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:32 smithi152 ceph-mon[93456]: pgmap v222: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5541/289686 objects degraded (1.913%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:09:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/411846109' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-mon[106729]: pgmap v222: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5541/289686 objects degraded (1.913%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:09:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/411846109' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-mon[103614]: pgmap v222: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5541/289686 objects degraded (1.913%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:09:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/411846109' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:33 smithi152 ceph-mon[93456]: from='client.35028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:33 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:33 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:33 smithi152 ceph-mon[93456]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:34.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[106729]: from='client.35028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[106729]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[103614]: from='client.35028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:34.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:33 smithi142 ceph-mon[103614]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:34 smithi152 ceph-mon[93456]: pgmap v223: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 5378/281757 objects degraded (1.909%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:09:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:34 smithi142 ceph-mon[106729]: pgmap v223: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 5378/281757 objects degraded (1.909%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:09:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:34 smithi142 ceph-mon[103614]: pgmap v223: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 5378/281757 objects degraded (1.909%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:09:35.873 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5378/281757 objects degraded (1.909%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3473424803' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5378/281757 objects degraded (1.909%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3473424803' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5378/281757 objects degraded (1.909%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3473424803' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:36 smithi152 ceph-mon[93456]: pgmap v224: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 5378/281010 objects degraded (1.914%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:09:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:36 smithi152 ceph-mon[93456]: from='client.35034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:36 smithi142 ceph-mon[103614]: pgmap v224: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 5378/281010 objects degraded (1.914%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:09:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:36 smithi142 ceph-mon[103614]: from='client.35034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:36 smithi142 ceph-mon[106729]: pgmap v224: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 5378/281010 objects degraded (1.914%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:09:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:36 smithi142 ceph-mon[106729]: from='client.35034 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:38.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:37 smithi152 ceph-mon[93456]: from='client.35040 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:38.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:37 smithi142 ceph-mon[103614]: from='client.35040 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:38.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:37 smithi142 ceph-mon[106729]: from='client.35040 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:38 smithi152 ceph-mon[93456]: pgmap v225: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 5378/277284 objects degraded (1.940%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:09:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:38 smithi142 ceph-mon[106729]: pgmap v225: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 5378/277284 objects degraded (1.940%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:09:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:38 smithi142 ceph-mon[103614]: pgmap v225: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 5378/277284 objects degraded (1.940%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:09:39.493 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3760211396' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3760211396' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3760211396' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:09:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:09:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:09:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:40 smithi152 ceph-mon[93456]: pgmap v226: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 5211/269262 objects degraded (1.935%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:09:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:40 smithi152 ceph-mon[93456]: from='client.54775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5211/269262 objects degraded (1.935%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:40 smithi142 ceph-mon[106729]: pgmap v226: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 5211/269262 objects degraded (1.935%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:09:41.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:40 smithi142 ceph-mon[106729]: from='client.54775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:41.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5211/269262 objects degraded (1.935%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:40 smithi142 ceph-mon[103614]: pgmap v226: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 5211/269262 objects degraded (1.935%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:09:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:40 smithi142 ceph-mon[103614]: from='client.54775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5211/269262 objects degraded (1.935%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:42.023 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:41 smithi142 ceph-mon[103614]: from='client.54781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:42.023 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:42.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:41 smithi142 ceph-mon[106729]: from='client.54781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:42.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:42.024 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:09:41] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.51.0" 2024-08-17T17:09:42.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:41 smithi152 ceph-mon[93456]: from='client.54781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:42 smithi152 ceph-mon[93456]: pgmap v227: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 5211/266379 objects degraded (1.956%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:43.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1990655765' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:43.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:42 smithi142 ceph-mon[103614]: pgmap v227: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 5211/266379 objects degraded (1.956%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:43.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1990655765' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:43.120 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:42 smithi142 ceph-mon[106729]: pgmap v227: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 5211/266379 objects degraded (1.956%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:43.120 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1990655765' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:43.121 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:44 smithi152 ceph-mon[93456]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:44 smithi152 ceph-mon[93456]: pgmap v228: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 5042/258357 objects degraded (1.952%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:09:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:44 smithi152 ceph-mon[93456]: from='client.44855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:44 smithi142 ceph-mon[103614]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:44 smithi142 ceph-mon[103614]: pgmap v228: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 5042/258357 objects degraded (1.952%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:09:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:44 smithi142 ceph-mon[103614]: from='client.44855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:44 smithi142 ceph-mon[106729]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:44 smithi142 ceph-mon[106729]: pgmap v228: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 5042/258357 objects degraded (1.952%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:09:45.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:44 smithi142 ceph-mon[106729]: from='client.44855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:46.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5042/258357 objects degraded (1.952%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5042/258357 objects degraded (1.952%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:46.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5042/258357 objects degraded (1.952%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:46.737 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:46 smithi152 ceph-mon[93456]: pgmap v229: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5042/257616 objects degraded (1.957%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1785395787' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:46 smithi142 ceph-mon[106729]: pgmap v229: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5042/257616 objects degraded (1.957%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1785395787' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:46 smithi142 ceph-mon[103614]: pgmap v229: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5042/257616 objects degraded (1.957%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1785395787' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:47.748 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:47 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:09:47.412+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:47 smithi152 ceph-mon[93456]: from='client.54796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:48.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:47 smithi142 ceph-mon[103614]: from='client.54796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:48.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:48.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:47 smithi142 ceph-mon[106729]: from='client.54796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:48.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:09:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:48 smithi152 ceph-mon[93456]: pgmap v230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 5042/254001 objects degraded (1.985%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:48 smithi152 ceph-mon[93456]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:48 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:48 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:49.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[103614]: pgmap v230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 5042/254001 objects degraded (1.985%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:49.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[103614]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:49.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[106729]: pgmap v230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 5042/254001 objects degraded (1.985%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:09:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[106729]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:09:49.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:48 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:09:50.361 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:09:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:09:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:09:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:50 smithi152 ceph-mon[93456]: pgmap v231: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4862/246846 objects degraded (1.970%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:09:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/861295947' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4862/246846 objects degraded (1.970%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:51.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:50 smithi142 ceph-mon[106729]: pgmap v231: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4862/246846 objects degraded (1.970%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:09:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/861295947' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4862/246846 objects degraded (1.970%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:50 smithi142 ceph-mon[103614]: pgmap v231: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 4862/246846 objects degraded (1.970%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:09:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/861295947' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4862/246846 objects degraded (1.970%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:51 smithi152 ceph-mon[93456]: from='client.35085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:51 smithi152 ceph-mon[93456]: from='client.44882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:52.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:51 smithi142 ceph-mon[106729]: from='client.35085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:51 smithi142 ceph-mon[106729]: from='client.44882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:52.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:09:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:09:51] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.51.0" 2024-08-17T17:09:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:51 smithi142 ceph-mon[103614]: from='client.35085 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:51 smithi142 ceph-mon[103614]: from='client.44882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:52 smithi152 ceph-mon[93456]: pgmap v232: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 4862/243279 objects degraded (1.999%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:09:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:52 smithi142 ceph-mon[103614]: pgmap v232: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 4862/243279 objects degraded (1.999%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:09:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:52 smithi142 ceph-mon[106729]: pgmap v232: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 4862/243279 objects degraded (1.999%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:09:53.974 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/210780312' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/210780312' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/210780312' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:54 smithi152 ceph-mon[93456]: pgmap v233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4666/235203 objects degraded (1.984%); 2.0 MiB/s, 45 objects/s recovering 2024-08-17T17:09:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:54 smithi152 ceph-mon[93456]: from='client.44894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:54 smithi142 ceph-mon[103614]: pgmap v233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4666/235203 objects degraded (1.984%); 2.0 MiB/s, 45 objects/s recovering 2024-08-17T17:09:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:54 smithi142 ceph-mon[103614]: from='client.44894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:54 smithi142 ceph-mon[106729]: pgmap v233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4666/235203 objects degraded (1.984%); 2.0 MiB/s, 45 objects/s recovering 2024-08-17T17:09:55.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:54 smithi142 ceph-mon[106729]: from='client.44894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:55 smithi152 ceph-mon[93456]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4666/235203 objects degraded (1.984%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:55 smithi142 ceph-mon[103614]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4666/235203 objects degraded (1.984%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:56.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:55 smithi142 ceph-mon[106729]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:56.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4666/235203 objects degraded (1.984%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:09:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:56 smithi152 ceph-mon[93456]: pgmap v234: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 4666/234525 objects degraded (1.990%); 1.2 MiB/s, 31 objects/s recovering 2024-08-17T17:09:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:56 smithi142 ceph-mon[103614]: pgmap v234: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 4666/234525 objects degraded (1.990%); 1.2 MiB/s, 31 objects/s recovering 2024-08-17T17:09:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:56 smithi142 ceph-mon[106729]: pgmap v234: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 4666/234525 objects degraded (1.990%); 1.2 MiB/s, 31 objects/s recovering 2024-08-17T17:09:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:57.623 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:09:58.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1807464237' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:58.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1807464237' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:58.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1807464237' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:09:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:58 smithi152 ceph-mon[93456]: pgmap v235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 4666/230985 objects degraded (2.020%); 1.2 MiB/s, 31 objects/s recovering 2024-08-17T17:09:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:58 smithi152 ceph-mon[93456]: from='client.54829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:58 smithi142 ceph-mon[103614]: pgmap v235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 4666/230985 objects degraded (2.020%); 1.2 MiB/s, 31 objects/s recovering 2024-08-17T17:09:59.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:58 smithi142 ceph-mon[103614]: from='client.54829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:09:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:58 smithi142 ceph-mon[106729]: pgmap v235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 4666/230985 objects degraded (2.020%); 1.2 MiB/s, 31 objects/s recovering 2024-08-17T17:09:59.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:58 smithi142 ceph-mon[106729]: from='client.54829 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:09:59 smithi152 ceph-mon[93456]: from='client.35112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:09:59 smithi142 ceph-mon[103614]: from='client.35112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:09:59 smithi142 ceph-mon[106729]: from='client.35112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:10:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: pgmap v236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 4440/223953 objects degraded (1.983%); 1.2 MiB/s, 50 objects/s recovering 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: Health detail: HEALTH_WARN Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: [WRN] PG_DEGRADED: Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: pg 7.2 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: pg 7.9 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2024-08-17T17:10:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:01.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1546438269' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: pgmap v236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 4440/223953 objects degraded (1.983%); 1.2 MiB/s, 50 objects/s recovering 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: Health detail: HEALTH_WARN Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: [WRN] PG_DEGRADED: Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: pg 7.2 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: pg 7.9 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2024-08-17T17:10:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1546438269' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: pgmap v236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 4440/223953 objects degraded (1.983%); 1.2 MiB/s, 50 objects/s recovering 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: Health detail: HEALTH_WARN Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: [WRN] PG_DEGRADED: Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: pg 7.2 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: pg 7.9 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4440/223953 objects degraded (1.983%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:01.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1546438269' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:01.236 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:10:01] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[103614]: from='client.54844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[103614]: pgmap v237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 4440/220404 objects degraded (2.014%); 235 KiB/s, 35 objects/s recovering 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[103614]: from='client.54847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[106729]: from='client.54844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[106729]: pgmap v237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 4440/220404 objects degraded (2.014%); 235 KiB/s, 35 objects/s recovering 2024-08-17T17:10:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[106729]: from='client.54847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:03.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:02 smithi152 ceph-mon[93456]: from='client.54844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:02 smithi152 ceph-mon[93456]: pgmap v237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 4440/220404 objects degraded (2.014%); 235 KiB/s, 35 objects/s recovering 2024-08-17T17:10:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:02 smithi152 ceph-mon[93456]: from='client.54847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:04.867 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:04 smithi142 ceph-mon[103614]: pgmap v238: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4202/212769 objects degraded (1.975%); 235 KiB/s, 54 objects/s recovering 2024-08-17T17:10:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1754676941' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:04 smithi142 ceph-mon[106729]: pgmap v238: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4202/212769 objects degraded (1.975%); 235 KiB/s, 54 objects/s recovering 2024-08-17T17:10:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1754676941' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:04 smithi152 ceph-mon[93456]: pgmap v238: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4202/212769 objects degraded (1.975%); 235 KiB/s, 54 objects/s recovering 2024-08-17T17:10:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1754676941' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:05.990 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:05 smithi152 ceph-mon[93456]: from='client.54853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:05.990 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4202/212769 objects degraded (1.975%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:06.093 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:05 smithi142 ceph-mon[103614]: from='client.54853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:06.094 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4202/212769 objects degraded (1.975%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:06.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:05 smithi142 ceph-mon[106729]: from='client.54853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:06.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4202/212769 objects degraded (1.975%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:06 smithi152 ceph-mon[93456]: pgmap v239: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 4202/212145 objects degraded (1.981%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:06 smithi152 ceph-mon[93456]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:07.112 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:06 smithi142 ceph-mon[103614]: pgmap v239: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 4202/212145 objects degraded (1.981%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:07.113 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:06 smithi142 ceph-mon[103614]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:07.113 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:06 smithi142 ceph-mon[106729]: pgmap v239: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 4202/212145 objects degraded (1.981%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:07.113 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:06 smithi142 ceph-mon[106729]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:08.540 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: pgmap v240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 4202/209421 objects degraded (2.006%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/921809939' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:10:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: pgmap v240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 4202/209421 objects degraded (2.006%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/921809939' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.125 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:10:09.126 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:08 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:10:08.862+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: pgmap v240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 4202/209421 objects degraded (2.006%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/921809939' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:09.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:10:10.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:09 smithi152 ceph-mon[93456]: from='client.35136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:10.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:09 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:10:10.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:09 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:10:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:09 smithi142 ceph-mon[106729]: from='client.35136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:09 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:10:10.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:09 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:10:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:09 smithi142 ceph-mon[103614]: from='client.35136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:09 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:10:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:09 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-08-17T17:10:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:10:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:10:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:10:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:10 smithi152 ceph-mon[93456]: from='client.44936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:10 smithi152 ceph-mon[93456]: pgmap v241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 3971/202191 objects degraded (1.964%); 0 B/s, 57 objects/s recovering 2024-08-17T17:10:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:10 smithi152 ceph-mon[93456]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:10:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3971/202191 objects degraded (1.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[106729]: from='client.44936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[106729]: pgmap v241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 3971/202191 objects degraded (1.964%); 0 B/s, 57 objects/s recovering 2024-08-17T17:10:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[106729]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:10:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3971/202191 objects degraded (1.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[103614]: from='client.44936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[103614]: pgmap v241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 3971/202191 objects degraded (1.964%); 0 B/s, 57 objects/s recovering 2024-08-17T17:10:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[103614]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:10:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3971/202191 objects degraded (1.964%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-mon[106729]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:10:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2701605002' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-mon[103614]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:10:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2701605002' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:10:11] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.51.0" 2024-08-17T17:10:12.155 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:11 smithi152 ceph-mon[93456]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:10:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2701605002' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:12 smithi152 ceph-mon[93456]: pgmap v244: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 3895/199014 objects degraded (1.957%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:12 smithi142 ceph-mon[103614]: pgmap v244: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 3895/199014 objects degraded (1.957%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:12 smithi142 ceph-mon[106729]: pgmap v244: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 3895/199014 objects degraded (1.957%); 0 B/s, 38 objects/s recovering 2024-08-17T17:10:14.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:13 smithi142 ceph-mon[106729]: from='client.54877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:14.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:13 smithi142 ceph-mon[106729]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:14.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:13 smithi142 ceph-mon[103614]: from='client.54877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:14.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:13 smithi142 ceph-mon[103614]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:13 smithi152 ceph-mon[93456]: from='client.54877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:13 smithi152 ceph-mon[93456]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:14 smithi152 ceph-mon[93456]: pgmap v245: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 3818/192846 objects degraded (1.980%); 0 B/s, 47 objects/s recovering 2024-08-17T17:10:15.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:14 smithi142 ceph-mon[103614]: pgmap v245: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 3818/192846 objects degraded (1.980%); 0 B/s, 47 objects/s recovering 2024-08-17T17:10:15.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:14 smithi142 ceph-mon[106729]: pgmap v245: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 3818/192846 objects degraded (1.980%); 0 B/s, 47 objects/s recovering 2024-08-17T17:10:15.818 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4010839394' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3818/192846 objects degraded (1.980%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4010839394' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3818/192846 objects degraded (1.980%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4010839394' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3818/192846 objects degraded (1.980%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:16 smithi152 ceph-mon[93456]: pgmap v246: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 3818/192255 objects degraded (1.986%); 0 B/s, 47 objects/s recovering 2024-08-17T17:10:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:16 smithi152 ceph-mon[93456]: from='client.44957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:17.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:16 smithi142 ceph-mon[106729]: pgmap v246: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 3818/192255 objects degraded (1.986%); 0 B/s, 47 objects/s recovering 2024-08-17T17:10:17.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:16 smithi142 ceph-mon[106729]: from='client.44957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:16 smithi142 ceph-mon[103614]: pgmap v246: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 3818/192255 objects degraded (1.986%); 0 B/s, 47 objects/s recovering 2024-08-17T17:10:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:16 smithi142 ceph-mon[103614]: from='client.44957 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:18.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:17 smithi152 ceph-mon[93456]: from='client.44963 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:18.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:17 smithi142 ceph-mon[106729]: from='client.44963 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:18.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:17 smithi142 ceph-mon[103614]: from='client.44963 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:19.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:18 smithi152 ceph-mon[93456]: pgmap v247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 3538/190017 objects degraded (1.862%); 0 B/s, 54 objects/s recovering 2024-08-17T17:10:19.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/968291700' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:19.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:18 smithi142 ceph-mon[103614]: pgmap v247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 3538/190017 objects degraded (1.862%); 0 B/s, 54 objects/s recovering 2024-08-17T17:10:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/968291700' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:18 smithi142 ceph-mon[106729]: pgmap v247: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 3538/190017 objects degraded (1.862%); 0 B/s, 54 objects/s recovering 2024-08-17T17:10:19.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/968291700' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:19.436 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:10:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:10:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:10:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:20 smithi152 ceph-mon[93456]: pgmap v248: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 3538/182709 objects degraded (1.936%); 0 B/s, 46 objects/s recovering 2024-08-17T17:10:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:20 smithi152 ceph-mon[93456]: from='client.54883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:20 smithi152 ceph-mon[93456]: from='client.44978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3538/182709 objects degraded (1.936%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[106729]: pgmap v248: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 3538/182709 objects degraded (1.936%); 0 B/s, 46 objects/s recovering 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[106729]: from='client.54883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[106729]: from='client.44978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3538/182709 objects degraded (1.936%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[103614]: pgmap v248: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 3538/182709 objects degraded (1.936%); 0 B/s, 46 objects/s recovering 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[103614]: from='client.54883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[103614]: from='client.44978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3538/182709 objects degraded (1.936%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:10:21] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.51.0" 2024-08-17T17:10:23.079 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:22 smithi152 ceph-mon[93456]: pgmap v249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3344/179322 objects degraded (1.865%); 0 B/s, 53 objects/s recovering 2024-08-17T17:10:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2324470897' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:22 smithi142 ceph-mon[106729]: pgmap v249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3344/179322 objects degraded (1.865%); 0 B/s, 53 objects/s recovering 2024-08-17T17:10:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2324470897' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:22 smithi142 ceph-mon[103614]: pgmap v249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 3344/179322 objects degraded (1.865%); 0 B/s, 53 objects/s recovering 2024-08-17T17:10:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2324470897' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:24.266 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:24 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:10:24.267 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:24 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:10:24.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:24 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[106729]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[106729]: pgmap v250: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 3344/171576 objects degraded (1.949%); 0 B/s, 45 objects/s recovering 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[106729]: from='client.35181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[106729]: Upgrade: osd.2 is safe to restart 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[103614]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[103614]: pgmap v250: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 3344/171576 objects degraded (1.949%); 0 B/s, 45 objects/s recovering 2024-08-17T17:10:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[103614]: from='client.35181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:25.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:10:25.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:25 smithi142 ceph-mon[103614]: Upgrade: osd.2 is safe to restart 2024-08-17T17:10:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:25 smithi152 ceph-mon[93456]: from='client.54892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:25 smithi152 ceph-mon[93456]: pgmap v250: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 3344/171576 objects degraded (1.949%); 0 B/s, 45 objects/s recovering 2024-08-17T17:10:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:25 smithi152 ceph-mon[93456]: from='client.35181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:25 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:10:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:25 smithi152 ceph-mon[93456]: Upgrade: osd.2 is safe to restart 2024-08-17T17:10:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3344/171576 objects degraded (1.949%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:26.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3344/171576 objects degraded (1.949%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:26.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3344/171576 objects degraded (1.949%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:26.654 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: pgmap v251: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 3344/170850 objects degraded (1.957%); 0 B/s, 39 objects/s recovering 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1527002110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: from='client.54904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: Upgrade: Updating osd.2 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:27 smithi152 ceph-mon[93456]: Deploying daemon osd.2 on smithi142 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: pgmap v251: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 3344/170850 objects degraded (1.957%); 0 B/s, 39 objects/s recovering 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1527002110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: from='client.54904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: Upgrade: Updating osd.2 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[106729]: Deploying daemon osd.2 on smithi142 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: pgmap v251: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 3344/170850 objects degraded (1.957%); 0 B/s, 39 objects/s recovering 2024-08-17T17:10:27.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1527002110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: from='client.54904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: Upgrade: Updating osd.2 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:27.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:27 smithi142 ceph-mon[103614]: Deploying daemon osd.2 on smithi142 2024-08-17T17:10:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:28 smithi152 ceph-mon[93456]: from='client.35196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:28.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:28 smithi152 ceph-mon[93456]: pgmap v252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3145/167532 objects degraded (1.877%); 0 B/s, 56 objects/s recovering 2024-08-17T17:10:28.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:28 smithi142 ceph-mon[106729]: from='client.35196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:28.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:28 smithi142 ceph-mon[106729]: pgmap v252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3145/167532 objects degraded (1.877%); 0 B/s, 56 objects/s recovering 2024-08-17T17:10:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:28 smithi142 ceph-mon[103614]: from='client.35196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:28 smithi142 ceph-mon[103614]: pgmap v252: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3145/167532 objects degraded (1.877%); 0 B/s, 56 objects/s recovering 2024-08-17T17:10:30.277 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:30.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:30 smithi142 ceph-mon[106729]: pgmap v253: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 3145/160992 objects degraded (1.954%); 0 B/s, 32 objects/s recovering 2024-08-17T17:10:30.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2455630837' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:30.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:30 smithi142 ceph-mon[103614]: pgmap v253: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 3145/160992 objects degraded (1.954%); 0 B/s, 32 objects/s recovering 2024-08-17T17:10:30.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2455630837' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:30 smithi152 ceph-mon[93456]: pgmap v253: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 3145/160992 objects degraded (1.954%); 0 B/s, 32 objects/s recovering 2024-08-17T17:10:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2455630837' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:10:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:10:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:10:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-mon[106729]: from='client.45014 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3145/160992 objects degraded (1.954%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-mon[106729]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.373 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:31 smithi142 systemd[1]: Stopping Ceph osd.2 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:10:31.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-mon[103614]: from='client.45014 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3145/160992 objects degraded (1.954%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:31.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-mon[103614]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:31 smithi152 ceph-mon[93456]: from='client.45014 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3145/160992 objects degraded (1.954%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:31 smithi152 ceph-mon[93456]: from='client.54910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:10:31] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:10:31.873 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[56556]: 2024-08-17T17:10:31.585+0000 7f5ee4d39640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:10:31.873 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[56556]: 2024-08-17T17:10:31.585+0000 7f5ee4d39640 -1 osd.2 116 *** Got signal Terminated *** 2024-08-17T17:10:31.873 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[56556]: 2024-08-17T17:10:31.585+0000 7f5ee4d39640 -1 osd.2 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:10:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:32 smithi152 ceph-mon[93456]: pgmap v254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 2945/157827 objects degraded (1.866%); 0 B/s, 49 objects/s recovering 2024-08-17T17:10:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:32 smithi152 ceph-mon[93456]: osd.2 marked itself down and dead 2024-08-17T17:10:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:32 smithi142 ceph-mon[103614]: pgmap v254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 2945/157827 objects degraded (1.866%); 0 B/s, 49 objects/s recovering 2024-08-17T17:10:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:32 smithi142 ceph-mon[103614]: osd.2 marked itself down and dead 2024-08-17T17:10:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:32 smithi142 ceph-mon[106729]: pgmap v254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 2945/157827 objects degraded (1.866%); 0 B/s, 49 objects/s recovering 2024-08-17T17:10:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:32 smithi142 ceph-mon[106729]: osd.2 marked itself down and dead 2024-08-17T17:10:33.247 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:32 smithi142 podman[132018]: 2024-08-17 17:10:32.943329647 +0000 UTC m=+1.510093328 container died b599f0d555187f55490a73dcab1c310fc0de7502414f98997a99ff912c9ca06d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:10:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:33 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:10:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:33 smithi152 ceph-mon[93456]: osdmap e117: 8 total, 7 up, 8 in 2024-08-17T17:10:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:33 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4052256777' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:33.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:33 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:10:33.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:33 smithi142 ceph-mon[106729]: osdmap e117: 8 total, 7 up, 8 in 2024-08-17T17:10:33.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:33 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4052256777' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:33.614 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:33 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:10:33.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:33 smithi142 ceph-mon[103614]: osdmap e117: 8 total, 7 up, 8 in 2024-08-17T17:10:33.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:33 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4052256777' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:33.615 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:33.872 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:33 smithi142 podman[132018]: 2024-08-17 17:10:33.813789235 +0000 UTC m=+2.380552923 container cleanup b599f0d555187f55490a73dcab1c310fc0de7502414f98997a99ff912c9ca06d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T17:10:33.873 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:33 smithi142 bash[132018]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:10:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:34 smithi152 ceph-mon[93456]: pgmap v256: 89 pgs: 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 2945/152316 objects degraded (1.933%); 0 B/s, 39 objects/s recovering 2024-08-17T17:10:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:34 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:10:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:34 smithi152 ceph-mon[93456]: osdmap e118: 8 total, 7 up, 8 in 2024-08-17T17:10:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:34 smithi152 ceph-mon[93456]: from='client.45029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[106729]: pgmap v256: 89 pgs: 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 2945/152316 objects degraded (1.933%); 0 B/s, 39 objects/s recovering 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[106729]: osdmap e118: 8 total, 7 up, 8 in 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[106729]: from='client.45029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[103614]: pgmap v256: 89 pgs: 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 2945/152316 objects degraded (1.933%); 0 B/s, 39 objects/s recovering 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[103614]: osdmap e118: 8 total, 7 up, 8 in 2024-08-17T17:10:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:34 smithi142 ceph-mon[103614]: from='client.45029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:35 smithi152 ceph-mon[93456]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2945/152316 objects degraded (1.933%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:35 smithi142 ceph-mon[106729]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2945/152316 objects degraded (1.933%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:35 smithi142 ceph-mon[103614]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2945/152316 objects degraded (1.933%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:36.122 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:35 smithi142 podman[132058]: 2024-08-17 17:10:35.775217814 +0000 UTC m=+2.826594479 container remove b599f0d555187f55490a73dcab1c310fc0de7502414f98997a99ff912c9ca06d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-08-17T17:10:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:36 smithi152 ceph-mon[93456]: pgmap v258: 89 pgs: 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 stale+active+clean, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 73 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 2945/151770 objects degraded (1.940%); 0 B/s, 24 objects/s recovering 2024-08-17T17:10:36.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:36 smithi142 ceph-mon[106729]: pgmap v258: 89 pgs: 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 stale+active+clean, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 73 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 2945/151770 objects degraded (1.940%); 0 B/s, 24 objects/s recovering 2024-08-17T17:10:36.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:36 smithi142 ceph-mon[103614]: pgmap v258: 89 pgs: 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 stale+active+clean, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 73 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 2945/151770 objects degraded (1.940%); 0 B/s, 24 objects/s recovering 2024-08-17T17:10:36.910 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:36 smithi142 podman[132204]: 2024-08-17 17:10:36.56391242 +0000 UTC m=+0.033179302 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:10:36.911 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:36 smithi142 podman[132204]: 2024-08-17 17:10:36.743103071 +0000 UTC m=+0.212369938 container create 8440f6fb67b0dc25649a6c1dab96682646eea9a251262ca01bcf18499092edb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:10:37.184 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:37.321 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:37 smithi142 podman[132204]: 2024-08-17 17:10:37.030347317 +0000 UTC m=+0.499614186 container init 8440f6fb67b0dc25649a6c1dab96682646eea9a251262ca01bcf18499092edb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0) 2024-08-17T17:10:37.321 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:37 smithi142 podman[132204]: 2024-08-17 17:10:37.039155653 +0000 UTC m=+0.508422524 container start 8440f6fb67b0dc25649a6c1dab96682646eea9a251262ca01bcf18499092edb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:10:37.321 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:37 smithi142 podman[132204]: 2024-08-17 17:10:37.121146847 +0000 UTC m=+0.590413721 container attach 8440f6fb67b0dc25649a6c1dab96682646eea9a251262ca01bcf18499092edb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:10:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/320957877' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/320957877' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:37.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:37 smithi142 podman[132285]: 2024-08-17 17:10:37.385720292 +0000 UTC m=+0.035753801 container died 8440f6fb67b0dc25649a6c1dab96682646eea9a251262ca01bcf18499092edb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD) 2024-08-17T17:10:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/320957877' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:38.325 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 podman[132285]: 2024-08-17 17:10:38.053126898 +0000 UTC m=+0.703160412 container remove 8440f6fb67b0dc25649a6c1dab96682646eea9a251262ca01bcf18499092edb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:10:38.325 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.2.service: Deactivated successfully. 2024-08-17T17:10:38.325 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 systemd[1]: Stopped Ceph osd.2 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:10:38.325 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.2.service: Consumed 18min 50.934s CPU time. 2024-08-17T17:10:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:38 smithi142 ceph-mon[106729]: from='client.54940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:38 smithi142 ceph-mon[106729]: pgmap v259: 89 pgs: 7 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 stale+active+clean, 6 active+undersized, 1 active+undersized+degraded, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 2472/149226 objects degraded (1.657%); 0 B/s, 84 objects/s recovering 2024-08-17T17:10:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:38 smithi142 ceph-mon[106729]: from='client.45044 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:38.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 systemd[1]: Starting Ceph osd.2 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:10:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:38 smithi142 ceph-mon[103614]: from='client.54940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:38 smithi142 ceph-mon[103614]: pgmap v259: 89 pgs: 7 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 stale+active+clean, 6 active+undersized, 1 active+undersized+degraded, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 2472/149226 objects degraded (1.657%); 0 B/s, 84 objects/s recovering 2024-08-17T17:10:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:38 smithi142 ceph-mon[103614]: from='client.45044 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:38 smithi152 ceph-mon[93456]: from='client.54940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:38 smithi152 ceph-mon[93456]: pgmap v259: 89 pgs: 7 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 stale+active+clean, 6 active+undersized, 1 active+undersized+degraded, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 2472/149226 objects degraded (1.657%); 0 B/s, 84 objects/s recovering 2024-08-17T17:10:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:38 smithi152 ceph-mon[93456]: from='client.45044 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:39.122 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 podman[132389]: 2024-08-17 17:10:38.628317165 +0000 UTC m=+0.039669124 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:10:39.123 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:38 smithi142 podman[132389]: 2024-08-17 17:10:38.753911973 +0000 UTC m=+0.165263916 container create c95b78eb89ad044c8414c01a8f26d2eb05627558f1d56b25422b0a033883f73f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T17:10:39.427 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:10:39.458 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 podman[132389]: 2024-08-17 17:10:39.165815004 +0000 UTC m=+0.577166964 container init c95b78eb89ad044c8414c01a8f26d2eb05627558f1d56b25422b0a033883f73f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-17T17:10:39.458 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 podman[132389]: 2024-08-17 17:10:39.175485315 +0000 UTC m=+0.586837260 container start c95b78eb89ad044c8414c01a8f26d2eb05627558f1d56b25422b0a033883f73f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-17T17:10:39.458 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 podman[132389]: 2024-08-17 17:10:39.257263983 +0000 UTC m=+0.668615929 container attach c95b78eb89ad044c8414c01a8f26d2eb05627558f1d56b25422b0a033883f73f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:10:39.458 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.458 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.458 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.459 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.459 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:10:39.738 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:10:39.739 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate[132401]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 bash[132389]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:10:39.740 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:39 smithi142 podman[132537]: 2024-08-17 17:10:39.737070082 +0000 UTC m=+0.044370703 container died c95b78eb89ad044c8414c01a8f26d2eb05627558f1d56b25422b0a033883f73f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:10:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:39 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:10:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:40 smithi152 ceph-mon[93456]: pgmap v260: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 19429/140529 objects degraded (13.826%); 0 B/s, 81 objects/s recovering 2024-08-17T17:10:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2610211549' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 19429/140529 objects degraded (13.826%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:10:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:10:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:10:40.815 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:40.815 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:40 smithi142 podman[132537]: 2024-08-17 17:10:40.518524148 +0000 UTC m=+0.825824766 container remove c95b78eb89ad044c8414c01a8f26d2eb05627558f1d56b25422b0a033883f73f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-activate, RELEASE=main-cd8bdbe, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-17T17:10:40.815 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:40 smithi142 ceph-mon[103614]: pgmap v260: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 19429/140529 objects degraded (13.826%); 0 B/s, 81 objects/s recovering 2024-08-17T17:10:40.815 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2610211549' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:40.815 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 19429/140529 objects degraded (13.826%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:40.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:40 smithi142 ceph-mon[106729]: pgmap v260: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 19429/140529 objects degraded (13.826%); 0 B/s, 81 objects/s recovering 2024-08-17T17:10:40.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2610211549' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:40.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 19429/140529 objects degraded (13.826%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:41.122 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:40 smithi142 podman[132635]: 2024-08-17 17:10:40.755687832 +0000 UTC m=+0.038819879 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:10:41.123 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:40 smithi142 podman[132635]: 2024-08-17 17:10:40.873251633 +0000 UTC m=+0.156383671 container create 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-08-17T17:10:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:41 smithi142 ceph-mon[103614]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:41 smithi142 ceph-mon[106729]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:41.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:41 smithi142 podman[132635]: 2024-08-17 17:10:41.184632765 +0000 UTC m=+0.467764821 container init 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-17T17:10:41.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:41 smithi142 podman[132635]: 2024-08-17 17:10:41.20044428 +0000 UTC m=+0.483576325 container start 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:10:41.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:41 smithi142 ceph-osd[132694]: -- 172.21.15.142:0/3303861908 <== mon.0 v2:172.21.15.142:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b73131fa40 con 0x55b7314c2400 2024-08-17T17:10:41.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:41 smithi142 bash[132635]: 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de 2024-08-17T17:10:41.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:41 smithi142 systemd[1]: Started Ceph osd.2 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:10:41.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:41 smithi152 ceph-mon[93456]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:42.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:10:41] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.51.0" 2024-08-17T17:10:42.818 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: pgmap v261: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 19429/136437 objects degraded (14.240%); 0 B/s, 73 objects/s recovering 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: from='client.35250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: pgmap v261: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 19429/136437 objects degraded (14.240%); 0 B/s, 73 objects/s recovering 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: from='client.35250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:42.819 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: pgmap v261: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 19429/136437 objects degraded (14.240%); 0 B/s, 73 objects/s recovering 2024-08-17T17:10:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: from='client.35250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:43.123 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:42 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[132671]: 2024-08-17T17:10:42.868+0000 7ffbf51c1640 -1 Falling back to public interface 2024-08-17T17:10:44.427 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:44 smithi152 ceph-mon[93456]: pgmap v262: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 17641/127677 objects degraded (13.817%); 0 B/s, 87 objects/s recovering 2024-08-17T17:10:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3587274744' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:44.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:44 smithi142 ceph-mon[106729]: pgmap v262: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 17641/127677 objects degraded (13.817%); 0 B/s, 87 objects/s recovering 2024-08-17T17:10:44.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3587274744' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:44.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:44 smithi142 ceph-mon[103614]: pgmap v262: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 17641/127677 objects degraded (13.817%); 0 B/s, 87 objects/s recovering 2024-08-17T17:10:44.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3587274744' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:45.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:45 smithi142 ceph-mon[103614]: from='client.54967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:45.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:45 smithi142 ceph-mon[103614]: from='client.54970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:45.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17641/127677 objects degraded (13.817%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:45.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:45 smithi142 ceph-mon[106729]: from='client.54967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:45.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:45 smithi142 ceph-mon[106729]: from='client.54970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:45.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17641/127677 objects degraded (13.817%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:45 smithi152 ceph-mon[93456]: from='client.54967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:45 smithi152 ceph-mon[93456]: from='client.54970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17641/127677 objects degraded (13.817%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:46 smithi152 ceph-mon[93456]: pgmap v263: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17641/127002 objects degraded (13.890%); 0 B/s, 73 objects/s recovering 2024-08-17T17:10:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:46 smithi142 ceph-mon[106729]: pgmap v263: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17641/127002 objects degraded (13.890%); 0 B/s, 73 objects/s recovering 2024-08-17T17:10:46.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:46.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:46 smithi142 ceph-mon[103614]: pgmap v263: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17641/127002 objects degraded (13.890%); 0 B/s, 73 objects/s recovering 2024-08-17T17:10:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:48.059 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:48.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[103614]: pgmap v264: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 17641/124086 objects degraded (14.217%); 0 B/s, 72 objects/s recovering 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3846951968' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[106729]: pgmap v264: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 17641/124086 objects degraded (14.217%); 0 B/s, 72 objects/s recovering 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3846951968' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:48.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:48 smithi152 ceph-mon[93456]: pgmap v264: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 17641/124086 objects degraded (14.217%); 0 B/s, 72 objects/s recovering 2024-08-17T17:10:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3846951968' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:49.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:49 smithi142 ceph-mon[103614]: from='client.35271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:49.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:49 smithi142 ceph-mon[103614]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:49.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:49 smithi142 ceph-mon[106729]: from='client.35271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:49.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:49 smithi142 ceph-mon[106729]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:49 smithi152 ceph-mon[93456]: from='client.35271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:49 smithi152 ceph-mon[93456]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:50 smithi152 ceph-mon[93456]: pgmap v265: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 15971/115986 objects degraded (13.770%); 0 B/s, 51 objects/s recovering 2024-08-17T17:10:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15971/115986 objects degraded (13.770%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:10:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:10:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:10:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:50 smithi142 ceph-mon[106729]: pgmap v265: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 15971/115986 objects degraded (13.770%); 0 B/s, 51 objects/s recovering 2024-08-17T17:10:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15971/115986 objects degraded (13.770%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:50 smithi142 ceph-mon[103614]: pgmap v265: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 15971/115986 objects degraded (13.770%); 0 B/s, 51 objects/s recovering 2024-08-17T17:10:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15971/115986 objects degraded (13.770%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:51.753 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:51.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3099369505' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:51.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3099369505' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3099369505' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:10:51] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.51.0" 2024-08-17T17:10:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:52 smithi152 ceph-mon[93456]: pgmap v266: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 15971/112410 objects degraded (14.208%); 0 B/s, 36 objects/s recovering 2024-08-17T17:10:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:52 smithi152 ceph-mon[93456]: from='client.54997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:52 smithi142 ceph-mon[106729]: pgmap v266: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 15971/112410 objects degraded (14.208%); 0 B/s, 36 objects/s recovering 2024-08-17T17:10:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:52 smithi142 ceph-mon[106729]: from='client.54997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:52 smithi142 ceph-mon[103614]: pgmap v266: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 15971/112410 objects degraded (14.208%); 0 B/s, 36 objects/s recovering 2024-08-17T17:10:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:52 smithi142 ceph-mon[103614]: from='client.54997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-mon[103614]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-mon[103614]: from='osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:10:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-mon[103614]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:10:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-mon[106729]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-mon[106729]: from='osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:10:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-mon[106729]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:10:53.624 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[132671]: 2024-08-17T17:10:53.281+0000 7ffbf51c1640 -1 osd.2 116 log_to_monitors true 2024-08-17T17:10:53.624 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[132671]: 2024-08-17T17:10:53.508+0000 7ffbecf6b640 -1 osd.2 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:10:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:53 smithi152 ceph-mon[93456]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:53 smithi152 ceph-mon[93456]: from='osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:10:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:53 smithi152 ceph-mon[93456]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:10:54.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:10:53 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:10:53.996+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: pgmap v267: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 14315/104586 objects degraded (13.687%); 0 B/s, 55 objects/s recovering 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: osdmap e119: 8 total, 7 up, 8 in 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:10:54.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:54 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: pgmap v267: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 14315/104586 objects degraded (13.687%); 0 B/s, 55 objects/s recovering 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: osdmap e119: 8 total, 7 up, 8 in 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:10:54.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: pgmap v267: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 14315/104586 objects degraded (13.687%); 0 B/s, 55 objects/s recovering 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:10:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: osdmap e119: 8 total, 7 up, 8 in 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:10:54.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:54 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-08-17T17:10:55.003 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937] boot 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/371705818' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14315/104586 objects degraded (13.687%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:55 smithi152 ceph-mon[93456]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T17:10:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:10:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937] boot 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/371705818' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14315/104586 objects degraded (13.687%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[106729]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: osd.2 [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937] boot 2024-08-17T17:10:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:10:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:10:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/371705818' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14315/104586 objects degraded (13.687%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:10:55.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:55 smithi142 ceph-mon[103614]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T17:10:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:56 smithi152 ceph-mon[93456]: pgmap v270: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 14315/103896 objects degraded (13.778%); 0 B/s, 55 objects/s recovering 2024-08-17T17:10:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:56 smithi152 ceph-mon[93456]: from='client.35304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:56 smithi152 ceph-mon[93456]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T17:10:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[106729]: pgmap v270: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 14315/103896 objects degraded (13.778%); 0 B/s, 55 objects/s recovering 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[106729]: from='client.35304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[106729]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[103614]: pgmap v270: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 9 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 14315/103896 objects degraded (13.778%); 0 B/s, 55 objects/s recovering 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[103614]: from='client.35304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[103614]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:10:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:58.427 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:10:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:58 smithi152 ceph-mon[93456]: pgmap v273: 89 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 6 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 350 op/s; 11064/100380 objects degraded (11.022%) 2024-08-17T17:10:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/202052717' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:58 smithi142 ceph-mon[106729]: pgmap v273: 89 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 6 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 350 op/s; 11064/100380 objects degraded (11.022%) 2024-08-17T17:10:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/202052717' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:58 smithi142 ceph-mon[103614]: pgmap v273: 89 pgs: 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 6 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 350 op/s; 11064/100380 objects degraded (11.022%) 2024-08-17T17:10:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/202052717' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:10:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:59 smithi152 ceph-mon[93456]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:10:59 smithi152 ceph-mon[93456]: from='client.55033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:59 smithi142 ceph-mon[106729]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:59.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:10:59 smithi142 ceph-mon[106729]: from='client.55033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:59 smithi142 ceph-mon[103614]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:10:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:10:59 smithi142 ceph-mon[103614]: from='client.55033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:00.516 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:11:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:11:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:11:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:00 smithi152 ceph-mon[93456]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized, 77 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 5115/96546 objects degraded (5.298%); 0 B/s, 160 objects/s recovering 2024-08-17T17:11:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5115/96546 objects degraded (5.298%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:11:00.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:00 smithi142 ceph-mon[103614]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized, 77 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 5115/96546 objects degraded (5.298%); 0 B/s, 160 objects/s recovering 2024-08-17T17:11:00.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5115/96546 objects degraded (5.298%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:11:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:00 smithi142 ceph-mon[106729]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized, 77 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 5115/96546 objects degraded (5.298%); 0 B/s, 160 objects/s recovering 2024-08-17T17:11:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5115/96546 objects degraded (5.298%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:11:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1838831764' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:01.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1838831764' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:01.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:11:01] "GET /metrics HTTP/1.1" 200 36225 "" "Prometheus/2.51.0" 2024-08-17T17:11:01.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1838831764' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:02.055 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:02 smithi152 ceph-mon[93456]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 6156/93285 objects degraded (6.599%); 0 B/s, 137 objects/s recovering 2024-08-17T17:11:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:02 smithi152 ceph-mon[93456]: from='client.55036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:02.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:02 smithi142 ceph-mon[103614]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 6156/93285 objects degraded (6.599%); 0 B/s, 137 objects/s recovering 2024-08-17T17:11:02.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:02 smithi142 ceph-mon[103614]: from='client.55036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:02.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:02 smithi142 ceph-mon[106729]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 6156/93285 objects degraded (6.599%); 0 B/s, 137 objects/s recovering 2024-08-17T17:11:02.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:02 smithi142 ceph-mon[106729]: from='client.55036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:03 smithi152 ceph-mon[93456]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T17:11:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:03 smithi152 ceph-mon[93456]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:03 smithi142 ceph-mon[106729]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T17:11:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:03 smithi142 ceph-mon[106729]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:03 smithi142 ceph-mon[103614]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T17:11:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:03 smithi142 ceph-mon[103614]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:04 smithi152 ceph-mon[93456]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 5932/86517 objects degraded (6.856%); 0 B/s, 188 objects/s recovering 2024-08-17T17:11:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:04 smithi152 ceph-mon[93456]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T17:11:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:04 smithi142 ceph-mon[106729]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 5932/86517 objects degraded (6.856%); 0 B/s, 188 objects/s recovering 2024-08-17T17:11:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:04 smithi142 ceph-mon[106729]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T17:11:04.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:04 smithi142 ceph-mon[103614]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 5932/86517 objects degraded (6.856%); 0 B/s, 188 objects/s recovering 2024-08-17T17:11:04.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:04 smithi142 ceph-mon[103614]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T17:11:05.664 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2260701368' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5932/86517 objects degraded (6.856%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:11:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2260701368' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5932/86517 objects degraded (6.856%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:11:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2260701368' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5932/86517 objects degraded (6.856%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:11:06.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:06 smithi152 ceph-mon[93456]: pgmap v279: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5932/85932 objects degraded (6.903%); 0 B/s, 187 objects/s recovering 2024-08-17T17:11:06.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:06 smithi152 ceph-mon[93456]: from='client.45155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:06 smithi142 ceph-mon[103614]: pgmap v279: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5932/85932 objects degraded (6.903%); 0 B/s, 187 objects/s recovering 2024-08-17T17:11:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:06 smithi142 ceph-mon[103614]: from='client.45155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:06 smithi142 ceph-mon[106729]: pgmap v279: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5932/85932 objects degraded (6.903%); 0 B/s, 187 objects/s recovering 2024-08-17T17:11:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:06 smithi142 ceph-mon[106729]: from='client.45155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:07.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:07 smithi142 ceph-mon[103614]: from='client.55048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:07.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:07 smithi142 ceph-mon[106729]: from='client.55048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:07 smithi152 ceph-mon[93456]: from='client.55048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:08 smithi142 ceph-mon[103614]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5566/82110 objects degraded (6.779%); 0 B/s, 117 objects/s recovering 2024-08-17T17:11:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:08 smithi142 ceph-mon[106729]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5566/82110 objects degraded (6.779%); 0 B/s, 117 objects/s recovering 2024-08-17T17:11:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:08 smithi152 ceph-mon[93456]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5566/82110 objects degraded (6.779%); 0 B/s, 117 objects/s recovering 2024-08-17T17:11:09.175 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:09.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:11:09.016+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[106729]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3336895729' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[103614]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3336895729' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:09 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:09 smithi152 ceph-mon[93456]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T17:11:10.110 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3336895729' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:10.111 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:10.111 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:09 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:10.111 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:09 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:11:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:11:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:11:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[106729]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[106729]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5566/77487 objects degraded (7.183%); 0 B/s, 147 objects/s recovering 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[106729]: from='client.45173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[106729]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5566/77487 objects degraded (7.183%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[103614]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[103614]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5566/77487 objects degraded (7.183%); 0 B/s, 147 objects/s recovering 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[103614]: from='client.45173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[103614]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T17:11:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5566/77487 objects degraded (7.183%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:11:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:10 smithi152 ceph-mon[93456]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:10 smithi152 ceph-mon[93456]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5566/77487 objects degraded (7.183%); 0 B/s, 147 objects/s recovering 2024-08-17T17:11:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:10 smithi152 ceph-mon[93456]: from='client.45173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:10 smithi152 ceph-mon[93456]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T17:11:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5566/77487 objects degraded (7.183%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:11:11.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:11 smithi142 ceph-mon[106729]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:11:11.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:11.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:11:11] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.51.0" 2024-08-17T17:11:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:11 smithi142 ceph-mon[103614]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:11:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:11 smithi152 ceph-mon[93456]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:11:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:12.571 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:12 smithi142 ceph-mon[106729]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 5389/74334 objects degraded (7.250%); 0 B/s, 193 objects/s recovering 2024-08-17T17:11:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4200884157' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:12 smithi142 ceph-mon[103614]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 5389/74334 objects degraded (7.250%); 0 B/s, 193 objects/s recovering 2024-08-17T17:11:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4200884157' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:12 smithi152 ceph-mon[93456]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 5389/74334 objects degraded (7.250%); 0 B/s, 193 objects/s recovering 2024-08-17T17:11:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4200884157' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:13 smithi142 ceph-mon[106729]: from='client.45182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:13 smithi142 ceph-mon[103614]: from='client.45182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:13 smithi152 ceph-mon[93456]: from='client.45182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:14 smithi152 ceph-mon[93456]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:14 smithi152 ceph-mon[93456]: pgmap v286: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 4302/69273 objects degraded (6.210%); 0 B/s, 245 objects/s recovering 2024-08-17T17:11:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:14 smithi142 ceph-mon[106729]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:14 smithi142 ceph-mon[106729]: pgmap v286: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 4302/69273 objects degraded (6.210%); 0 B/s, 245 objects/s recovering 2024-08-17T17:11:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:14 smithi142 ceph-mon[103614]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:14 smithi142 ceph-mon[103614]: pgmap v286: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 4302/69273 objects degraded (6.210%); 0 B/s, 245 objects/s recovering 2024-08-17T17:11:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4302/69273 objects degraded (6.210%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4118542812' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4302/69273 objects degraded (6.210%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4118542812' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4302/69273 objects degraded (6.210%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4118542812' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:16.206 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:16 smithi152 ceph-mon[93456]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 4302/68676 objects degraded (6.264%); 0 B/s, 127 objects/s recovering 2024-08-17T17:11:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:16 smithi142 ceph-mon[106729]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 4302/68676 objects degraded (6.264%); 0 B/s, 127 objects/s recovering 2024-08-17T17:11:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:16 smithi142 ceph-mon[103614]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 4302/68676 objects degraded (6.264%); 0 B/s, 127 objects/s recovering 2024-08-17T17:11:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:17 smithi152 ceph-mon[93456]: from='client.55087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:17 smithi152 ceph-mon[93456]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:18.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:17 smithi142 ceph-mon[106729]: from='client.55087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:18.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:17 smithi142 ceph-mon[106729]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:18.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:17 smithi142 ceph-mon[103614]: from='client.55087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:18.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:17 smithi142 ceph-mon[103614]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:18 smithi152 ceph-mon[93456]: pgmap v288: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 4302/65331 objects degraded (6.585%); 0 B/s, 106 objects/s recovering 2024-08-17T17:11:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:18 smithi142 ceph-mon[106729]: pgmap v288: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 4302/65331 objects degraded (6.585%); 0 B/s, 106 objects/s recovering 2024-08-17T17:11:19.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:18 smithi142 ceph-mon[103614]: pgmap v288: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 4302/65331 objects degraded (6.585%); 0 B/s, 106 objects/s recovering 2024-08-17T17:11:19.819 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3034368454' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:20.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3034368454' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:20.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3034368454' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:11:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:11:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:11:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:20 smithi152 ceph-mon[93456]: pgmap v289: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4302/58551 objects degraded (7.347%); 0 B/s, 122 objects/s recovering 2024-08-17T17:11:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:20 smithi152 ceph-mon[93456]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4302/58551 objects degraded (7.347%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:20 smithi152 ceph-mon[93456]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:11:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[106729]: pgmap v289: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4302/58551 objects degraded (7.347%); 0 B/s, 122 objects/s recovering 2024-08-17T17:11:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[106729]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4302/58551 objects degraded (7.347%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[106729]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:11:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[103614]: pgmap v289: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4302/58551 objects degraded (7.347%); 0 B/s, 122 objects/s recovering 2024-08-17T17:11:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[103614]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4302/58551 objects degraded (7.347%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:20 smithi142 ceph-mon[103614]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:11:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:21 smithi152 ceph-mon[93456]: from='client.35385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:21 smithi152 ceph-mon[93456]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:11:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:11:21] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.51.0" 2024-08-17T17:11:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:21 smithi142 ceph-mon[106729]: from='client.35385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:22.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:21 smithi142 ceph-mon[106729]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:11:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:21 smithi142 ceph-mon[103614]: from='client.35385 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:21 smithi142 ceph-mon[103614]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:11:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:22 smithi152 ceph-mon[93456]: pgmap v291: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 4302/54336 objects degraded (7.917%); 0 B/s, 100 objects/s recovering 2024-08-17T17:11:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:22 smithi142 ceph-mon[103614]: pgmap v291: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 4302/54336 objects degraded (7.917%); 0 B/s, 100 objects/s recovering 2024-08-17T17:11:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:22 smithi142 ceph-mon[106729]: pgmap v291: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 4302/54336 objects degraded (7.917%); 0 B/s, 100 objects/s recovering 2024-08-17T17:11:23.490 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2617898362' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:23.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2617898362' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2617898362' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:24.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:11:24.032+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:24.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:24 smithi152 ceph-mon[93456]: pgmap v293: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3252/49695 objects degraded (6.544%); 0 B/s, 93 objects/s recovering 2024-08-17T17:11:24.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:24 smithi152 ceph-mon[93456]: from='client.45203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:24.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:24 smithi152 ceph-mon[93456]: from='client.45209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:24.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:24 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:24.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:24 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:24.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:24 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[106729]: pgmap v293: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3252/49695 objects degraded (6.544%); 0 B/s, 93 objects/s recovering 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[106729]: from='client.45203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[106729]: from='client.45209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[103614]: pgmap v293: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3252/49695 objects degraded (6.544%); 0 B/s, 93 objects/s recovering 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[103614]: from='client.45203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[103614]: from='client.45209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:25.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:25.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:25.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:24 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-08-17T17:11:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3252/49695 objects degraded (6.544%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3252/49695 objects degraded (6.544%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:26.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3252/49695 objects degraded (6.544%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:26 smithi152 ceph-mon[93456]: pgmap v294: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 984 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 3252/47295 objects degraded (6.876%); 0 B/s, 107 objects/s recovering 2024-08-17T17:11:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:26 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1702271065' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:27.114 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:27.115 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:26 smithi142 ceph-mon[103614]: pgmap v294: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 984 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 3252/47295 objects degraded (6.876%); 0 B/s, 107 objects/s recovering 2024-08-17T17:11:27.115 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:27.115 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:26 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1702271065' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:27.115 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:26 smithi142 ceph-mon[106729]: pgmap v294: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 984 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 3252/47295 objects degraded (6.876%); 0 B/s, 107 objects/s recovering 2024-08-17T17:11:27.115 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:27.115 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:26 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1702271065' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:28 smithi152 ceph-mon[93456]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:28 smithi152 ceph-mon[93456]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 919 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 3252/44184 objects degraded (7.360%); 0 B/s, 65 objects/s recovering 2024-08-17T17:11:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:28 smithi152 ceph-mon[93456]: from='client.45218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:28 smithi142 ceph-mon[103614]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:28 smithi142 ceph-mon[103614]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 919 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 3252/44184 objects degraded (7.360%); 0 B/s, 65 objects/s recovering 2024-08-17T17:11:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:28 smithi142 ceph-mon[103614]: from='client.45218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:28 smithi142 ceph-mon[106729]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:28 smithi142 ceph-mon[106729]: pgmap v295: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 919 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 3252/44184 objects degraded (7.360%); 0 B/s, 65 objects/s recovering 2024-08-17T17:11:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:28 smithi142 ceph-mon[106729]: from='client.45218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:11:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:11:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:11:30.720 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:30 smithi152 ceph-mon[93456]: pgmap v296: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 771 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 3252/37080 objects degraded (8.770%); 0 B/s, 100 objects/s recovering 2024-08-17T17:11:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/764923541' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3252/37080 objects degraded (8.770%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:30 smithi152 ceph-mon[93456]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:11:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[106729]: pgmap v296: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 771 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 3252/37080 objects degraded (8.770%); 0 B/s, 100 objects/s recovering 2024-08-17T17:11:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/764923541' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3252/37080 objects degraded (8.770%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[106729]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:11:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[103614]: pgmap v296: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 771 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 3252/37080 objects degraded (8.770%); 0 B/s, 100 objects/s recovering 2024-08-17T17:11:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/764923541' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3252/37080 objects degraded (8.770%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:30 smithi142 ceph-mon[103614]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:11:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:31 smithi152 ceph-mon[93456]: from='client.35424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:31 smithi152 ceph-mon[93456]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:11:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:11:31] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.51.0" 2024-08-17T17:11:32.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:31 smithi142 ceph-mon[106729]: from='client.35424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:32.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:31 smithi142 ceph-mon[106729]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:11:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:31 smithi142 ceph-mon[103614]: from='client.35424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:32.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:31 smithi142 ceph-mon[103614]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:11:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:32 smithi152 ceph-mon[93456]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:32 smithi152 ceph-mon[93456]: pgmap v298: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 682 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2211/32805 objects degraded (6.740%); 0 B/s, 92 objects/s recovering 2024-08-17T17:11:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:32 smithi142 ceph-mon[106729]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:32 smithi142 ceph-mon[106729]: pgmap v298: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 682 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2211/32805 objects degraded (6.740%); 0 B/s, 92 objects/s recovering 2024-08-17T17:11:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:32 smithi142 ceph-mon[103614]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:32 smithi142 ceph-mon[103614]: pgmap v298: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 682 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2211/32805 objects degraded (6.740%); 0 B/s, 92 objects/s recovering 2024-08-17T17:11:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:33 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/42649561' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:34.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:33 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/42649561' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:34.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:33 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/42649561' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:34.166 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:34 smithi152 ceph-mon[93456]: pgmap v300: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 596 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2211/28671 objects degraded (7.712%); 0 B/s, 49 objects/s recovering 2024-08-17T17:11:35.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:34 smithi142 ceph-mon[103614]: pgmap v300: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 596 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2211/28671 objects degraded (7.712%); 0 B/s, 49 objects/s recovering 2024-08-17T17:11:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:34 smithi142 ceph-mon[106729]: pgmap v300: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 596 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2211/28671 objects degraded (7.712%); 0 B/s, 49 objects/s recovering 2024-08-17T17:11:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:35 smithi152 ceph-mon[93456]: from='client.45233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:35 smithi152 ceph-mon[93456]: from='client.35442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2211/28671 objects degraded (7.712%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:35 smithi142 ceph-mon[106729]: from='client.45233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:35 smithi142 ceph-mon[106729]: from='client.35442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2211/28671 objects degraded (7.712%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:35 smithi142 ceph-mon[103614]: from='client.45233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:35 smithi142 ceph-mon[103614]: from='client.35442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2211/28671 objects degraded (7.712%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:36 smithi152 ceph-mon[93456]: pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 526 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 2211/25287 objects degraded (8.744%); 0 B/s, 82 objects/s recovering 2024-08-17T17:11:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:36 smithi142 ceph-mon[103614]: pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 526 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 2211/25287 objects degraded (8.744%); 0 B/s, 82 objects/s recovering 2024-08-17T17:11:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:36 smithi142 ceph-mon[106729]: pgmap v301: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 526 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 2211/25287 objects degraded (8.744%); 0 B/s, 82 objects/s recovering 2024-08-17T17:11:37.822 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:38.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4002890732' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4002890732' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:38.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4002890732' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:38 smithi152 ceph-mon[93456]: pgmap v302: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 454 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 2211/21864 objects degraded (10.113%); 0 B/s, 76 objects/s recovering 2024-08-17T17:11:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:38 smithi152 ceph-mon[93456]: from='client.45239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:38 smithi142 ceph-mon[103614]: pgmap v302: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 454 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 2211/21864 objects degraded (10.113%); 0 B/s, 76 objects/s recovering 2024-08-17T17:11:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:38 smithi142 ceph-mon[103614]: from='client.45239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:39.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:11:39.051+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-08-17T17:11:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:38 smithi142 ceph-mon[106729]: pgmap v302: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 454 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 2211/21864 objects degraded (10.113%); 0 B/s, 76 objects/s recovering 2024-08-17T17:11:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:38 smithi142 ceph-mon[106729]: from='client.45239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:40.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:39 smithi142 ceph-mon[106729]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:40.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:39 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:39 smithi142 ceph-mon[103614]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:39 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:39 smithi152 ceph-mon[93456]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:39 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:11:40.213 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:11:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=cleanup t=2024-08-17T17:11:39.869877717Z level=info msg="Completed cleanup jobs" duration=12.157623ms 2024-08-17T17:11:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:11:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:11:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:11:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-08-17T17:11:41.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[106729]: pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 308 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2211/14817 objects degraded (14.922%); 0 B/s, 68 objects/s recovering 2024-08-17T17:11:41.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2211/14817 objects degraded (14.922%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-08-17T17:11:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[103614]: pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 308 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2211/14817 objects degraded (14.922%); 0 B/s, 68 objects/s recovering 2024-08-17T17:11:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2211/14817 objects degraded (14.922%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:40 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:11:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:40 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-08-17T17:11:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:40 smithi152 ceph-mon[93456]: pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 308 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2211/14817 objects degraded (14.922%); 0 B/s, 68 objects/s recovering 2024-08-17T17:11:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2211/14817 objects degraded (14.922%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:41.476 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:42.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/576455560' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:42.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:11:41] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.51.0" 2024-08-17T17:11:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/576455560' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:42.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/576455560' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:43.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:42 smithi142 ceph-mon[103614]: pgmap v304: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 216 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 2211/10434 objects degraded (21.190%); 0 B/s, 83 objects/s recovering 2024-08-17T17:11:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:42 smithi142 ceph-mon[103614]: from='client.35463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:42 smithi142 ceph-mon[103614]: from='client.35466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:42 smithi142 ceph-mon[106729]: pgmap v304: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 216 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 2211/10434 objects degraded (21.190%); 0 B/s, 83 objects/s recovering 2024-08-17T17:11:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:42 smithi142 ceph-mon[106729]: from='client.35463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:43.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:42 smithi142 ceph-mon[106729]: from='client.35466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:42 smithi152 ceph-mon[93456]: pgmap v304: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 216 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 2211/10434 objects degraded (21.190%); 0 B/s, 83 objects/s recovering 2024-08-17T17:11:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:42 smithi152 ceph-mon[93456]: from='client.35463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:42 smithi152 ceph-mon[93456]: from='client.35466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:43.497 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 111691 objects 2024-08-17T17:11:43.497 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :154.259 2024-08-17T17:11:44.641 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:44 smithi142 ceph-mon[106729]: pgmap v305: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 128 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2211/6186 objects degraded (35.742%); 0 B/s, 70 objects/s recovering 2024-08-17T17:11:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2715160625' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:44 smithi142 ceph-mon[103614]: pgmap v305: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 128 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2211/6186 objects degraded (35.742%); 0 B/s, 70 objects/s recovering 2024-08-17T17:11:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2715160625' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:44 smithi152 ceph-mon[93456]: pgmap v305: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 128 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2211/6186 objects degraded (35.742%); 0 B/s, 70 objects/s recovering 2024-08-17T17:11:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2715160625' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[106729]: from='client.45266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[106729]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:11:46.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[106729]: from='client.45272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:46.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2211/6186 objects degraded (35.742%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[103614]: from='client.45266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[103614]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:11:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[103614]: from='client.45272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2211/6186 objects degraded (35.742%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:45 smithi152 ceph-mon[93456]: from='client.45266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:45 smithi152 ceph-mon[93456]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:11:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:45 smithi152 ceph-mon[93456]: from='client.45272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2211/6186 objects degraded (35.742%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:11:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:46 smithi142 ceph-mon[106729]: pgmap v307: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 2211/2910 objects degraded (75.979%); 0 B/s, 57 objects/s recovering 2024-08-17T17:11:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:46 smithi142 ceph-mon[106729]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:11:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:46 smithi142 ceph-mon[103614]: pgmap v307: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 2211/2910 objects degraded (75.979%); 0 B/s, 57 objects/s recovering 2024-08-17T17:11:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:46 smithi142 ceph-mon[103614]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:11:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:46 smithi152 ceph-mon[93456]: pgmap v307: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 60 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 2211/2910 objects degraded (75.979%); 0 B/s, 57 objects/s recovering 2024-08-17T17:11:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:46 smithi152 ceph-mon[93456]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:11:47.737 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:47 smithi142 ceph-mon[106729]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T17:11:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2133664357' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:47 smithi142 ceph-mon[103614]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T17:11:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2133664357' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:47 smithi152 ceph-mon[93456]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T17:11:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2133664357' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:48 smithi152 ceph-mon[93456]: pgmap v310: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 31 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 2211/1539 objects degraded (143.665%) 2024-08-17T17:11:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:48 smithi152 ceph-mon[93456]: from='client.35496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:48 smithi152 ceph-mon[93456]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T17:11:49.325 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-08-17T17:11:49.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-08-17T17:11:49.351 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:48 smithi142 ceph-mon[106729]: pgmap v310: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 31 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 2211/1539 objects degraded (143.665%) 2024-08-17T17:11:49.351 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:48 smithi142 ceph-mon[106729]: from='client.35496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.351 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:48 smithi142 ceph-mon[106729]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T17:11:49.351 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:48 smithi142 ceph-mon[103614]: pgmap v310: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 31 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 2211/1539 objects degraded (143.665%) 2024-08-17T17:11:49.352 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:48 smithi142 ceph-mon[103614]: from='client.35496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.352 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:48 smithi142 ceph-mon[103614]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T17:11:49.583 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:49.982 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:49 smithi142 ceph-mon[103614]: from='client.35502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.982 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:49 smithi142 ceph-mon[103614]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T17:11:49.982 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:49 smithi142 ceph-mon[106729]: from='client.35502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:49.983 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:49 smithi142 ceph-mon[106729]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T17:11:50.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:49 smithi152 ceph-mon[93456]: from='client.35502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:50.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:49 smithi152 ceph-mon[93456]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T17:11:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:11:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:11:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:11:50.629 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:51.104 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[106729]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 0 B/s, 616 objects/s recovering 2024-08-17T17:11:51.104 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2211/1539 objects degraded (143.665%), 2 pgs degraded) 2024-08-17T17:11:51.104 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:11:51.104 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[106729]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T17:11:51.105 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1617259157' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:51.105 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[103614]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 0 B/s, 616 objects/s recovering 2024-08-17T17:11:51.105 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2211/1539 objects degraded (143.665%), 2 pgs degraded) 2024-08-17T17:11:51.105 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:11:51.105 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[103614]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T17:11:51.105 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1617259157' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:50 smithi152 ceph-mon[93456]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 0 B/s, 616 objects/s recovering 2024-08-17T17:11:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:50 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2211/1539 objects degraded (143.665%), 2 pgs degraded) 2024-08-17T17:11:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:50 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:11:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:50 smithi152 ceph-mon[93456]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T17:11:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1617259157' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:51.891 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_2' removed 2024-08-17T17:11:52.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-mon[103614]: from='client.45302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-mon[103614]: from='client.35520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1100927049' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:11:52.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:11:51] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.51.0" 2024-08-17T17:11:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-mon[106729]: from='client.45302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-mon[106729]: from='client.35520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:52.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1100927049' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:11:52.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:51 smithi152 ceph-mon[93456]: from='client.45302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:52.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:51 smithi152 ceph-mon[93456]: from='client.35520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:52.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1100927049' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:11:52.593 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:11:52.594 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:11:52.594 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-08-17T17:11:52.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-08-17T17:11:52.886 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:53.234 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:52 smithi142 ceph-mon[106729]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 0 B/s, 561 objects/s recovering 2024-08-17T17:11:53.234 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1100927049' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:11:53.234 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:52 smithi142 ceph-mon[106729]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T17:11:53.235 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:52 smithi142 ceph-mon[103614]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 0 B/s, 561 objects/s recovering 2024-08-17T17:11:53.235 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1100927049' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:11:53.235 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:52 smithi142 ceph-mon[103614]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T17:11:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:52 smithi152 ceph-mon[93456]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 0 B/s, 561 objects/s recovering 2024-08-17T17:11:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1100927049' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:11:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:52 smithi152 ceph-mon[93456]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T17:11:53.494 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:54.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/488845211' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:54.260 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/488845211' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:54.261 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/488845211' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='client.45326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='client.35544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3847182610' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:11:55.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:54 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='client.45326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='client.35544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3847182610' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:11:55.255 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: pgmap v317: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='client.45326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='client.35544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3847182610' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:11:55.256 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:54 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:11:55.457 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_3' created 2024-08-17T17:11:56.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-08-17T17:11:56.369 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:56.471 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[103614]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:56.471 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:11:56.471 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[103614]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T17:11:56.471 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2987738644' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:56.471 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:56.471 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:56.488 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:56.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[106729]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:56.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:11:56.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[106729]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T17:11:56.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2987738644' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:56.758 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:56.758 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:56 smithi152 ceph-mon[93456]: pgmap v318: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:11:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:56 smithi152 ceph-mon[93456]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T17:11:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2987738644' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:57.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[106729]: from='client.35562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:57.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[106729]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T17:11:57.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[106729]: from='client.45356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:57.816 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[106729]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T17:11:57.816 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[103614]: from='client.35562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:57.817 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[103614]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T17:11:57.817 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[103614]: from='client.45356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:57.817 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:57 smithi142 ceph-mon[103614]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T17:11:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:57 smithi152 ceph-mon[93456]: from='client.35562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:57 smithi152 ceph-mon[93456]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T17:11:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:57 smithi152 ceph-mon[93456]: from='client.45356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:11:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:57 smithi152 ceph-mon[93456]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T17:11:58.688 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[106729]: pgmap v321: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:58.688 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:11:58.688 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:58.688 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:58.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[103614]: pgmap v321: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:58.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:11:58.689 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:58.689 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:58 smithi152 ceph-mon[93456]: pgmap v321: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:11:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:58 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:11:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:11:59.288 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:11:59.482 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-08-17T17:11:59.498 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1578985824' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:11:59.498 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:11:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1623282427' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:59.499 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1578985824' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:11:59.499 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:11:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1623282427' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:11:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1578985824' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:11:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:11:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1623282427' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:00.483 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:12:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:12:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:12:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:00 smithi152 ceph-mon[93456]: pgmap v323: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:12:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:00 smithi152 ceph-mon[93456]: from='client.45374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1578985824' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:12:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:00 smithi152 ceph-mon[93456]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T17:12:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:00 smithi152 ceph-mon[93456]: from='client.45380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[106729]: pgmap v323: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[106729]: from='client.45374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1578985824' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[106729]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[106729]: from='client.45380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[103614]: pgmap v323: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[103614]: from='client.45374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1578985824' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[103614]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T17:12:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:00 smithi142 ceph-mon[103614]: from='client.45380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:01.113 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-08-17T17:12:01.115 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:12:01.115 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:12:01.161 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:12:01.161 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:12:01.161 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_139631 2024-08-17T17:12:01.161 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:12:01.161 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:12:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:12:01] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.51.0" 2024-08-17T17:12:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:12:02.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 3413 3397 212.238 212.312 0.00502342 0.00469509 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:02.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:12:02.775 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:03.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 6788 6772 211.579 210.938 0.00422489 0.00471332 2024-08-17T17:12:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:02 smithi152 ceph-mon[93456]: pgmap v325: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:12:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:02 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:12:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:02 smithi152 ceph-mon[93456]: Upgrade: osd.3 is safe to restart 2024-08-17T17:12:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:02 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:12:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:02 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:12:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3700270223' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:03.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[103614]: pgmap v325: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:12:03.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:12:03.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[103614]: Upgrade: osd.3 is safe to restart 2024-08-17T17:12:03.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:12:03.329 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3700270223' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[106729]: pgmap v325: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[106729]: Upgrade: osd.3 is safe to restart 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:12:03.330 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3700270223' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:04.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 9888 9872 205.629 193.75 0.00840617 0.0048543 2024-08-17T17:12:04.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:03 smithi152 ceph-mon[93456]: from='client.45401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:04.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:03 smithi142 ceph-mon[103614]: from='client.45401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:04.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:03 smithi142 ceph-mon[106729]: from='client.45401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:05.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 13129 13113 204.858 202.562 0.00341454 0.00487404 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: pgmap v326: 89 pgs: 89 active+clean; 88 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 206 op/s 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: from='client.55273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: Upgrade: Updating osd.3 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:05.201 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[106729]: Deploying daemon osd.3 on smithi142 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: pgmap v326: 89 pgs: 89 active+clean; 88 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 206 op/s 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: from='client.55273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: Upgrade: Updating osd.3 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:05.202 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:04 smithi142 ceph-mon[103614]: Deploying daemon osd.3 on smithi142 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: pgmap v326: 89 pgs: 89 active+clean; 88 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 206 op/s 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: from='client.55273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: Upgrade: Updating osd.3 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:04 smithi152 ceph-mon[93456]: Deploying daemon osd.3 on smithi142 2024-08-17T17:12:06.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 16038 16022 200.246 181.812 0.00401582 0.00498466 2024-08-17T17:12:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1056409241' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1056409241' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1056409241' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:06.491 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:07.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:06 smithi142 ceph-mon[103614]: pgmap v327: 89 pgs: 89 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2024-08-17T17:12:07.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:06 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:07.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:06 smithi142 ceph-mon[106729]: pgmap v327: 89 pgs: 89 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2024-08-17T17:12:07.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:06 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:07.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 18424 18408 191.724 149.125 0.00330896 0.00520991 2024-08-17T17:12:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:06 smithi152 ceph-mon[93456]: pgmap v327: 89 pgs: 89 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2024-08-17T17:12:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:06 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:08.162 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 21329 21313 190.261 181.562 0.00590396 0.00525055 2024-08-17T17:12:08.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:08 smithi152 ceph-mon[93456]: from='client.35613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:08.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:08 smithi142 ceph-mon[103614]: from='client.35613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:08.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:08 smithi142 ceph-mon[106729]: from='client.35613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:08.872 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:08 smithi142 systemd[1]: Stopping Ceph osd.3 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:12:09.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 23709 23693 185.071 148.75 0.00185451 0.00535971 2024-08-17T17:12:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:09 smithi152 ceph-mon[93456]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:09 smithi152 ceph-mon[93456]: pgmap v328: 89 pgs: 89 active+clean; 525 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2024-08-17T17:12:09.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-mon[103614]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-mon[103614]: pgmap v328: 89 pgs: 89 active+clean; 525 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2024-08-17T17:12:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-mon[106729]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-mon[106729]: pgmap v328: 89 pgs: 89 active+clean; 525 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2024-08-17T17:12:10.061 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:10.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 25697 25681 178.313 124.25 0.00216985 0.00552201 2024-08-17T17:12:10.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1644705663' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:10.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:10 smithi152 ceph-mon[93456]: osd.3 marked itself down and dead 2024-08-17T17:12:10.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:12:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:12:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:12:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1644705663' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:10 smithi142 ceph-mon[106729]: osd.3 marked itself down and dead 2024-08-17T17:12:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1644705663' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:10 smithi142 ceph-mon[103614]: osd.3 marked itself down and dead 2024-08-17T17:12:10.373 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[63049]: 2024-08-17T17:12:09.896+0000 7f0b65708640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:12:10.373 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[63049]: 2024-08-17T17:12:09.897+0000 7f0b65708640 -1 osd.3 142 *** Got signal Terminated *** 2024-08-17T17:12:10.373 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:09 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[63049]: 2024-08-17T17:12:09.897+0000 7f0b65708640 -1 osd.3 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:12:11.009 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:10 smithi142 podman[140296]: 2024-08-17 17:12:10.662633897 +0000 UTC m=+1.889543169 container died d53780492c5b1229da75b099d29d77da863b42396edc1cd934258c3b56183bd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-08-17T17:12:11.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 26072 26056 162.826 23.4375 0.00619497 0.00613688 2024-08-17T17:12:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:11 smithi152 ceph-mon[93456]: pgmap v329: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:12:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:11 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:12:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:11 smithi152 ceph-mon[93456]: osdmap e143: 8 total, 7 up, 8 in 2024-08-17T17:12:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-mon[106729]: pgmap v329: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:12:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:12:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-mon[106729]: osdmap e143: 8 total, 7 up, 8 in 2024-08-17T17:12:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-mon[103614]: pgmap v329: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:12:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:12:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-mon[103614]: osdmap e143: 8 total, 7 up, 8 in 2024-08-17T17:12:12.030 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:12:11] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-08-17T17:12:12.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 28186 28170 160.034 132.125 0.00487887 0.00624289 2024-08-17T17:12:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:12 smithi152 ceph-mon[93456]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:12 smithi152 ceph-mon[93456]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:12 smithi152 ceph-mon[93456]: osdmap e144: 8 total, 7 up, 8 in 2024-08-17T17:12:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:12.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[106729]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:12.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[106729]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[106729]: osdmap e144: 8 total, 7 up, 8 in 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[103614]: from='client.45428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[103614]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[103614]: osdmap e144: 8 total, 7 up, 8 in 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:12.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:13.109 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:13.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 29587 29571 153.995 87.5625 0.00322289 0.00648205 2024-08-17T17:12:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:13 smithi152 ceph-mon[93456]: pgmap v332: 89 pgs: 8 peering, 14 stale+active+clean, 67 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:12:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:13 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:12:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/870636024' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:13.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:13 smithi142 ceph-mon[106729]: pgmap v332: 89 pgs: 8 peering, 14 stale+active+clean, 67 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:12:13.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:13 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:12:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/870636024' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:13.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:13 smithi142 ceph-mon[103614]: pgmap v332: 89 pgs: 8 peering, 14 stale+active+clean, 67 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:12:13.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:13 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:12:13.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/870636024' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:14.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 31636 31620 151.999 128.062 0.0118644 0.0065731 2024-08-17T17:12:14.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[106729]: from='client.45446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:14.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[106729]: pgmap v333: 89 pgs: 13 active+undersized, 8 peering, 6 stale+active+clean, 5 active+undersized+degraded, 57 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s; 5335/79830 objects degraded (6.683%) 2024-08-17T17:12:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[106729]: from='client.35649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:14.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 5335/79830 objects degraded (6.683%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[103614]: from='client.45446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[103614]: pgmap v333: 89 pgs: 13 active+undersized, 8 peering, 6 stale+active+clean, 5 active+undersized+degraded, 57 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s; 5335/79830 objects degraded (6.683%) 2024-08-17T17:12:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[103614]: from='client.35649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:14.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:14 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 5335/79830 objects degraded (6.683%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:14 smithi152 ceph-mon[93456]: from='client.45446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:14 smithi152 ceph-mon[93456]: pgmap v333: 89 pgs: 13 active+undersized, 8 peering, 6 stale+active+clean, 5 active+undersized+degraded, 57 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s; 5335/79830 objects degraded (6.683%) 2024-08-17T17:12:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:14 smithi152 ceph-mon[93456]: from='client.35649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:14 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 5335/79830 objects degraded (6.683%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:15.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 33767 33751 150.655 133.188 0.00707049 0.00663259 2024-08-17T17:12:16.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 35994 35978 149.889 139.188 0.00236963 0.00666612 2024-08-17T17:12:16.429 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:16.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:16 smithi152 ceph-mon[93456]: pgmap v334: 89 pgs: 22 active+undersized, 8 peering, 7 active+undersized+degraded, 52 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s; 9242/87315 objects degraded (10.585%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:12:16.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/761284398' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:16.596 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:16 smithi142 ceph-mon[106729]: pgmap v334: 89 pgs: 22 active+undersized, 8 peering, 7 active+undersized+degraded, 52 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s; 9242/87315 objects degraded (10.585%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:12:16.596 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/761284398' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:16.597 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:16 smithi142 ceph-mon[103614]: pgmap v334: 89 pgs: 22 active+undersized, 8 peering, 7 active+undersized+degraded, 52 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s; 9242/87315 objects degraded (10.585%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:12:16.597 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/761284398' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:16.872 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:16 smithi142 podman[140296]: 2024-08-17 17:12:16.595806643 +0000 UTC m=+7.822715905 container cleanup d53780492c5b1229da75b099d29d77da863b42396edc1cd934258c3b56183bd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:12:16.872 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:16 smithi142 bash[140296]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:12:17.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 37600 37584 146.794 100.375 0.00641579 0.0068078 2024-08-17T17:12:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:17 smithi142 ceph-mon[106729]: from='client.35661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:17 smithi142 ceph-mon[106729]: from='client.35667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:17 smithi142 ceph-mon[103614]: from='client.35661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:17 smithi142 ceph-mon[103614]: from='client.35667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:17 smithi152 ceph-mon[93456]: from='client.35661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:17 smithi152 ceph-mon[93456]: from='client.35667 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:18.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 15 40014 39999 147.037 150.938 0.00480118 0.00679621 2024-08-17T17:12:18.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:18 smithi142 ceph-mon[106729]: pgmap v335: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 11444/92778 objects degraded (12.335%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:12:18.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:18 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:12:18.623 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:18 smithi142 podman[140392]: 2024-08-17 17:12:18.214572464 +0000 UTC m=+7.551063326 container remove d53780492c5b1229da75b099d29d77da863b42396edc1cd934258c3b56183bd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:12:18.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:18 smithi142 ceph-mon[103614]: pgmap v335: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 11444/92778 objects degraded (12.335%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:12:18.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:18 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:12:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:18 smithi152 ceph-mon[93456]: pgmap v335: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 1.9 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 11444/92778 objects degraded (12.335%); 8.0 KiB/s, 0 objects/s recovering 2024-08-17T17:12:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:18 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-08-17T17:12:19.163 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 42300 42284 146.802 142.812 0.0088788 0.00680664 2024-08-17T17:12:19.228 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:18 smithi142 podman[140637]: 2024-08-17 17:12:18.818149614 +0000 UTC m=+0.038762695 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:19.228 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:18 smithi142 podman[140637]: 2024-08-17 17:12:18.949626375 +0000 UTC m=+0.170239438 container create 2cfa90584feeaba85199f9864815db0879dc6516e0972e4b1e186fd8a82371e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:12:19.543 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:19 smithi142 podman[140637]: 2024-08-17 17:12:19.227463325 +0000 UTC m=+0.448076405 container init 2cfa90584feeaba85199f9864815db0879dc6516e0972e4b1e186fd8a82371e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe) 2024-08-17T17:12:19.543 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:19 smithi142 podman[140637]: 2024-08-17 17:12:19.238144775 +0000 UTC m=+0.458757845 container start 2cfa90584feeaba85199f9864815db0879dc6516e0972e4b1e186fd8a82371e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:12:19.543 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:19 smithi142 podman[140637]: 2024-08-17 17:12:19.352464606 +0000 UTC m=+0.573077675 container attach 2cfa90584feeaba85199f9864815db0879dc6516e0972e4b1e186fd8a82371e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe) 2024-08-17T17:12:19.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4096686683' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:19.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4096686683' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4096686683' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:19.844 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:19 smithi142 podman[140637]: 2024-08-17 17:12:19.544438044 +0000 UTC m=+0.765051113 container died 2cfa90584feeaba85199f9864815db0879dc6516e0972e4b1e186fd8a82371e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:12:19.845 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:20.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 44477 44461 146.236 136.062 0.00339782 0.00683313 2024-08-17T17:12:20.537 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:20 smithi142 podman[140725]: 2024-08-17 17:12:20.182995663 +0000 UTC m=+0.628461207 container remove 2cfa90584feeaba85199f9864815db0879dc6516e0972e4b1e186fd8a82371e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-08-17T17:12:20.537 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:20 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.3.service: Deactivated successfully. 2024-08-17T17:12:20.537 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:20 smithi142 systemd[1]: Stopped Ceph osd.3 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:12:20.537 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:20 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.3.service: Consumed 21min 31.182s CPU time. 2024-08-17T17:12:20.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:20 smithi142 ceph-mon[103614]: pgmap v336: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s; 13887/108840 objects degraded (12.759%); 6.9 KiB/s, 0 objects/s recovering 2024-08-17T17:12:20.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13887/108840 objects degraded (12.759%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:20.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:20 smithi142 ceph-mon[103614]: from='client.45473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:20.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:20 smithi142 ceph-mon[106729]: pgmap v336: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s; 13887/108840 objects degraded (12.759%); 6.9 KiB/s, 0 objects/s recovering 2024-08-17T17:12:20.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13887/108840 objects degraded (12.759%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:20.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:20 smithi142 ceph-mon[106729]: from='client.45473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:20 smithi152 ceph-mon[93456]: pgmap v336: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s; 13887/108840 objects degraded (12.759%); 6.9 KiB/s, 0 objects/s recovering 2024-08-17T17:12:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13887/108840 objects degraded (12.759%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:20 smithi152 ceph-mon[93456]: from='client.45473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:12:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:12:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:12:20.849 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:20 smithi142 systemd[1]: Starting Ceph osd.3 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:12:21.122 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:20 smithi142 podman[140852]: 2024-08-17 17:12:20.759089094 +0000 UTC m=+0.037185744 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:21.122 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 podman[140852]: 2024-08-17 17:12:21.026567048 +0000 UTC m=+0.304663685 container create 9f801a49427df37bbda268f9be56ef75ea09483210c6f12700f1da7da5770120 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:12:21.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:12:21.163876+0000 min lat: 0.00102176 max lat: 1.02694 avg lat: 0.00685958 2024-08-17T17:12:21.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:12:21.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 46638 46622 145.677 135.062 0.00641094 0.00685958 2024-08-17T17:12:21.514 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-mon[103614]: from='client.45479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:21.515 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-mon[106729]: from='client.45479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:21.515 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 podman[140852]: 2024-08-17 17:12:21.28779646 +0000 UTC m=+0.565893114 container init 9f801a49427df37bbda268f9be56ef75ea09483210c6f12700f1da7da5770120 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:12:21.515 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 podman[140852]: 2024-08-17 17:12:21.297887689 +0000 UTC m=+0.575984327 container start 9f801a49427df37bbda268f9be56ef75ea09483210c6f12700f1da7da5770120 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:12:21.515 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 podman[140852]: 2024-08-17 17:12:21.388125963 +0000 UTC m=+0.666222605 container attach 9f801a49427df37bbda268f9be56ef75ea09483210c6f12700f1da7da5770120 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:12:21.771 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.772 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.772 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:12:21.772 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:12:21.772 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:12:21.772 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:12:21.772 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:12:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:21 smithi152 ceph-mon[93456]: from='client.45479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:22.028 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:12:22.028 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate[140864]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:12:22.029 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:21 smithi142 bash[140852]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:12:22.030 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:12:21] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-08-17T17:12:22.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 48811 48795 145.206 135.812 0.00357104 0.00688231 2024-08-17T17:12:22.352 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:22 smithi142 podman[140852]: 2024-08-17 17:12:22.028850778 +0000 UTC m=+1.306947414 container died 9f801a49427df37bbda268f9be56ef75ea09483210c6f12700f1da7da5770120 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:12:22.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:22 smithi142 ceph-mon[103614]: pgmap v337: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s; 15222/122772 objects degraded (12.399%); 6.2 KiB/s, 0 objects/s recovering 2024-08-17T17:12:22.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:22.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:22 smithi142 ceph-mon[106729]: pgmap v337: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s; 15222/122772 objects degraded (12.399%); 6.2 KiB/s, 0 objects/s recovering 2024-08-17T17:12:22.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:22 smithi152 ceph-mon[93456]: pgmap v337: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s; 15222/122772 objects degraded (12.399%); 6.2 KiB/s, 0 objects/s recovering 2024-08-17T17:12:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:23.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 50781 50765 144.202 123.125 0.00544204 0.00692973 2024-08-17T17:12:23.353 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:23.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/635310364' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:23.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/635310364' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/635310364' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:24.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 52813 52797 143.454 127 0.00469254 0.00696647 2024-08-17T17:12:24.554 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:24 smithi142 podman[140989]: 2024-08-17 17:12:24.478431708 +0000 UTC m=+2.435041183 container remove 9f801a49427df37bbda268f9be56ef75ea09483210c6f12700f1da7da5770120 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-17T17:12:24.805 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:24 smithi142 podman[141115]: 2024-08-17 17:12:24.678167936 +0000 UTC m=+0.031341715 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:24.805 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:24 smithi142 ceph-mon[106729]: pgmap v338: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 17323/139605 objects degraded (12.409%); 5.3 KiB/s, 0 objects/s recovering 2024-08-17T17:12:24.805 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:24 smithi142 ceph-mon[106729]: from='client.35691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:24.805 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:24 smithi142 ceph-mon[106729]: from='client.45497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:24.806 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:24 smithi142 ceph-mon[103614]: pgmap v338: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 17323/139605 objects degraded (12.409%); 5.3 KiB/s, 0 objects/s recovering 2024-08-17T17:12:24.806 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:24 smithi142 ceph-mon[103614]: from='client.35691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:24.806 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:24 smithi142 ceph-mon[103614]: from='client.45497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:24 smithi152 ceph-mon[93456]: pgmap v338: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s; 17323/139605 objects degraded (12.409%); 5.3 KiB/s, 0 objects/s recovering 2024-08-17T17:12:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:24 smithi152 ceph-mon[93456]: from='client.35691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:24 smithi152 ceph-mon[93456]: from='client.45497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:25.061 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:24 smithi142 podman[141115]: 2024-08-17 17:12:24.804900455 +0000 UTC m=+0.158074231 container create e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:12:25.061 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:25 smithi142 podman[141115]: 2024-08-17 17:12:25.049791594 +0000 UTC m=+0.402965389 container init e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:12:25.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 54606 54590 142.145 112.062 0.00435651 0.00702987 2024-08-17T17:12:25.372 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:25 smithi142 podman[141115]: 2024-08-17 17:12:25.060269313 +0000 UTC m=+0.413443086 container start e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, RELEASE=main-cd8bdbe, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:12:25.372 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:25 smithi142 bash[141115]: e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 2024-08-17T17:12:25.373 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:25 smithi142 systemd[1]: Started Ceph osd.3 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:12:26.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 56266 56250 140.609 103.75 0.0126412 0.00710682 2024-08-17T17:12:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: pgmap v339: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s; 18663/149430 objects degraded (12.489%); 5.3 KiB/s, 0 objects/s recovering 2024-08-17T17:12:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17323/139605 objects degraded (12.409%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: pgmap v339: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s; 18663/149430 objects degraded (12.489%); 5.3 KiB/s, 0 objects/s recovering 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17323/139605 objects degraded (12.409%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: pgmap v339: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s; 18663/149430 objects degraded (12.489%); 5.3 KiB/s, 0 objects/s recovering 2024-08-17T17:12:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17323/139605 objects degraded (12.409%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:26.849 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:27.122 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:26 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:12:26.684+0000 7fda63984640 -1 Falling back to public interface 2024-08-17T17:12:27.164 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 57904 57888 139.138 102.375 0.0120265 0.00718243 2024-08-17T17:12:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/907710290' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:27 smithi152 ceph-mon[93456]: from='client.45512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:27.583 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/907710290' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:27.583 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:27.583 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:27.584 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[106729]: from='client.45512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:27.584 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/907710290' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:27.584 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:27.584 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:27.584 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:27 smithi142 ceph-mon[103614]: from='client.45512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:28.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 59755 59739 138.26 115.688 0.0103975 0.00722742 2024-08-17T17:12:28.690 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:28 smithi142 ceph-mon[106729]: pgmap v340: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s; 19316/155007 objects degraded (12.461%) 2024-08-17T17:12:28.690 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:28 smithi142 ceph-mon[106729]: from='client.45518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:28.691 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:28 smithi142 ceph-mon[103614]: pgmap v340: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s; 19316/155007 objects degraded (12.461%) 2024-08-17T17:12:28.691 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:28 smithi142 ceph-mon[103614]: from='client.45518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:28 smithi152 ceph-mon[93456]: pgmap v340: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s; 19316/155007 objects degraded (12.461%) 2024-08-17T17:12:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:28 smithi152 ceph-mon[93456]: from='client.45518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:29.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 61592 61576 137.422 114.812 0.0138584 0.00727232 2024-08-17T17:12:30.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 63196 63180 136.14 100.25 0.00972846 0.00734063 2024-08-17T17:12:30.386 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:30 smithi152 ceph-mon[93456]: pgmap v341: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s; 21540/170247 objects degraded (12.652%) 2024-08-17T17:12:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3479939335' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 21540/170247 objects degraded (12.652%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:12:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:12:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:12:30.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:30 smithi142 ceph-mon[106729]: pgmap v341: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s; 21540/170247 objects degraded (12.652%) 2024-08-17T17:12:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3479939335' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 21540/170247 objects degraded (12.652%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:30 smithi142 ceph-mon[103614]: pgmap v341: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s; 21540/170247 objects degraded (12.652%) 2024-08-17T17:12:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3479939335' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:30.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 21540/170247 objects degraded (12.652%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:31.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 64619 64603 134.566 88.9375 0.00617223 0.00742603 2024-08-17T17:12:31.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[103614]: from='client.55363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:31.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:31.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:31.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[103614]: from='client.35724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:31.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[106729]: from='client.55363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:31.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:31.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:31.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-mon[106729]: from='client.35724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:31.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:12:31] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.51.0" 2024-08-17T17:12:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:31 smithi152 ceph-mon[93456]: from='client.55363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:31 smithi152 ceph-mon[93456]: from='client.35724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:32.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 66214 66198 133.439 99.6875 0.00493977 0.00748913 2024-08-17T17:12:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:32 smithi152 ceph-mon[93456]: pgmap v342: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 22628/181617 objects degraded (12.459%) 2024-08-17T17:12:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:32 smithi142 ceph-mon[106729]: pgmap v342: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 22628/181617 objects degraded (12.459%) 2024-08-17T17:12:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:33.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:32 smithi142 ceph-mon[103614]: pgmap v342: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 22628/181617 objects degraded (12.459%) 2024-08-17T17:12:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:33.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 67886 67870 132.535 104.5 0.00634642 0.00754045 2024-08-17T17:12:33.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:33 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2435053134' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:33.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:33 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2435053134' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:34.034 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:34.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:33 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2435053134' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:34.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 69279 69263 131.157 87.0625 0.00844325 0.00761924 2024-08-17T17:12:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:34 smithi152 ceph-mon[93456]: pgmap v343: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 24185/194061 objects degraded (12.463%) 2024-08-17T17:12:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:34 smithi152 ceph-mon[93456]: from='client.45542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:34 smithi142 ceph-mon[106729]: pgmap v343: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 24185/194061 objects degraded (12.463%) 2024-08-17T17:12:35.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:34 smithi142 ceph-mon[106729]: from='client.45542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:34 smithi142 ceph-mon[103614]: pgmap v343: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 24185/194061 objects degraded (12.463%) 2024-08-17T17:12:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:34 smithi142 ceph-mon[103614]: from='client.45542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:35.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 70717 70701 129.943 89.875 0.0159595 0.00769106 2024-08-17T17:12:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:35 smithi142 ceph-mon[103614]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 24185/194061 objects degraded (12.463%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:35.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:35 smithi142 ceph-mon[106729]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:35.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 24185/194061 objects degraded (12.463%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:35 smithi152 ceph-mon[93456]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 24185/194061 objects degraded (12.463%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:36.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 72310 72294 129.074 99.5625 0.0107484 0.00774302 2024-08-17T17:12:36.913 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: pgmap v344: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 25134/201081 objects degraded (12.499%) 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:36.914 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:12:36.656+0000 7f25a1585640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: pgmap v344: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 25134/201081 objects degraded (12.499%) 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:36.915 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:36.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:36.916 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: pgmap v344: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 25134/201081 objects degraded (12.499%) 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:37.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:37.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:37.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-08-17T17:12:37.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 73737 73721 127.966 89.1875 0.00995216 0.00780976 2024-08-17T17:12:37.372 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:36 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:12:36.911+0000 7fda63984640 -1 osd.3 142 log_to_monitors true 2024-08-17T17:12:37.624 INFO:teuthology.orchestra.run.smithi142.stdout:true 2024-08-17T17:12:37.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:37 smithi152 ceph-mon[93456]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-08-17T17:12:37.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:37 smithi152 ceph-mon[93456]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-08-17T17:12:37.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:37 smithi152 ceph-mon[93456]: from='osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:12:37.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:37 smithi152 ceph-mon[93456]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:12:37.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2338733149' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[106729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[106729]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[106729]: from='osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[106729]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2338733149' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[103614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[103614]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[103614]: from='osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[103614]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:12:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2338733149' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:38.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 74811 74795 126.322 67.125 0.124651 0.00789957 2024-08-17T17:12:38.622 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:12:38.314+0000 7fda57726640 -1 osd.3 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: pgmap v345: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 25604/205320 objects degraded (12.470%) 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: from='client.35748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: osdmap e145: 8 total, 7 up, 8 in 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: from='osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:12:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: from='client.45563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:39.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:38 smithi152 ceph-mon[93456]: from='osd.3 ' entity='osd.3' 2024-08-17T17:12:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: pgmap v345: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 25604/205320 objects degraded (12.470%) 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: from='client.35748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: osdmap e145: 8 total, 7 up, 8 in 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: from='osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: from='client.45563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[103614]: from='osd.3 ' entity='osd.3' 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: pgmap v345: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 25604/205320 objects degraded (12.470%) 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: from='client.35748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:39.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:12:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: osdmap e145: 8 total, 7 up, 8 in 2024-08-17T17:12:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: from='osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:12:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi142", "root=default"]} : dispatch 2024-08-17T17:12:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: from='client.45563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:39.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:38 smithi142 ceph-mon[106729]: from='osd.3 ' entity='osd.3' 2024-08-17T17:12:39.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 76420 76404 125.644 100.562 0.00615046 0.00795413 2024-08-17T17:12:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:39 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:12:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:39 smithi152 ceph-mon[93456]: osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216] boot 2024-08-17T17:12:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:39 smithi152 ceph-mon[93456]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T17:12:40.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:39 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:12:40.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[103614]: osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216] boot 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[103614]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[106729]: osd.3 [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216] boot 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[106729]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T17:12:40.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:39 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:12:40.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 76744 76728 122.941 20.25 0.00271445 0.00796083 2024-08-17T17:12:40.301 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:12:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:12:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:12:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:12:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[106729]: pgmap v347: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 27504/218190 objects degraded (12.606%) 2024-08-17T17:12:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2951800824' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[106729]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T17:12:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 27504/218190 objects degraded (12.606%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:40.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[103614]: pgmap v347: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 27504/218190 objects degraded (12.606%) 2024-08-17T17:12:40.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2951800824' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:40.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[103614]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T17:12:40.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 27504/218190 objects degraded (12.606%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:41.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:12:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:40 smithi152 ceph-mon[93456]: pgmap v347: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 27504/218190 objects degraded (12.606%) 2024-08-17T17:12:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2951800824' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:12:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:40 smithi152 ceph-mon[93456]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T17:12:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 27504/218190 objects degraded (12.606%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:41.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:12:41.168042+0000 min lat: 0.00102176 max lat: 1.02694 avg lat: 0.00796083 2024-08-17T17:12:41.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:12:41.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 76744 76728 119.868 0 - 0.00796083 2024-08-17T17:12:41.311 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:12:41] "GET /metrics HTTP/1.1" 200 36223 "" "Prometheus/2.51.0" 2024-08-17T17:12:42.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 77250 77234 117.716 15.8125 0.0108726 0.00848901 2024-08-17T17:12:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:42 smithi152 ceph-mon[93456]: pgmap v350: 89 pgs: 1 remapped+peering, 3 peering, 21 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 18544/225792 objects degraded (8.213%) 2024-08-17T17:12:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:42 smithi152 ceph-mon[93456]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T17:12:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:42.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[106729]: pgmap v350: 89 pgs: 1 remapped+peering, 3 peering, 21 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 18544/225792 objects degraded (8.213%) 2024-08-17T17:12:42.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[106729]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T17:12:42.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:42.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:42.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[103614]: pgmap v350: 89 pgs: 1 remapped+peering, 3 peering, 21 active+undersized, 7 active+undersized+degraded, 57 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 18544/225792 objects degraded (8.213%) 2024-08-17T17:12:42.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[103614]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T17:12:42.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:42.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:43.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 78248 78232 116.398 62.375 0.0127858 0.00858394 2024-08-17T17:12:43.716 INFO:teuthology.orchestra.run.smithi142.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:44.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 79095 79079 114.922 52.9375 0.0106752 0.00869473 2024-08-17T17:12:44.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:12:44.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[103614]: pgmap v352: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 peering, 2 active+undersized, 2 active+undersized+degraded, 79 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 15615/230349 objects degraded (6.779%); 23 KiB/s, 0 objects/s recovering 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[103614]: from='client.55402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[103614]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[106729]: pgmap v352: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 peering, 2 active+undersized, 2 active+undersized+degraded, 79 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 15615/230349 objects degraded (6.779%); 23 KiB/s, 0 objects/s recovering 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[106729]: from='client.55402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[106729]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:44.876 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:45.044 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:44 smithi152 ceph-mon[93456]: pgmap v352: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 3 peering, 2 active+undersized, 2 active+undersized+degraded, 79 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 15615/230349 objects degraded (6.779%); 23 KiB/s, 0 objects/s recovering 2024-08-17T17:12:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:44 smithi152 ceph-mon[93456]: from='client.55402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:44 smithi152 ceph-mon[93456]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:12:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:45.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 79933 79917 113.5 52.375 0.00915812 0.00880523 2024-08-17T17:12:45.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15615/230349 objects degraded (6.779%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:45.991 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15615/230349 objects degraded (6.779%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:46.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15615/230349 objects degraded (6.779%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:46.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 80879 80863 112.292 59.125 0.0186099 0.00890015 2024-08-17T17:12:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:46 smithi152 ceph-mon[93456]: pgmap v353: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 15923/234201 objects degraded (6.799%); 746 KiB/s, 11 objects/s recovering 2024-08-17T17:12:47.109 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:46 smithi142 ceph-mon[103614]: pgmap v353: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 15923/234201 objects degraded (6.799%); 746 KiB/s, 11 objects/s recovering 2024-08-17T17:12:47.110 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:46 smithi142 ceph-mon[106729]: pgmap v353: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 15923/234201 objects degraded (6.799%); 746 KiB/s, 11 objects/s recovering 2024-08-17T17:12:47.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 81892 81876 111.227 63.3125 0.0202905 0.00898513 2024-08-17T17:12:48.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 82864 82848 110.153 60.75 0.0217184 0.00907244 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:alertmanager.a smithi142 *:9093,9094 running (11m) 17s ago 16m 20.5M - 0.27.0 11f11916f8cd 95562b1fc088 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:grafana.a smithi152 *:3000 running (11m) 49s ago 16m 88.2M - 9.4.7 954c08fa6188 d463717746ae 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi142.ndmoid smithi142 running (15m) 17s ago 15m 19.6M - 17.2.7-1404-gf2c39d4d f69471c56189 bd857f554782 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi152.tacamo smithi152 running (15m) 49s ago 15m 17.6M - 17.2.7-1404-gf2c39d4d f69471c56189 8913efd6a27d 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.x smithi152 *:8443,9283,8765 running (10m) 49s ago 20m 457M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb bc654892e833 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.y smithi142 *:8443,9283,8765 running (10m) 17s ago 21m 522M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 323039997e22 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a smithi142 running (9m) 17s ago 21m 62.0M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 3a2a93ab0fbb 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b smithi152 running (7m) 49s ago 20m 46.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb a690c34b9332 2024-08-17T17:12:48.200 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c smithi142 running (9m) 17s ago 20m 52.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb e6fccaa16a72 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.a smithi142 *:9100 running (11m) 17s ago 15m 11.7M - 1.7.0 72c9c2088986 c6cd5b4c28e1 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.b smithi152 *:9100 running (11m) 49s ago 15m 10.9M - 1.7.0 72c9c2088986 8e3769ed942e 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 smithi142 running (5m) 17s ago 19m 734M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 72a9d7f0d685 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 smithi142 running (5m) 17s ago 19m 283M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb d435dd533791 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 smithi142 running (2m) 17s ago 19m 249M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9fd11a07e948 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 smithi142 running (23s) 17s ago 18m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb e93ead24f7cc 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 smithi152 running (18m) 49s ago 18m 560M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 7a1cab3a1049 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 smithi152 running (17m) 49s ago 17m 962M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 cac09b3f3a95 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 smithi152 running (17m) 49s ago 17m 600M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 81327b9b9e78 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 smithi152 running (17m) 49s ago 17m 510M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 0a45f0507de1 2024-08-17T17:12:48.201 INFO:teuthology.orchestra.run.smithi142.stdout:prometheus.a smithi152 *:9095 running (10m) 49s ago 16m 56.0M - 2.51.0 1d3b7f56885b f12943011a9f 2024-08-17T17:12:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:48 smithi142 ceph-mon[103614]: pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 19099/236484 objects degraded (8.076%); 563 KiB/s, 8 objects/s recovering 2024-08-17T17:12:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:48 smithi142 ceph-mon[106729]: pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 19099/236484 objects degraded (8.076%); 563 KiB/s, 8 objects/s recovering 2024-08-17T17:12:49.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:12:49.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:48 smithi152 ceph-mon[93456]: pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 19099/236484 objects degraded (8.076%); 563 KiB/s, 8 objects/s recovering 2024-08-17T17:12:49.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 83872 83856 109.17 63 0.0194212 0.00915476 2024-08-17T17:12:49.511 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:49.998 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:49 smithi142 ceph-mon[103614]: from='client.45575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:49.999 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:49 smithi142 ceph-mon[106729]: from='client.45575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:49 smithi152 ceph-mon[93456]: from='client.45575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:50.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 84804 84788 108.131 58.25 0.00494714 0.00924244 2024-08-17T17:12:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:12:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:12:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:12:50.830 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:50 smithi142 ceph-mon[103614]: pgmap v355: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s; 19099/242550 objects degraded (7.874%); 500 KiB/s, 7 objects/s recovering 2024-08-17T17:12:50.831 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 19099/242550 objects degraded (7.874%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:50.831 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:50 smithi142 ceph-mon[106729]: pgmap v355: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s; 19099/242550 objects degraded (7.874%); 500 KiB/s, 7 objects/s recovering 2024-08-17T17:12:50.831 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 19099/242550 objects degraded (7.874%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:50 smithi152 ceph-mon[93456]: pgmap v355: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s; 19099/242550 objects degraded (7.874%); 500 KiB/s, 7 objects/s recovering 2024-08-17T17:12:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 19099/242550 objects degraded (7.874%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:51.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 85790 85774 107.201 61.625 0.0118279 0.00932317 2024-08-17T17:12:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:52.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:52.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:52.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:12:51] "GET /metrics HTTP/1.1" 200 36223 "" "Prometheus/2.51.0" 2024-08-17T17:12:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:52.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:52.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:52.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:52.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 86726 86710 106.246 58.5 0.00334954 0.00940265 2024-08-17T17:12:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:52 smithi152 ceph-mon[93456]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 18969/250680 objects degraded (7.567%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:12:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:53.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[106729]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 18969/250680 objects degraded (7.567%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[103614]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 18969/250680 objects degraded (7.567%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:53.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:53.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:12:53.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 87653 87637 105.317 57.9375 0.00996483 0.00948968 2024-08-17T17:12:54.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 88465 88449 104.287 50.75 0.0194238 0.00958213 2024-08-17T17:12:54.296 INFO:teuthology.orchestra.run.smithi142.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:alertmanager.a smithi142 *:9093,9094 running (11m) 23s ago 16m 20.5M - 0.27.0 11f11916f8cd 95562b1fc088 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:grafana.a smithi152 *:3000 running (11m) 55s ago 16m 88.2M - 9.4.7 954c08fa6188 d463717746ae 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi142.ndmoid smithi142 running (15m) 23s ago 15m 19.6M - 17.2.7-1404-gf2c39d4d f69471c56189 bd857f554782 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mds.foo.smithi152.tacamo smithi152 running (15m) 55s ago 15m 17.6M - 17.2.7-1404-gf2c39d4d f69471c56189 8913efd6a27d 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.x smithi152 *:8443,9283,8765 running (10m) 55s ago 20m 457M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb bc654892e833 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mgr.y smithi142 *:8443,9283,8765 running (10m) 23s ago 21m 522M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 323039997e22 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a smithi142 running (9m) 23s ago 22m 62.0M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 3a2a93ab0fbb 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b smithi152 running (7m) 55s ago 20m 46.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb a690c34b9332 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c smithi142 running (9m) 23s ago 20m 52.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb e6fccaa16a72 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.a smithi142 *:9100 running (11m) 23s ago 15m 11.7M - 1.7.0 72c9c2088986 c6cd5b4c28e1 2024-08-17T17:12:54.297 INFO:teuthology.orchestra.run.smithi142.stdout:node-exporter.b smithi152 *:9100 running (11m) 55s ago 15m 10.9M - 1.7.0 72c9c2088986 8e3769ed942e 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 smithi142 running (5m) 23s ago 20m 734M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 72a9d7f0d685 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 smithi142 running (5m) 23s ago 19m 283M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb d435dd533791 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 smithi142 running (2m) 23s ago 19m 249M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 9fd11a07e948 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 smithi142 running (29s) 23s ago 18m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb e93ead24f7cc 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 smithi152 running (18m) 55s ago 18m 560M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 7a1cab3a1049 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 smithi152 running (18m) 55s ago 18m 962M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 cac09b3f3a95 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 smithi152 running (17m) 55s ago 17m 600M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 81327b9b9e78 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 smithi152 running (17m) 55s ago 17m 510M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 0a45f0507de1 2024-08-17T17:12:54.298 INFO:teuthology.orchestra.run.smithi142.stdout:prometheus.a smithi152 *:9095 running (10m) 55s ago 16m 56.0M - 2.51.0 1d3b7f56885b f12943011a9f 2024-08-17T17:12:55.042 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:54 smithi142 ceph-mon[103614]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 18969/257073 objects degraded (7.379%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:12:55.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:54 smithi142 ceph-mon[103614]: from='client.35769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:55.043 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:54 smithi142 ceph-mon[106729]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 18969/257073 objects degraded (7.379%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:12:55.043 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:54 smithi142 ceph-mon[106729]: from='client.35769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:55.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:54 smithi152 ceph-mon[93456]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 18969/257073 objects degraded (7.379%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:12:55.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:54 smithi152 ceph-mon[93456]: from='client.35769 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:12:55.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph versions' 2024-08-17T17:12:55.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 89397 89381 103.432 58.25 0.042801 0.00966237 2024-08-17T17:12:55.588 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18969/257073 objects degraded (7.379%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:56.096 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18969/257073 objects degraded (7.379%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:56.097 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18969/257073 objects degraded (7.379%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:12:56.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 90283 90267 102.559 55.375 0.00403371 0.00974463 2024-08-17T17:12:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:56 smithi142 ceph-mon[106729]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18839/262389 objects degraded (7.180%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:12:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:56 smithi142 ceph-mon[103614]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18839/262389 objects degraded (7.180%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:12:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:56 smithi152 ceph-mon[93456]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 18839/262389 objects degraded (7.180%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:12:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:57.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 91259 91243 101.816 61 0.0098831 0.00981614 2024-08-17T17:12:58.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 92161 92145 101.019 56.375 0.00452622 0.00989296 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "mon": { 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr": { 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "osd": { 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)": 4, 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "mds": { 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)": 2 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi142.stdout: "overall": { 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)": 6, 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi142.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 9 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:12:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:58 smithi142 ceph-mon[103614]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 18839/264771 objects degraded (7.115%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:12:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:12:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/140460561' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:58 smithi142 ceph-mon[106729]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 18839/264771 objects degraded (7.115%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:12:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:12:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/140460561' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:58 smithi152 ceph-mon[93456]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 18839/264771 objects degraded (7.115%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:12:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:12:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/140460561' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:12:59.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 93051 93035 100.236 55.625 0.0124218 0.00997055 2024-08-17T17:12:59.378 DEBUG:teuthology.parallel:result is None 2024-08-17T17:13:00.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 94035 94019 99.5798 61.5 0.0216971 0.0100358 2024-08-17T17:13:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:13:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:13:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:13:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:00 smithi152 ceph-mon[93456]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 18839/270765 objects degraded (6.958%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:13:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18839/270765 objects degraded (6.958%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:00 smithi142 ceph-mon[103614]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 18839/270765 objects degraded (6.958%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:13:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18839/270765 objects degraded (6.958%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:00 smithi142 ceph-mon[106729]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 18839/270765 objects degraded (6.958%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:13:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18839/270765 objects degraded (6.958%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:01.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:13:01.171479+0000 min lat: 0.00102176 max lat: 2.38617 avg lat: 0.0101009 2024-08-17T17:13:01.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:13:01.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 15 95026 95011 98.9534 62 0.011569 0.0101009 2024-08-17T17:13:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:13:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:13:01] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.51.0" 2024-08-17T17:13:02.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 95972 95956 98.2994 59.0625 0.0105887 0.0101676 2024-08-17T17:13:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:02 smithi152 ceph-mon[93456]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 18706/278643 objects degraded (6.713%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:13:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:02 smithi142 ceph-mon[103614]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 18706/278643 objects degraded (6.713%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:13:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:02 smithi142 ceph-mon[106729]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 18706/278643 objects degraded (6.713%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:13:03.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 96950 96934 97.6998 61.125 0.0119922 0.0102301 2024-08-17T17:13:04.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 97871 97855 97.0626 57.5625 0.0187329 0.0102969 2024-08-17T17:13:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:04 smithi152 ceph-mon[93456]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 18706/284847 objects degraded (6.567%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:13:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:04 smithi142 ceph-mon[106729]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 18706/284847 objects degraded (6.567%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:13:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:04 smithi142 ceph-mon[103614]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 18706/284847 objects degraded (6.567%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:13:05.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 98832 98816 96.4845 60.0625 0.0300297 0.010359 2024-08-17T17:13:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18706/284847 objects degraded (6.567%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:06.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18706/284847 objects degraded (6.567%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:06.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18706/284847 objects degraded (6.567%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:06.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 99790 99774 95.9212 59.875 0.011447 0.0104194 2024-08-17T17:13:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:06 smithi152 ceph-mon[93456]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 18568/290307 objects degraded (6.396%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:07.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:06 smithi142 ceph-mon[106729]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 18568/290307 objects degraded (6.396%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:06 smithi142 ceph-mon[103614]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 18568/290307 objects degraded (6.396%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:07.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 100731 100715 95.3589 58.8125 0.00550676 0.0104803 2024-08-17T17:13:08.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 101660 101644 94.8022 58.0625 0.0228542 0.010543 2024-08-17T17:13:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:08 smithi152 ceph-mon[93456]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 18568/292980 objects degraded (6.338%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:13:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:08 smithi142 ceph-mon[106729]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 18568/292980 objects degraded (6.338%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:13:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:08 smithi142 ceph-mon[103614]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 18568/292980 objects degraded (6.338%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:13:09.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 102569 102553 94.2435 56.8125 0.0395788 0.0106057 2024-08-17T17:13:10.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 103482 103466 93.7046 57.0625 0.0121299 0.0106658 2024-08-17T17:13:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:13:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:13:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:13:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:10 smithi152 ceph-mon[93456]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 18568/299238 objects degraded (6.205%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:13:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18568/299238 objects degraded (6.205%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:11.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:10 smithi142 ceph-mon[106729]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 18568/299238 objects degraded (6.205%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:13:11.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18568/299238 objects degraded (6.205%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:10 smithi142 ceph-mon[103614]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 18568/299238 objects degraded (6.205%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:13:11.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18568/299238 objects degraded (6.205%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:11.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 104390 104374 93.1766 56.75 0.0104868 0.0107264 2024-08-17T17:13:12.023 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:12.024 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:13:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:13:11] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-08-17T17:13:12.024 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:12.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 105220 105204 92.5949 51.875 0.0279783 0.0107943 2024-08-17T17:13:12.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:13.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:12 smithi142 ceph-mon[103614]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18419/307155 objects degraded (5.997%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:13:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:12 smithi142 ceph-mon[106729]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18419/307155 objects degraded (5.997%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:13:13.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 15 105986 105971 91.9746 47.9375 0.00473002 0.0108667 2024-08-17T17:13:13.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:12 smithi152 ceph-mon[93456]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18419/307155 objects degraded (5.997%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:13:14.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 106768 106752 91.3833 48.8125 0.0095678 0.0109362 2024-08-17T17:13:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:14 smithi142 ceph-mon[103614]: pgmap v367: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 18419/312804 objects degraded (5.888%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:14 smithi142 ceph-mon[106729]: pgmap v367: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 18419/312804 objects degraded (5.888%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:15.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 107647 107631 90.8908 54.9375 0.00650317 0.0109949 2024-08-17T17:13:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:14 smithi152 ceph-mon[93456]: pgmap v367: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 18419/312804 objects degraded (5.888%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18419/312804 objects degraded (5.888%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:16.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18419/312804 objects degraded (5.888%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:16.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 108549 108533 90.4305 56.375 0.0114882 0.0110517 2024-08-17T17:13:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18419/312804 objects degraded (5.888%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:16 smithi142 ceph-mon[106729]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 18289/317748 objects degraded (5.756%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:13:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:16 smithi142 ceph-mon[103614]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 18289/317748 objects degraded (5.756%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:13:17.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 109408 109392 89.947 53.6875 0.0201645 0.0111112 2024-08-17T17:13:17.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:16 smithi152 ceph-mon[93456]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 18289/317748 objects degraded (5.756%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:13:18.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 110238 110222 89.4525 51.875 0.0125468 0.011173 2024-08-17T17:13:19.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:18 smithi142 ceph-mon[103614]: pgmap v369: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 18289/320019 objects degraded (5.715%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:19.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:18 smithi142 ceph-mon[106729]: pgmap v369: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 18289/320019 objects degraded (5.715%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:19.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 111134 111118 89.0236 56 0.0152624 0.0112269 2024-08-17T17:13:19.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:18 smithi152 ceph-mon[93456]: pgmap v369: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 18289/320019 objects degraded (5.715%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:20.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 111954 111938 88.5454 51.25 0.0128254 0.011288 2024-08-17T17:13:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:13:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:13:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:13:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:20 smithi142 ceph-mon[106729]: pgmap v370: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 18289/325563 objects degraded (5.618%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18289/325563 objects degraded (5.618%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:20 smithi142 ceph-mon[103614]: pgmap v370: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 18289/325563 objects degraded (5.618%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18289/325563 objects degraded (5.618%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:21.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:13:21.173337+0000 min lat: 0.00102176 max lat: 2.38617 avg lat: 0.0113441 2024-08-17T17:13:21.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:13:21.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 112802 112786 88.1011 53 0.0514189 0.0113441 2024-08-17T17:13:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:20 smithi152 ceph-mon[93456]: pgmap v370: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 18289/325563 objects degraded (5.618%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:13:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18289/325563 objects degraded (5.618%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:13:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:13:21] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-08-17T17:13:22.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 113670 113654 87.6831 54.25 0.0186581 0.0113992 2024-08-17T17:13:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:22 smithi142 ceph-mon[106729]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 18123/332847 objects degraded (5.445%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:13:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:22 smithi142 ceph-mon[103614]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 18123/332847 objects degraded (5.445%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:13:23.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 114512 114496 87.2555 52.625 0.00963646 0.0114552 2024-08-17T17:13:23.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:22 smithi152 ceph-mon[93456]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s; 18123/332847 objects degraded (5.445%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:13:24.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 115369 115353 86.8496 53.5625 0.0122292 0.0115086 2024-08-17T17:13:25.124 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:24 smithi152 ceph-mon[93456]: pgmap v372: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s; 18123/338511 objects degraded (5.354%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:13:25.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 116199 116183 86.4332 51.875 0.0279002 0.0115638 2024-08-17T17:13:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:24 smithi142 ceph-mon[103614]: pgmap v372: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s; 18123/338511 objects degraded (5.354%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:13:25.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:24 smithi142 ceph-mon[106729]: pgmap v372: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s; 18123/338511 objects degraded (5.354%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:13:26.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 117073 117057 86.0589 54.625 0.0295255 0.0116141 2024-08-17T17:13:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18123/338511 objects degraded (5.354%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:26.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18123/338511 objects degraded (5.354%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:26.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18123/338511 objects degraded (5.354%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:27.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 117946 117930 85.6927 54.5625 0.00491131 0.0116633 2024-08-17T17:13:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:26 smithi152 ceph-mon[93456]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 17966/343278 objects degraded (5.234%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:13:27.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:27.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:26 smithi142 ceph-mon[103614]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 17966/343278 objects degraded (5.234%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:13:27.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:27.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:26 smithi142 ceph-mon[106729]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 17966/343278 objects degraded (5.234%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:13:27.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:28.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 118792 118776 85.3154 52.875 0.0347457 0.0117155 2024-08-17T17:13:29.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 119656 119640 84.9595 54 0.009181 0.0117639 2024-08-17T17:13:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:28 smithi152 ceph-mon[93456]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s; 17966/345819 objects degraded (5.195%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:13:29.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:28 smithi142 ceph-mon[106729]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s; 17966/345819 objects degraded (5.195%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:13:29.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:28 smithi142 ceph-mon[103614]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s; 17966/345819 objects degraded (5.195%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:13:30.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 120483 120467 84.5856 51.6875 0.00650222 0.0118147 2024-08-17T17:13:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:13:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:13:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:13:31.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 90 3 121293 121290 84.2173 51.4375 0.020383 0.0118688 2024-08-17T17:13:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:30 smithi152 ceph-mon[93456]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 17966/351546 objects degraded (5.111%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:13:31.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17966/351546 objects degraded (5.111%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:31.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:30 smithi142 ceph-mon[106729]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 17966/351546 objects degraded (5.111%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:13:31.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17966/351546 objects degraded (5.111%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:30 smithi142 ceph-mon[103614]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 17966/351546 objects degraded (5.111%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:13:31.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17966/351546 objects degraded (5.111%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:13:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:13:31] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.51.0" 2024-08-17T17:13:32.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0156 2024-08-17T17:13:32.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 121293 2024-08-17T17:13:32.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:13:32.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 84.2167 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 45.034 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 212.312 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1347 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 720.544 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 3397 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 0 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.011869 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0310856 2024-08-17T17:13:32.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 2.38617 2024-08-17T17:13:32.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00102176 2024-08-17T17:13:32.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:13:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:32 smithi152 ceph-mon[93456]: pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 17788/358461 objects degraded (4.962%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:13:33.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:32 smithi142 ceph-mon[106729]: pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 17788/358461 objects degraded (4.962%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:13:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:32 smithi142 ceph-mon[103614]: pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 17788/358461 objects degraded (4.962%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:13:35.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:34 smithi152 ceph-mon[93456]: pgmap v377: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 834 op/s; 17788/362613 objects degraded (4.906%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:13:35.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:34 smithi142 ceph-mon[106729]: pgmap v377: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 834 op/s; 17788/362613 objects degraded (4.906%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:13:35.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:34 smithi142 ceph-mon[103614]: pgmap v377: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 834 op/s; 17788/362613 objects degraded (4.906%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:13:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17788/362613 objects degraded (4.906%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:36.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17788/362613 objects degraded (4.906%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:36.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17788/362613 objects degraded (4.906%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:36 smithi152 ceph-mon[93456]: pgmap v378: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 786 op/s; 17444/360870 objects degraded (4.834%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:13:37.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:36 smithi142 ceph-mon[106729]: pgmap v378: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 786 op/s; 17444/360870 objects degraded (4.834%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:13:37.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:36 smithi142 ceph-mon[103614]: pgmap v378: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 786 op/s; 17444/360870 objects degraded (4.834%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:13:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:38 smithi152 ceph-mon[93456]: pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 710 op/s; 17444/359076 objects degraded (4.858%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:13:39.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:38 smithi142 ceph-mon[103614]: pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 710 op/s; 17444/359076 objects degraded (4.858%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:13:39.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:38 smithi142 ceph-mon[106729]: pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 710 op/s; 17444/359076 objects degraded (4.858%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:13:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:13:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:13:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:13:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:40 smithi152 ceph-mon[93456]: pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 796 op/s; 17444/353421 objects degraded (4.936%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:13:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17444/353421 objects degraded (4.936%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:41.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:40 smithi142 ceph-mon[106729]: pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 796 op/s; 17444/353421 objects degraded (4.936%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:13:41.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17444/353421 objects degraded (4.936%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:41.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:40 smithi142 ceph-mon[103614]: pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 796 op/s; 17444/353421 objects degraded (4.936%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:13:41.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17444/353421 objects degraded (4.936%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:13:42.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:42.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:13:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:13:41] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:13:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:43.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:42 smithi142 ceph-mon[106729]: pgmap v381: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 839 op/s; 17255/346152 objects degraded (4.985%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:13:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:42 smithi142 ceph-mon[103614]: pgmap v381: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 839 op/s; 17255/346152 objects degraded (4.985%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:13:43.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:42 smithi152 ceph-mon[93456]: pgmap v381: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 839 op/s; 17255/346152 objects degraded (4.985%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:13:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:44 smithi152 ceph-mon[93456]: pgmap v382: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.5 MiB/s wr, 804 op/s; 17255/340479 objects degraded (5.068%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:13:45.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:44 smithi142 ceph-mon[106729]: pgmap v382: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.5 MiB/s wr, 804 op/s; 17255/340479 objects degraded (5.068%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:13:45.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:44 smithi142 ceph-mon[103614]: pgmap v382: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.5 MiB/s wr, 804 op/s; 17255/340479 objects degraded (5.068%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:13:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17255/340479 objects degraded (5.068%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:46.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17255/340479 objects degraded (5.068%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:46.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17255/340479 objects degraded (5.068%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:46 smithi152 ceph-mon[93456]: pgmap v383: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 2.1 MiB/s wr, 806 op/s; 17042/335985 objects degraded (5.072%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:13:47.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:46 smithi142 ceph-mon[106729]: pgmap v383: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 2.1 MiB/s wr, 806 op/s; 17042/335985 objects degraded (5.072%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:13:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:46 smithi142 ceph-mon[103614]: pgmap v383: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 2.1 MiB/s wr, 806 op/s; 17042/335985 objects degraded (5.072%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:13:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:48 smithi152 ceph-mon[93456]: pgmap v384: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 197 KiB/s wr, 756 op/s; 17042/333825 objects degraded (5.105%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:49.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:48 smithi142 ceph-mon[106729]: pgmap v384: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 197 KiB/s wr, 756 op/s; 17042/333825 objects degraded (5.105%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:48 smithi142 ceph-mon[103614]: pgmap v384: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 197 KiB/s wr, 756 op/s; 17042/333825 objects degraded (5.105%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:13:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:13:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:13:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:50 smithi152 ceph-mon[93456]: pgmap v385: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 17042/328557 objects degraded (5.187%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 17042/328557 objects degraded (5.187%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:50 smithi142 ceph-mon[106729]: pgmap v385: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 17042/328557 objects degraded (5.187%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 17042/328557 objects degraded (5.187%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:50 smithi142 ceph-mon[103614]: pgmap v385: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 17042/328557 objects degraded (5.187%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:13:51.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 17042/328557 objects degraded (5.187%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:13:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:13:51] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:13:53.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:52 smithi152 ceph-mon[93456]: pgmap v386: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 16799/321558 objects degraded (5.224%); 3.4 MiB/s, 53 objects/s recovering 2024-08-17T17:13:53.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:53.291 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:52 smithi142 ceph-mon[103614]: pgmap v386: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 16799/321558 objects degraded (5.224%); 3.4 MiB/s, 53 objects/s recovering 2024-08-17T17:13:53.291 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:53.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:52 smithi142 ceph-mon[106729]: pgmap v386: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 16799/321558 objects degraded (5.224%); 3.4 MiB/s, 53 objects/s recovering 2024-08-17T17:13:53.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:55.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:55 smithi152 ceph-mon[93456]: pgmap v387: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 16799/315954 objects degraded (5.317%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:13:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:55 smithi142 ceph-mon[103614]: pgmap v387: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 16799/315954 objects degraded (5.317%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:13:55.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:55 smithi142 ceph-mon[106729]: pgmap v387: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 16799/315954 objects degraded (5.317%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:13:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16799/315954 objects degraded (5.317%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:56.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16799/315954 objects degraded (5.317%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:56.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16799/315954 objects degraded (5.317%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:13:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:57 smithi142 ceph-mon[106729]: pgmap v388: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 16540/311517 objects degraded (5.310%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:13:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:57 smithi142 ceph-mon[103614]: pgmap v388: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 16540/311517 objects degraded (5.310%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:13:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:57 smithi152 ceph-mon[93456]: pgmap v388: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 16540/311517 objects degraded (5.310%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:13:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:59 smithi152 ceph-mon[93456]: pgmap v389: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 16540/308619 objects degraded (5.359%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:13:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:13:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[103614]: pgmap v389: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 16540/308619 objects degraded (5.359%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[106729]: pgmap v389: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 16540/308619 objects degraded (5.359%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:13:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:13:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:14:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:00.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:14:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:14:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:14:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:14:00.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:00.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:00.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:14:00.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:00.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:00.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:14:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:01 smithi152 ceph-mon[93456]: pgmap v390: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 16540/304170 objects degraded (5.438%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16540/304170 objects degraded (5.438%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:01.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:01 smithi142 ceph-mon[106729]: pgmap v390: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 16540/304170 objects degraded (5.438%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:01.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16540/304170 objects degraded (5.438%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:01 smithi142 ceph-mon[103614]: pgmap v390: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 16540/304170 objects degraded (5.438%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16540/304170 objects degraded (5.438%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:14:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:14:01] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:14:03.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:03 smithi152 ceph-mon[93456]: pgmap v391: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 16258/297588 objects degraded (5.463%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:14:03.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:03 smithi142 ceph-mon[106729]: pgmap v391: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 16258/297588 objects degraded (5.463%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:14:03.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:03 smithi142 ceph-mon[103614]: pgmap v391: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 16258/297588 objects degraded (5.463%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:14:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:05 smithi152 ceph-mon[93456]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 16258/292386 objects degraded (5.560%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:14:05.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:05 smithi142 ceph-mon[106729]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 16258/292386 objects degraded (5.560%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:14:05.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:05 smithi142 ceph-mon[103614]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 16258/292386 objects degraded (5.560%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:14:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:06 smithi152 ceph-mon[93456]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T17:14:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16258/292386 objects degraded (5.560%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:06 smithi142 ceph-mon[106729]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T17:14:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16258/292386 objects degraded (5.560%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:06 smithi142 ceph-mon[103614]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T17:14:06.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16258/292386 objects degraded (5.560%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:07 smithi152 ceph-mon[93456]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 16258/288678 objects degraded (5.632%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:14:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:07 smithi152 ceph-mon[93456]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T17:14:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:07 smithi142 ceph-mon[106729]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 16258/288678 objects degraded (5.632%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:14:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:07 smithi142 ceph-mon[106729]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T17:14:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:07 smithi142 ceph-mon[103614]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 16258/288678 objects degraded (5.632%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:14:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:07 smithi142 ceph-mon[103614]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T17:14:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:09 smithi152 ceph-mon[93456]: pgmap v396: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 16154/286740 objects degraded (5.634%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:14:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:09 smithi142 ceph-mon[106729]: pgmap v396: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 16154/286740 objects degraded (5.634%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:14:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:09 smithi142 ceph-mon[103614]: pgmap v396: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 16154/286740 objects degraded (5.634%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:14:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:14:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:14:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:14:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:11 smithi152 ceph-mon[93456]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 15871/281130 objects degraded (5.645%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:14:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15871/281130 objects degraded (5.645%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:11 smithi142 ceph-mon[106729]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 15871/281130 objects degraded (5.645%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:14:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15871/281130 objects degraded (5.645%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:11.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:11 smithi142 ceph-mon[103614]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 15871/281130 objects degraded (5.645%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:14:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15871/281130 objects degraded (5.645%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:12.079 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:14:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:14:11] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:14:12.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:12.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:12.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:13.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:13 smithi142 ceph-mon[106729]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 15732/275556 objects degraded (5.709%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:14:13.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:13 smithi142 ceph-mon[103614]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 15732/275556 objects degraded (5.709%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:14:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:13 smithi152 ceph-mon[93456]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 15732/275556 objects degraded (5.709%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:14:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:14 smithi152 ceph-mon[93456]: pgmap v399: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 15732/269595 objects degraded (5.835%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:14:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:14 smithi142 ceph-mon[106729]: pgmap v399: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 15732/269595 objects degraded (5.835%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:14:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:14 smithi142 ceph-mon[103614]: pgmap v399: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 15732/269595 objects degraded (5.835%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:14:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15732/269595 objects degraded (5.835%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15732/269595 objects degraded (5.835%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:15.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15732/269595 objects degraded (5.835%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:16 smithi152 ceph-mon[93456]: pgmap v400: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 15732/265788 objects degraded (5.919%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:14:16.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:16 smithi142 ceph-mon[106729]: pgmap v400: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 15732/265788 objects degraded (5.919%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:14:16.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:16 smithi142 ceph-mon[103614]: pgmap v400: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 15732/265788 objects degraded (5.919%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:14:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:18 smithi152 ceph-mon[93456]: pgmap v401: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 15592/263511 objects degraded (5.917%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:14:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:18 smithi142 ceph-mon[106729]: pgmap v401: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 15592/263511 objects degraded (5.917%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:14:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:18 smithi142 ceph-mon[103614]: pgmap v401: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 15592/263511 objects degraded (5.917%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T17:14:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:20 smithi152 ceph-mon[93456]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 15592/257571 objects degraded (6.053%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:14:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15592/257571 objects degraded (6.053%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:20.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:14:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:14:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:14:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:20 smithi142 ceph-mon[103614]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 15592/257571 objects degraded (6.053%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:14:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15592/257571 objects degraded (6.053%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:20.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:20 smithi142 ceph-mon[106729]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 15592/257571 objects degraded (6.053%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:14:20.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15592/257571 objects degraded (6.053%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:14:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:14:21] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:14:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:22 smithi152 ceph-mon[93456]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 15444/251655 objects degraded (6.137%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:14:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:22 smithi142 ceph-mon[106729]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 15444/251655 objects degraded (6.137%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:14:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:22 smithi142 ceph-mon[103614]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 15444/251655 objects degraded (6.137%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:14:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:24 smithi152 ceph-mon[93456]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 15444/245697 objects degraded (6.286%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:14:24.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:24 smithi142 ceph-mon[103614]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 15444/245697 objects degraded (6.286%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:14:24.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:24 smithi142 ceph-mon[106729]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 15444/245697 objects degraded (6.286%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:14:25.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15444/245697 objects degraded (6.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15444/245697 objects degraded (6.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15444/245697 objects degraded (6.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:26 smithi152 ceph-mon[93456]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 15444/241887 objects degraded (6.385%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:14:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:26 smithi142 ceph-mon[103614]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 15444/241887 objects degraded (6.385%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:14:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:26 smithi142 ceph-mon[106729]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 15444/241887 objects degraded (6.385%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:14:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:28 smithi152 ceph-mon[93456]: pgmap v406: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 15292/239655 objects degraded (6.381%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:14:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:28 smithi142 ceph-mon[103614]: pgmap v406: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 15292/239655 objects degraded (6.381%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:14:28.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:28 smithi142 ceph-mon[106729]: pgmap v406: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 15292/239655 objects degraded (6.381%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:14:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:30 smithi152 ceph-mon[93456]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 15292/233715 objects degraded (6.543%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:14:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15292/233715 objects degraded (6.543%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:14:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:14:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:14:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:30 smithi142 ceph-mon[103614]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 15292/233715 objects degraded (6.543%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:14:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15292/233715 objects degraded (6.543%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:30 smithi142 ceph-mon[106729]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 15292/233715 objects degraded (6.543%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:14:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15292/233715 objects degraded (6.543%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:14:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:14:31] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.51.0" 2024-08-17T17:14:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:32 smithi152 ceph-mon[93456]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 15123/227838 objects degraded (6.638%); 2.4 MiB/s, 39 objects/s recovering 2024-08-17T17:14:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:32 smithi142 ceph-mon[103614]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 15123/227838 objects degraded (6.638%); 2.4 MiB/s, 39 objects/s recovering 2024-08-17T17:14:32.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:32 smithi142 ceph-mon[106729]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 15123/227838 objects degraded (6.638%); 2.4 MiB/s, 39 objects/s recovering 2024-08-17T17:14:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:34 smithi152 ceph-mon[93456]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 15123/221865 objects degraded (6.816%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:34.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:34 smithi142 ceph-mon[103614]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 15123/221865 objects degraded (6.816%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:34.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:34 smithi142 ceph-mon[106729]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 15123/221865 objects degraded (6.816%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15123/221865 objects degraded (6.816%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15123/221865 objects degraded (6.816%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15123/221865 objects degraded (6.816%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:36 smithi152 ceph-mon[93456]: pgmap v410: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 15123/218157 objects degraded (6.932%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:36 smithi142 ceph-mon[106729]: pgmap v410: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 15123/218157 objects degraded (6.932%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:36 smithi142 ceph-mon[103614]: pgmap v410: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 15123/218157 objects degraded (6.932%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:38 smithi152 ceph-mon[93456]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14972/215901 objects degraded (6.935%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:14:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:38 smithi142 ceph-mon[106729]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14972/215901 objects degraded (6.935%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:14:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:38 smithi142 ceph-mon[103614]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14972/215901 objects degraded (6.935%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:14:40.481 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:14:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:14:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:14:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:40 smithi152 ceph-mon[93456]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14972/209862 objects degraded (7.134%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14972/209862 objects degraded (7.134%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:40.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:40 smithi142 ceph-mon[106729]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14972/209862 objects degraded (7.134%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:40.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14972/209862 objects degraded (7.134%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:40 smithi142 ceph-mon[103614]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14972/209862 objects degraded (7.134%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:14:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14972/209862 objects degraded (7.134%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:41.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:14:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:14:41] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.51.0" 2024-08-17T17:14:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:42 smithi152 ceph-mon[93456]: pgmap v413: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 14796/203823 objects degraded (7.259%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:42 smithi142 ceph-mon[106729]: pgmap v413: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 14796/203823 objects degraded (7.259%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:42 smithi142 ceph-mon[103614]: pgmap v413: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 14796/203823 objects degraded (7.259%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:44 smithi152 ceph-mon[93456]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 14796/197886 objects degraded (7.477%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:14:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:44 smithi142 ceph-mon[106729]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 14796/197886 objects degraded (7.477%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:14:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:44 smithi142 ceph-mon[103614]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 14796/197886 objects degraded (7.477%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:14:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14796/197886 objects degraded (7.477%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14796/197886 objects degraded (7.477%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:45.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14796/197886 objects degraded (7.477%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:46 smithi152 ceph-mon[93456]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14796/194247 objects degraded (7.617%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:14:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:46 smithi142 ceph-mon[106729]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14796/194247 objects degraded (7.617%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:14:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:46 smithi142 ceph-mon[103614]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14796/194247 objects degraded (7.617%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:14:48.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:48 smithi152 ceph-mon[93456]: pgmap v416: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 14619/191895 objects degraded (7.618%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:48 smithi142 ceph-mon[106729]: pgmap v416: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 14619/191895 objects degraded (7.618%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:48 smithi142 ceph-mon[103614]: pgmap v416: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 14619/191895 objects degraded (7.618%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:14:50.528 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:14:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:14:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:14:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:50 smithi152 ceph-mon[93456]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 14619/185973 objects degraded (7.861%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:14:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14619/185973 objects degraded (7.861%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:50 smithi142 ceph-mon[106729]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 14619/185973 objects degraded (7.861%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:14:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14619/185973 objects degraded (7.861%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:50.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:50 smithi142 ceph-mon[103614]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 14619/185973 objects degraded (7.861%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:14:50.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14619/185973 objects degraded (7.861%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:14:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:14:51] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.51.0" 2024-08-17T17:14:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:52 smithi152 ceph-mon[93456]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14433/179949 objects degraded (8.021%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:14:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:52 smithi142 ceph-mon[106729]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14433/179949 objects degraded (8.021%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:14:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:52 smithi142 ceph-mon[103614]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14433/179949 objects degraded (8.021%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:14:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:54 smithi152 ceph-mon[93456]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14433/173904 objects degraded (8.299%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:14:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:54 smithi142 ceph-mon[106729]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14433/173904 objects degraded (8.299%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:14:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:54 smithi142 ceph-mon[103614]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 14433/173904 objects degraded (8.299%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:14:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14433/173904 objects degraded (8.299%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14433/173904 objects degraded (8.299%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14433/173904 objects degraded (8.299%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:14:56.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:56 smithi152 ceph-mon[93456]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14433/170298 objects degraded (8.475%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:14:56.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:56 smithi142 ceph-mon[103614]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14433/170298 objects degraded (8.475%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:14:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:56 smithi142 ceph-mon[106729]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14433/170298 objects degraded (8.475%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:14:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:58 smithi152 ceph-mon[93456]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 14221/167991 objects degraded (8.465%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:14:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:58 smithi142 ceph-mon[106729]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 14221/167991 objects degraded (8.465%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:14:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:58 smithi142 ceph-mon[103614]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 14221/167991 objects degraded (8.465%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:14:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:14:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:14:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:14:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:00.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:15:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:15:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:15:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:00 smithi152 ceph-mon[93456]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 14221/161847 objects degraded (8.787%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:15:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14221/161847 objects degraded (8.787%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:00 smithi142 ceph-mon[106729]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 14221/161847 objects degraded (8.787%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:15:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14221/161847 objects degraded (8.787%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:00 smithi142 ceph-mon[103614]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 14221/161847 objects degraded (8.787%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:15:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14221/161847 objects degraded (8.787%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:15:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:15:01] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:15:02.863 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:02 smithi152 ceph-mon[93456]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14012/156162 objects degraded (8.973%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:15:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:02 smithi142 ceph-mon[106729]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14012/156162 objects degraded (8.973%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:15:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:02 smithi142 ceph-mon[103614]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 14012/156162 objects degraded (8.973%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:15:04.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:04 smithi142 ceph-mon[103614]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 14012/151143 objects degraded (9.271%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:15:04.851 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:04 smithi142 ceph-mon[106729]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 14012/151143 objects degraded (9.271%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:15:04.864 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:04 smithi152 ceph-mon[93456]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 14012/151143 objects degraded (9.271%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:15:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14012/151143 objects degraded (9.271%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14012/151143 objects degraded (9.271%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14012/151143 objects degraded (9.271%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 14012/147051 objects degraded (9.529%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:07.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 14012/147051 objects degraded (9.529%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 14012/147051 objects degraded (9.529%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:15:07.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:15:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:09 smithi152 ceph-mon[93456]: pgmap v426: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 13776/144996 objects degraded (9.501%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:15:09.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:09 smithi142 ceph-mon[103614]: pgmap v426: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 13776/144996 objects degraded (9.501%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:15:09.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:09 smithi142 ceph-mon[106729]: pgmap v426: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 13776/144996 objects degraded (9.501%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:15:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:15:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:15:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:15:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:11 smithi152 ceph-mon[93456]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 13776/139251 objects degraded (9.893%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:15:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13776/139251 objects degraded (9.893%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:11.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:11 smithi142 ceph-mon[103614]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 13776/139251 objects degraded (9.893%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:15:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13776/139251 objects degraded (9.893%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:11 smithi142 ceph-mon[106729]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 13776/139251 objects degraded (9.893%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:15:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13776/139251 objects degraded (9.893%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:12.034 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:15:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:15:11] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-08-17T17:15:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:12.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:12.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:13.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:13 smithi142 ceph-mon[103614]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 13493/133371 objects degraded (10.117%); 2.6 MiB/s, 60 objects/s recovering 2024-08-17T17:15:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:13 smithi142 ceph-mon[106729]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 13493/133371 objects degraded (10.117%); 2.6 MiB/s, 60 objects/s recovering 2024-08-17T17:15:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:13 smithi152 ceph-mon[93456]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 13493/133371 objects degraded (10.117%); 2.6 MiB/s, 60 objects/s recovering 2024-08-17T17:15:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:15 smithi152 ceph-mon[93456]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 13493/128085 objects degraded (10.534%); 1.6 MiB/s, 43 objects/s recovering 2024-08-17T17:15:15.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:15 smithi142 ceph-mon[103614]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 13493/128085 objects degraded (10.534%); 1.6 MiB/s, 43 objects/s recovering 2024-08-17T17:15:15.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:15 smithi142 ceph-mon[106729]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 13493/128085 objects degraded (10.534%); 1.6 MiB/s, 43 objects/s recovering 2024-08-17T17:15:16.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13493/128085 objects degraded (10.534%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:16.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13493/128085 objects degraded (10.534%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:16.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13493/128085 objects degraded (10.534%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:17.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:17 smithi152 ceph-mon[93456]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 13493/127278 objects degraded (10.601%); 1.6 MiB/s, 43 objects/s recovering 2024-08-17T17:15:17.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:17 smithi142 ceph-mon[106729]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 13493/127278 objects degraded (10.601%); 1.6 MiB/s, 43 objects/s recovering 2024-08-17T17:15:17.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:17 smithi142 ceph-mon[103614]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 13493/127278 objects degraded (10.601%); 1.6 MiB/s, 43 objects/s recovering 2024-08-17T17:15:19.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:19 smithi152 ceph-mon[93456]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 13226/121386 objects degraded (10.896%); 1.6 MiB/s, 65 objects/s recovering 2024-08-17T17:15:19.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:19 smithi142 ceph-mon[106729]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 13226/121386 objects degraded (10.896%); 1.6 MiB/s, 65 objects/s recovering 2024-08-17T17:15:19.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:19 smithi142 ceph-mon[103614]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 13226/121386 objects degraded (10.896%); 1.6 MiB/s, 65 objects/s recovering 2024-08-17T17:15:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:15:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:15:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:15:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:21 smithi152 ceph-mon[93456]: pgmap v432: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 13226/115461 objects degraded (11.455%); 336 KiB/s, 45 objects/s recovering 2024-08-17T17:15:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13226/115461 objects degraded (11.455%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:21 smithi142 ceph-mon[106729]: pgmap v432: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 13226/115461 objects degraded (11.455%); 336 KiB/s, 45 objects/s recovering 2024-08-17T17:15:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13226/115461 objects degraded (11.455%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:21 smithi142 ceph-mon[103614]: pgmap v432: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 13226/115461 objects degraded (11.455%); 336 KiB/s, 45 objects/s recovering 2024-08-17T17:15:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13226/115461 objects degraded (11.455%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:15:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:15:21] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-08-17T17:15:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:23 smithi152 ceph-mon[93456]: pgmap v433: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 13226/111606 objects degraded (11.851%); 336 KiB/s, 45 objects/s recovering 2024-08-17T17:15:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:23 smithi142 ceph-mon[106729]: pgmap v433: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 13226/111606 objects degraded (11.851%); 336 KiB/s, 45 objects/s recovering 2024-08-17T17:15:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:23 smithi142 ceph-mon[103614]: pgmap v433: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 13226/111606 objects degraded (11.851%); 336 KiB/s, 45 objects/s recovering 2024-08-17T17:15:24.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:24 smithi142 ceph-mon[103614]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T17:15:24.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:24 smithi142 ceph-mon[106729]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T17:15:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:24 smithi152 ceph-mon[93456]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T17:15:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:25 smithi142 ceph-mon[103614]: pgmap v435: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12891/104025 objects degraded (12.392%); 0 B/s, 60 objects/s recovering 2024-08-17T17:15:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:25 smithi142 ceph-mon[103614]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T17:15:25.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:25 smithi142 ceph-mon[106729]: pgmap v435: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12891/104025 objects degraded (12.392%); 0 B/s, 60 objects/s recovering 2024-08-17T17:15:25.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:25 smithi142 ceph-mon[106729]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T17:15:25.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:25 smithi152 ceph-mon[93456]: pgmap v435: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12891/104025 objects degraded (12.392%); 0 B/s, 60 objects/s recovering 2024-08-17T17:15:25.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:25 smithi152 ceph-mon[93456]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T17:15:26.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 12891/104025 objects degraded (12.392%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:26.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 12891/104025 objects degraded (12.392%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 12891/104025 objects degraded (12.392%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:27 smithi142 ceph-mon[106729]: pgmap v437: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12891/103308 objects degraded (12.478%); 0 B/s, 41 objects/s recovering 2024-08-17T17:15:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:27.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:27 smithi142 ceph-mon[103614]: pgmap v437: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12891/103308 objects degraded (12.478%); 0 B/s, 41 objects/s recovering 2024-08-17T17:15:27.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:27 smithi152 ceph-mon[93456]: pgmap v437: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 12891/103308 objects degraded (12.478%); 0 B/s, 41 objects/s recovering 2024-08-17T17:15:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:29.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:29 smithi142 ceph-mon[103614]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12796/98052 objects degraded (13.050%); 0 B/s, 53 objects/s recovering 2024-08-17T17:15:29.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:29 smithi142 ceph-mon[106729]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12796/98052 objects degraded (13.050%); 0 B/s, 53 objects/s recovering 2024-08-17T17:15:29.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:29 smithi152 ceph-mon[93456]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 12796/98052 objects degraded (13.050%); 0 B/s, 53 objects/s recovering 2024-08-17T17:15:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:15:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:15:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:15:31.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:31 smithi142 ceph-mon[103614]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 12567/92781 objects degraded (13.545%); 0 B/s, 82 objects/s recovering 2024-08-17T17:15:31.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 12567/92781 objects degraded (13.545%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:31 smithi142 ceph-mon[106729]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 12567/92781 objects degraded (13.545%); 0 B/s, 82 objects/s recovering 2024-08-17T17:15:31.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 12567/92781 objects degraded (13.545%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:31.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:31 smithi152 ceph-mon[93456]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 12567/92781 objects degraded (13.545%); 0 B/s, 82 objects/s recovering 2024-08-17T17:15:31.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 12567/92781 objects degraded (13.545%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:15:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:15:31] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.51.0" 2024-08-17T17:15:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:33 smithi142 ceph-mon[103614]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 12567/88908 objects degraded (14.135%); 0 B/s, 39 objects/s recovering 2024-08-17T17:15:33.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:33 smithi142 ceph-mon[106729]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 12567/88908 objects degraded (14.135%); 0 B/s, 39 objects/s recovering 2024-08-17T17:15:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:33 smithi152 ceph-mon[93456]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 12567/88908 objects degraded (14.135%); 0 B/s, 39 objects/s recovering 2024-08-17T17:15:34.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:34 smithi142 ceph-mon[103614]: pgmap v441: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 12440/81093 objects degraded (15.340%); 0 B/s, 45 objects/s recovering 2024-08-17T17:15:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:34 smithi142 ceph-mon[106729]: pgmap v441: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 12440/81093 objects degraded (15.340%); 0 B/s, 45 objects/s recovering 2024-08-17T17:15:34.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:34 smithi152 ceph-mon[93456]: pgmap v441: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 12440/81093 objects degraded (15.340%); 0 B/s, 45 objects/s recovering 2024-08-17T17:15:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 12440/81093 objects degraded (15.340%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 12440/81093 objects degraded (15.340%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 12440/81093 objects degraded (15.340%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:36 smithi152 ceph-mon[93456]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 12440/80304 objects degraded (15.491%); 0 B/s, 40 objects/s recovering 2024-08-17T17:15:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:36 smithi142 ceph-mon[103614]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 12440/80304 objects degraded (15.491%); 0 B/s, 40 objects/s recovering 2024-08-17T17:15:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:36 smithi142 ceph-mon[106729]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 12440/80304 objects degraded (15.491%); 0 B/s, 40 objects/s recovering 2024-08-17T17:15:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:38 smithi152 ceph-mon[93456]: pgmap v443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 12440/74082 objects degraded (16.792%); 0 B/s, 37 objects/s recovering 2024-08-17T17:15:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:38 smithi142 ceph-mon[103614]: pgmap v443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 12440/74082 objects degraded (16.792%); 0 B/s, 37 objects/s recovering 2024-08-17T17:15:38.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:38 smithi142 ceph-mon[106729]: pgmap v443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 12440/74082 objects degraded (16.792%); 0 B/s, 37 objects/s recovering 2024-08-17T17:15:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:40 smithi152 ceph-mon[93456]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 12300/67989 objects degraded (18.091%); 0 B/s, 41 objects/s recovering 2024-08-17T17:15:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 12300/67989 objects degraded (18.091%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:15:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:15:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:15:40.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:40 smithi142 ceph-mon[103614]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 12300/67989 objects degraded (18.091%); 0 B/s, 41 objects/s recovering 2024-08-17T17:15:40.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 12300/67989 objects degraded (18.091%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:40 smithi142 ceph-mon[106729]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 12300/67989 objects degraded (18.091%); 0 B/s, 41 objects/s recovering 2024-08-17T17:15:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 12300/67989 objects degraded (18.091%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:41.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:15:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:15:41] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.51.0" 2024-08-17T17:15:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:42 smithi152 ceph-mon[93456]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 12300/64056 objects degraded (19.202%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:42 smithi142 ceph-mon[106729]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 12300/64056 objects degraded (19.202%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:42 smithi142 ceph-mon[103614]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 12300/64056 objects degraded (19.202%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:44 smithi152 ceph-mon[93456]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 12167/56307 objects degraded (21.608%); 0 B/s, 33 objects/s recovering 2024-08-17T17:15:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:44 smithi142 ceph-mon[106729]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 12167/56307 objects degraded (21.608%); 0 B/s, 33 objects/s recovering 2024-08-17T17:15:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:44 smithi142 ceph-mon[103614]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 12167/56307 objects degraded (21.608%); 0 B/s, 33 objects/s recovering 2024-08-17T17:15:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 12167/56307 objects degraded (21.608%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 12167/56307 objects degraded (21.608%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:45.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 12167/56307 objects degraded (21.608%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:46 smithi152 ceph-mon[93456]: pgmap v447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 12167/55536 objects degraded (21.908%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:46 smithi142 ceph-mon[106729]: pgmap v447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 12167/55536 objects degraded (21.908%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:46 smithi142 ceph-mon[103614]: pgmap v447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 12167/55536 objects degraded (21.908%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:48 smithi152 ceph-mon[93456]: pgmap v448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1022 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 12167/49119 objects degraded (24.770%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:48 smithi142 ceph-mon[103614]: pgmap v448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1022 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 12167/49119 objects degraded (24.770%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:48 smithi142 ceph-mon[106729]: pgmap v448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1022 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 12167/49119 objects degraded (24.770%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:50.477 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:15:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:15:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:15:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:50 smithi152 ceph-mon[93456]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 895 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 12031/43026 objects degraded (27.962%); 0 B/s, 34 objects/s recovering 2024-08-17T17:15:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 12031/43026 objects degraded (27.962%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:50 smithi142 ceph-mon[103614]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 895 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 12031/43026 objects degraded (27.962%); 0 B/s, 34 objects/s recovering 2024-08-17T17:15:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 12031/43026 objects degraded (27.962%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:50 smithi142 ceph-mon[106729]: pgmap v449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 895 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 12031/43026 objects degraded (27.962%); 0 B/s, 34 objects/s recovering 2024-08-17T17:15:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 12031/43026 objects degraded (27.962%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:15:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:15:51] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.51.0" 2024-08-17T17:15:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:52 smithi152 ceph-mon[93456]: pgmap v450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 816 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12031/39219 objects degraded (30.676%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:52 smithi142 ceph-mon[103614]: pgmap v450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 816 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12031/39219 objects degraded (30.676%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:52 smithi142 ceph-mon[106729]: pgmap v450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 816 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12031/39219 objects degraded (30.676%); 0 B/s, 22 objects/s recovering 2024-08-17T17:15:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:54 smithi152 ceph-mon[93456]: pgmap v451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 657 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11890/31575 objects degraded (37.656%); 0 B/s, 34 objects/s recovering 2024-08-17T17:15:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:54 smithi142 ceph-mon[106729]: pgmap v451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 657 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11890/31575 objects degraded (37.656%); 0 B/s, 34 objects/s recovering 2024-08-17T17:15:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:54 smithi142 ceph-mon[103614]: pgmap v451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 657 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11890/31575 objects degraded (37.656%); 0 B/s, 34 objects/s recovering 2024-08-17T17:15:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11890/31575 objects degraded (37.656%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11890/31575 objects degraded (37.656%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11890/31575 objects degraded (37.656%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:15:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:56 smithi152 ceph-mon[93456]: pgmap v452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 640 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 11890/30789 objects degraded (38.618%); 0 B/s, 23 objects/s recovering 2024-08-17T17:15:56.880 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:56.880 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:56 smithi142 ceph-mon[106729]: pgmap v452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 640 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 11890/30789 objects degraded (38.618%); 0 B/s, 23 objects/s recovering 2024-08-17T17:15:56.881 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:56.881 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:56 smithi142 ceph-mon[103614]: pgmap v452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 640 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 11890/30789 objects degraded (38.618%); 0 B/s, 23 objects/s recovering 2024-08-17T17:15:56.881 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:15:58 smithi152 ceph-mon[93456]: pgmap v453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 513 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 11890/24657 objects degraded (48.222%); 0 B/s, 23 objects/s recovering 2024-08-17T17:15:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:15:58 smithi142 ceph-mon[106729]: pgmap v453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 513 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 11890/24657 objects degraded (48.222%); 0 B/s, 23 objects/s recovering 2024-08-17T17:15:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:15:58 smithi142 ceph-mon[103614]: pgmap v453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 513 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 11890/24657 objects degraded (48.222%); 0 B/s, 23 objects/s recovering 2024-08-17T17:16:00.514 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:16:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:16:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:16:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:00 smithi152 ceph-mon[93456]: pgmap v454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 386 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 11752/18594 objects degraded (63.203%); 0 B/s, 34 objects/s recovering 2024-08-17T17:16:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11752/18594 objects degraded (63.203%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:16:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:00 smithi142 ceph-mon[106729]: pgmap v454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 386 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 11752/18594 objects degraded (63.203%); 0 B/s, 34 objects/s recovering 2024-08-17T17:16:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11752/18594 objects degraded (63.203%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:16:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:00 smithi142 ceph-mon[103614]: pgmap v454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 386 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 11752/18594 objects degraded (63.203%); 0 B/s, 34 objects/s recovering 2024-08-17T17:16:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11752/18594 objects degraded (63.203%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:16:02.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:16:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:16:01] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.51.0" 2024-08-17T17:16:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:02 smithi152 ceph-mon[93456]: pgmap v455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 309 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11752/14883 objects degraded (78.963%); 0 B/s, 23 objects/s recovering 2024-08-17T17:16:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:02 smithi142 ceph-mon[106729]: pgmap v455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 309 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11752/14883 objects degraded (78.963%); 0 B/s, 23 objects/s recovering 2024-08-17T17:16:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:02 smithi142 ceph-mon[103614]: pgmap v455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 309 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11752/14883 objects degraded (78.963%); 0 B/s, 23 objects/s recovering 2024-08-17T17:16:04.220 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 121293 objects 2024-08-17T17:16:04.221 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :152.037 2024-08-17T17:16:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:04 smithi152 ceph-mon[93456]: pgmap v456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 144 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11591/6945 objects degraded (166.897%); 0 B/s, 36 objects/s recovering 2024-08-17T17:16:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:04 smithi142 ceph-mon[106729]: pgmap v456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 144 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11591/6945 objects degraded (166.897%); 0 B/s, 36 objects/s recovering 2024-08-17T17:16:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:04 smithi142 ceph-mon[103614]: pgmap v456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 144 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11591/6945 objects degraded (166.897%); 0 B/s, 36 objects/s recovering 2024-08-17T17:16:05.170 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-08-17T17:16:05.171 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-08-17T17:16:05.467 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:05.808 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11591/6945 objects degraded (166.897%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:16:05.808 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11591/6945 objects degraded (166.897%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:16:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11591/6945 objects degraded (166.897%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:16:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:06 smithi152 ceph-mon[93456]: pgmap v457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 129 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 11591/6237 objects degraded (185.843%); 0 B/s, 24 objects/s recovering 2024-08-17T17:16:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:06 smithi142 ceph-mon[106729]: pgmap v457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 129 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 11591/6237 objects degraded (185.843%); 0 B/s, 24 objects/s recovering 2024-08-17T17:16:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:06 smithi142 ceph-mon[103614]: pgmap v457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 129 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 11591/6237 objects degraded (185.843%); 0 B/s, 24 objects/s recovering 2024-08-17T17:16:07.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:07.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:16:08.550 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_3' removed 2024-08-17T17:16:08.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:08 smithi152 ceph-mon[93456]: pgmap v458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 41 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 11591/2025 objects degraded (572.395%); 0 B/s, 24 objects/s recovering 2024-08-17T17:16:08.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4196838471' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:16:08.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:08 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:16:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:08 smithi142 ceph-mon[103614]: pgmap v458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 41 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 11591/2025 objects degraded (572.395%); 0 B/s, 24 objects/s recovering 2024-08-17T17:16:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4196838471' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:16:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:08 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:16:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:08 smithi142 ceph-mon[106729]: pgmap v458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 41 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 11591/2025 objects degraded (572.395%); 0 B/s, 24 objects/s recovering 2024-08-17T17:16:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4196838471' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:16:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:08 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:16:09.274 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:16:09.274 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-08-17T17:16:09.274 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:09.275 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:16:09.285 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-08-17T17:16:09.285 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T17:16:09.293 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:16:09.293 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:16:09.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:16:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:16:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:09 smithi152 ceph-mon[93456]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T17:16:09.866 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:16:09.866 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:09 smithi142 ceph-mon[106729]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T17:16:09.866 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:16:09.866 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:09 smithi142 ceph-mon[103614]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T17:16:10.516 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:10.548 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:16:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:16:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:16:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:10 smithi152 ceph-mon[93456]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:10 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11591/2025 objects degraded (572.395%), 4 pgs degraded, 4 pgs undersized) 2024-08-17T17:16:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:10 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:16:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:10 smithi142 ceph-mon[103614]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:10 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11591/2025 objects degraded (572.395%), 4 pgs degraded, 4 pgs undersized) 2024-08-17T17:16:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:10 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:16:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:10 smithi142 ceph-mon[106729]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:10 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11591/2025 objects degraded (572.395%), 4 pgs degraded, 4 pgs undersized) 2024-08-17T17:16:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:10 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:16:12.014 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:16:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:16:11] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.51.0" 2024-08-17T17:16:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:12 smithi142 ceph-mon[103614]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:12 smithi142 ceph-mon[106729]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:12.733 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:16:12.738 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:12 smithi152 ceph-mon[93456]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:12.738 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:12.738 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:13.626 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":41,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6800","nonce":2787043323},{"type":"v1","addr":"172.21.15.142:6801","nonce":2787043323}]},"active_addr":"172.21.15.142:6801/2787043323","active_change":"2024-08-17T17:03:11.046419+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25252,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":85,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":1002900933}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":2169826866}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":4224473030}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.142:0","nonce":2675396804}]}]} 2024-08-17T17:16:13.634 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:16:13.634 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:16:13.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:16:13.659 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1187024142' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:16:13.660 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1187024142' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:16:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1187024142' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:16:14.325 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:15.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:16.187 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:16:16.187 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":153,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T17:16:08.533950+0000","last_up_change":"2024-08-17T17:12:39.314486+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:57:27.258162+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:57:27.611866+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":151,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6803","nonce":608035512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6805","nonce":608035512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6809","nonce":608035512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6807","nonce":608035512}]},"public_addr":"172.21.15.142:6803/608035512","cluster_addr":"172.21.15.142:6805/608035512","heartbeat_back_addr":"172.21.15.142:6809/608035512","heartbeat_front_addr":"172.21.15.142:6807/608035512","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":100,"up_from":106,"up_thru":148,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2324159993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2324159993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2324159993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2324159993}]},"public_addr":"172.21.15.142:6811/2324159993","cluster_addr":"172.21.15.142:6813/2324159993","heartbeat_back_addr":"172.21.15.142:6817/2324159993","heartbeat_front_addr":"172.21.15.142:6815/2324159993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":146,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6819","nonce":366314937}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6821","nonce":366314937}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6825","nonce":366314937}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6823","nonce":366314937}]},"public_addr":"172.21.15.142:6819/366314937","cluster_addr":"172.21.15.142:6821/366314937","heartbeat_back_addr":"172.21.15.142:6825/366314937","heartbeat_front_addr":"172.21.15.142:6823/366314937","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":142,"up_from":146,"up_thru":149,"down_at":143,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6827","nonce":195335216}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6829","nonce":195335216}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6833","nonce":195335216}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6831","nonce":195335216}]},"public_addr":"172.21.15.142:6827/195335216","cluster_addr":"172.21.15.142:6829/195335216","heartbeat_back_addr":"172.21.15.142:6833/195335216","heartbeat_front_addr":"172.21.15.142:6831/195335216","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":146,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":147,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":143,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":146,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":100},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":142},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:6801/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1253216666":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/3034915416":"2024-08-18T17:03:11.046184+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.152:0/681595170":"2024-08-18T17:02:18.823112+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6800/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/267249921":"2024-08-18T17:03:11.046184+0000","172.21.15.152:6834/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3319043742":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1554300251":"2024-08-18T17:00:33.779952+0000","172.21.15.152:0/3235892463":"2024-08-18T17:02:18.823112+0000","172.21.15.152:0/3133806592":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2472242802":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/639679378":"2024-08-18T17:00:33.779952+0000","172.21.15.152:6835/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/1828093647":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/2627951813":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/3716608685":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.152:0/3126412235":"2024-08-18T17:02:18.823112+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:16:16.793 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:16:16.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:16:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:16 smithi152 ceph-mon[93456]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4119972837' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:16:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:16 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:17.092 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:16 smithi142 ceph-mon[106729]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4119972837' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:16:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:16 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:16 smithi142 ceph-mon[103614]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4119972837' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:16:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:16 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:16:18.939 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:16:18.940 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:18 smithi142 ceph-mon[106729]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:18.940 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:18 smithi142 ceph-mon[103614]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:18.940 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":153,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T17:16:08.533950+0000","last_up_change":"2024-08-17T17:12:39.314486+0000","last_in_change":"2024-08-17T16:55:25.325388+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:57:27.258162+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:57:27.611866+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":151,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6803","nonce":608035512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6805","nonce":608035512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6809","nonce":608035512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6807","nonce":608035512}]},"public_addr":"172.21.15.142:6803/608035512","cluster_addr":"172.21.15.142:6805/608035512","heartbeat_back_addr":"172.21.15.142:6809/608035512","heartbeat_front_addr":"172.21.15.142:6807/608035512","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":100,"up_from":106,"up_thru":148,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2324159993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2324159993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2324159993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2324159993}]},"public_addr":"172.21.15.142:6811/2324159993","cluster_addr":"172.21.15.142:6813/2324159993","heartbeat_back_addr":"172.21.15.142:6817/2324159993","heartbeat_front_addr":"172.21.15.142:6815/2324159993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":146,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6819","nonce":366314937}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6821","nonce":366314937}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6825","nonce":366314937}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6823","nonce":366314937}]},"public_addr":"172.21.15.142:6819/366314937","cluster_addr":"172.21.15.142:6821/366314937","heartbeat_back_addr":"172.21.15.142:6825/366314937","heartbeat_front_addr":"172.21.15.142:6823/366314937","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":142,"up_from":146,"up_thru":149,"down_at":143,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6827","nonce":195335216}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6829","nonce":195335216}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6833","nonce":195335216}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6831","nonce":195335216}]},"public_addr":"172.21.15.142:6827/195335216","cluster_addr":"172.21.15.142:6829/195335216","heartbeat_back_addr":"172.21.15.142:6833/195335216","heartbeat_front_addr":"172.21.15.142:6831/195335216","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":146,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":147,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6809","nonce":1844077029}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6811","nonce":1844077029}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6815","nonce":1844077029}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1844077029},{"type":"v1","addr":"172.21.15.152:6813","nonce":1844077029}]},"public_addr":"172.21.15.152:6809/1844077029","cluster_addr":"172.21.15.152:6811/1844077029","heartbeat_back_addr":"172.21.15.152:6815/1844077029","heartbeat_front_addr":"172.21.15.152:6813/1844077029","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":143,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6817","nonce":2120783576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6819","nonce":2120783576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6823","nonce":2120783576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":2120783576},{"type":"v1","addr":"172.21.15.152:6821","nonce":2120783576}]},"public_addr":"172.21.15.152:6817/2120783576","cluster_addr":"172.21.15.152:6819/2120783576","heartbeat_back_addr":"172.21.15.152:6823/2120783576","heartbeat_front_addr":"172.21.15.152:6821/2120783576","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":146,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6825","nonce":1562643638}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6827","nonce":1562643638}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6831","nonce":1562643638}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":1562643638},{"type":"v1","addr":"172.21.15.152:6829","nonce":1562643638}]},"public_addr":"172.21.15.152:6825/1562643638","cluster_addr":"172.21.15.152:6827/1562643638","heartbeat_back_addr":"172.21.15.152:6831/1562643638","heartbeat_front_addr":"172.21.15.152:6829/1562643638","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":100},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":142},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:6801/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1253216666":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/3034915416":"2024-08-18T17:03:11.046184+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.152:0/681595170":"2024-08-18T17:02:18.823112+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6800/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/267249921":"2024-08-18T17:03:11.046184+0000","172.21.15.152:6834/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3319043742":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1554300251":"2024-08-18T17:00:33.779952+0000","172.21.15.152:0/3235892463":"2024-08-18T17:02:18.823112+0000","172.21.15.152:0/3133806592":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2472242802":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/639679378":"2024-08-18T17:00:33.779952+0000","172.21.15.152:6835/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/1828093647":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/2627951813":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/3716608685":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.152:0/3126412235":"2024-08-18T17:02:18.823112+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:16:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:18 smithi152 ceph-mon[93456]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:19.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:16:19.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:16:19.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:16:19.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:16:19.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:16:19.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:16:19.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:16:19.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:16:19.802 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/985738769' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:16:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/985738769' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:16:20.101 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.102 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.104 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.105 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.109 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.109 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.110 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/985738769' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:16:20.140 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:16:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:16:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:16:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:20 smithi152 ceph-mon[93456]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:20 smithi142 ceph-mon[106729]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:20 smithi142 ceph-mon[103614]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:21.860 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:16:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:16:21] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.51.0" 2024-08-17T17:16:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:22 smithi152 ceph-mon[93456]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:22 smithi142 ceph-mon[106729]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:22 smithi142 ceph-mon[103614]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:24.950 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:24 smithi142 ceph-mon[106729]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:24.950 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:24 smithi142 ceph-mon[103614]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:24 smithi152 ceph-mon[93456]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:27.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:26 smithi142 ceph-mon[106729]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:27.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:27.033 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:26 smithi142 ceph-mon[103614]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:27.034 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:26 smithi152 ceph-mon[93456]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:28 smithi152 ceph-mon[93456]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:28 smithi142 ceph-mon[103614]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:28 smithi142 ceph-mon[106729]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:16:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:16:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:16:30.768 INFO:teuthology.orchestra.run.smithi142.stdout:455266533481 2024-08-17T17:16:30.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:16:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:30 smithi152 ceph-mon[93456]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:31.116 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:30 smithi142 ceph-mon[106729]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:31.116 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:30 smithi142 ceph-mon[103614]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:31.143 INFO:teuthology.orchestra.run.smithi142.stdout:167503724808 2024-08-17T17:16:31.143 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:16:31.687 INFO:teuthology.orchestra.run.smithi142.stdout:201863463170 2024-08-17T17:16:31.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:16:31.921 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:16:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:16:31] "GET /metrics HTTP/1.1" 200 34009 "" "Prometheus/2.51.0" 2024-08-17T17:16:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:32 smithi152 ceph-mon[93456]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:33.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:32 smithi142 ceph-mon[106729]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:33.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:32 smithi142 ceph-mon[103614]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:35.047 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:34 smithi142 ceph-mon[106729]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:35.047 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:34 smithi142 ceph-mon[103614]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:35.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:34 smithi152 ceph-mon[93456]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:35.345 INFO:teuthology.orchestra.run.smithi142.stdout:627065225264 2024-08-17T17:16:35.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:16:35.767 INFO:teuthology.orchestra.run.smithi142.stdout:133143986444 2024-08-17T17:16:35.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:16:36.157 INFO:teuthology.orchestra.run.smithi142.stdout:382252089459 2024-08-17T17:16:36.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:16:36.856 INFO:teuthology.orchestra.run.smithi142.stdout:236223201535 2024-08-17T17:16:36.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:16:36.892 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:36 smithi142 ceph-mon[106729]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:36 smithi142 ceph-mon[103614]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:36 smithi152 ceph-mon[93456]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:37.343 INFO:teuthology.orchestra.run.smithi142.stdout:515396075590 2024-08-17T17:16:37.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:16:38.054 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:38.098 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:38.099 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:38.530 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:38.572 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:16:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=sqlstore.transactions t=2024-08-17T17:16:38.36694556Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:16:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:38 smithi152 ceph-mon[93456]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:39.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:38 smithi142 ceph-mon[106729]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:39.084 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:38 smithi142 ceph-mon[103614]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:16:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:16:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:16:41.087 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:41.088 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:41.094 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:40 smithi142 ceph-mon[103614]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:41.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:40 smithi142 ceph-mon[106729]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:41.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:40 smithi152 ceph-mon[93456]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:41.957 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:41.957 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:16:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:16:41] "GET /metrics HTTP/1.1" 200 34010 "" "Prometheus/2.51.0" 2024-08-17T17:16:41.957 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:43.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:42 smithi142 ceph-mon[106729]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:43.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2559877416' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:16:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:42 smithi142 ceph-mon[103614]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:43.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2559877416' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:16:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:42 smithi152 ceph-mon[93456]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:43.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2559877416' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:16:43.481 INFO:teuthology.orchestra.run.smithi142.stdout:201863463172 2024-08-17T17:16:44.110 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463170 got 201863463172 for osd.6 2024-08-17T17:16:44.110 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:45.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:44 smithi142 ceph-mon[103614]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:45.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:44 smithi142 ceph-mon[106729]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:45.277 INFO:teuthology.orchestra.run.smithi142.stdout:167503724811 2024-08-17T17:16:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:44 smithi152 ceph-mon[93456]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:45.994 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1826359018' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:16:45.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1826359018' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:16:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1826359018' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:16:47.061 INFO:teuthology.orchestra.run.smithi142.stdout:627065225267 2024-08-17T17:16:47.061 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:46 smithi142 ceph-mon[106729]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:47.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:46 smithi142 ceph-mon[103614]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:47.146 INFO:teuthology.orchestra.run.smithi142.stdout:133143986447 2024-08-17T17:16:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:46 smithi152 ceph-mon[93456]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:47.870 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1821689531' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:16:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3226529433' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:16:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1821689531' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:16:48.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3226529433' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:16:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1821689531' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:16:48.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3226529433' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:16:49.019 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:48 smithi142 ceph-mon[106729]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:49.020 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4081045664' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:16:49.020 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:48 smithi142 ceph-mon[103614]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:49.020 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4081045664' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:16:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:48 smithi152 ceph-mon[93456]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4081045664' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:16:49.497 INFO:teuthology.orchestra.run.smithi142.stdout:515396075592 2024-08-17T17:16:50.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/533698466' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:16:50.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/533698466' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:16:50.102 INFO:tasks.cephadm.ceph_manager.ceph:need seq 515396075590 got 515396075592 for osd.2 2024-08-17T17:16:50.102 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:50.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/533698466' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:16:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:16:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:16:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:16:50.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724808 got 167503724811 for osd.5 2024-08-17T17:16:50.892 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:51.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:50 smithi142 ceph-mon[106729]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:51.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:50 smithi142 ceph-mon[103614]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:51.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:50 smithi152 ceph-mon[93456]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:51.489 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986444 got 133143986447 for osd.4 2024-08-17T17:16:51.490 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:51.900 INFO:tasks.cephadm.ceph_manager.ceph:need seq 627065225264 got 627065225267 for osd.3 2024-08-17T17:16:51.900 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:52.054 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:16:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:16:51] "GET /metrics HTTP/1.1" 200 34010 "" "Prometheus/2.51.0" 2024-08-17T17:16:52.154 INFO:teuthology.orchestra.run.smithi142.stdout:382252089462 2024-08-17T17:16:52.792 INFO:teuthology.orchestra.run.smithi142.stdout:455266533486 2024-08-17T17:16:53.062 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:52 smithi142 ceph-mon[106729]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:53.062 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1470797199' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:16:53.063 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:52 smithi142 ceph-mon[103614]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:53.063 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1470797199' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:16:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:52 smithi152 ceph-mon[93456]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1470797199' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:16:53.455 INFO:tasks.cephadm.ceph_manager.ceph:need seq 455266533481 got 455266533486 for osd.1 2024-08-17T17:16:53.455 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:54.146 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089459 got 382252089462 for osd.0 2024-08-17T17:16:54.146 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:54.507 INFO:teuthology.orchestra.run.smithi142.stdout:236223201539 2024-08-17T17:16:55.106 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:54 smithi142 ceph-mon[106729]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:55.107 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3128631266' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:16:55.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:54 smithi142 ceph-mon[103614]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:55.107 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3128631266' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:16:55.130 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201535 got 236223201539 for osd.7 2024-08-17T17:16:55.131 DEBUG:teuthology.parallel:result is None 2024-08-17T17:16:55.131 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:16:55.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:16:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:54 smithi152 ceph-mon[93456]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3128631266' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:16:55.390 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:56 smithi142 ceph-mon[106729]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:56 smithi142 ceph-mon[103614]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:57.256 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:16:57.257 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:16:57.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:56 smithi152 ceph-mon[93456]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:58.024 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":483,"stamp":"2024-08-17T17:16:55.363152+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3276584,"kb_used_data":15560,"kb_used_omap":29,"kb_used_meta":3260898,"kb_avail":746684632,"statfs":{"total":767960285184,"available":764605063168,"internally_reserved":0,"allocated":15933440,"data_stored":9812342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30202,"internal_metadata":3339160070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002981"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293249+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:16:09.293249+0000","last_peered":"2024-08-17T17:16:09.293249+0000","last_clean":"2024-08-17T17:16:09.293249+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:16:09.293249+0000","last_undegraded":"2024-08-17T17:16:09.293249+0000","last_fullsized":"2024-08-17T17:16:09.293249+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086937+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:15:24.086937+0000","last_peered":"2024-08-17T17:15:24.086937+0000","last_clean":"2024-08-17T17:15:24.086937+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:15:24.086937+0000","last_undegraded":"2024-08-17T17:15:24.086937+0000","last_fullsized":"2024-08-17T17:15:24.086937+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086573+0000","last_change":"2024-08-17T17:12:40.335427+0000","last_active":"2024-08-17T17:15:24.086573+0000","last_peered":"2024-08-17T17:15:24.086573+0000","last_clean":"2024-08-17T17:15:24.086573+0000","last_became_active":"2024-08-17T17:12:40.334535+0000","last_became_peered":"2024-08-17T17:12:40.334535+0000","last_unstale":"2024-08-17T17:15:24.086573+0000","last_undegraded":"2024-08-17T17:15:24.086573+0000","last_fullsized":"2024-08-17T17:15:24.086573+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:47:39.810507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.299324+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:16:09.299324+0000","last_peered":"2024-08-17T17:16:09.299324+0000","last_clean":"2024-08-17T17:16:09.299324+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:16:09.299324+0000","last_undegraded":"2024-08-17T17:16:09.299324+0000","last_fullsized":"2024-08-17T17:16:09.299324+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295526+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:16:09.295526+0000","last_peered":"2024-08-17T17:16:09.295526+0000","last_clean":"2024-08-17T17:16:09.295526+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:16:09.295526+0000","last_undegraded":"2024-08-17T17:16:09.295526+0000","last_fullsized":"2024-08-17T17:16:09.295526+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086438+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:15:24.086438+0000","last_peered":"2024-08-17T17:15:24.086438+0000","last_clean":"2024-08-17T17:15:24.086438+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:15:24.086438+0000","last_undegraded":"2024-08-17T17:15:24.086438+0000","last_fullsized":"2024-08-17T17:15:24.086438+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292483+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:16:09.292483+0000","last_peered":"2024-08-17T17:16:09.292483+0000","last_clean":"2024-08-17T17:16:09.292483+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:16:09.292483+0000","last_undegraded":"2024-08-17T17:16:09.292483+0000","last_fullsized":"2024-08-17T17:16:09.292483+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127961+0000","last_change":"2024-08-17T17:07:07.418480+0000","last_active":"2024-08-17T17:15:24.127961+0000","last_peered":"2024-08-17T17:15:24.127961+0000","last_clean":"2024-08-17T17:15:24.127961+0000","last_became_active":"2024-08-17T17:07:07.418124+0000","last_became_peered":"2024-08-17T17:07:07.418124+0000","last_unstale":"2024-08-17T17:15:24.127961+0000","last_undegraded":"2024-08-17T17:15:24.127961+0000","last_fullsized":"2024-08-17T17:15:24.127961+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:12.020815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087282+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:15:24.087282+0000","last_peered":"2024-08-17T17:15:24.087282+0000","last_clean":"2024-08-17T17:15:24.087282+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:15:24.087282+0000","last_undegraded":"2024-08-17T17:15:24.087282+0000","last_fullsized":"2024-08-17T17:15:24.087282+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.088543+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T17:15:24.088543+0000","last_peered":"2024-08-17T17:15:24.088543+0000","last_clean":"2024-08-17T17:15:24.088543+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T17:15:24.088543+0000","last_undegraded":"2024-08-17T17:15:24.088543+0000","last_fullsized":"2024-08-17T17:15:24.088543+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087299+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:15:24.087299+0000","last_peered":"2024-08-17T17:15:24.087299+0000","last_clean":"2024-08-17T17:15:24.087299+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:15:24.087299+0000","last_undegraded":"2024-08-17T17:15:24.087299+0000","last_fullsized":"2024-08-17T17:15:24.087299+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129675+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:15:24.129675+0000","last_peered":"2024-08-17T17:15:24.129675+0000","last_clean":"2024-08-17T17:15:24.129675+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:15:24.129675+0000","last_undegraded":"2024-08-17T17:15:24.129675+0000","last_fullsized":"2024-08-17T17:15:24.129675+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087183+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T17:15:24.087183+0000","last_peered":"2024-08-17T17:15:24.087183+0000","last_clean":"2024-08-17T17:15:24.087183+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T17:15:24.087183+0000","last_undegraded":"2024-08-17T17:15:24.087183+0000","last_fullsized":"2024-08-17T17:15:24.087183+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127970+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:15:24.127970+0000","last_peered":"2024-08-17T17:15:24.127970+0000","last_clean":"2024-08-17T17:15:24.127970+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:15:24.127970+0000","last_undegraded":"2024-08-17T17:15:24.127970+0000","last_fullsized":"2024-08-17T17:15:24.127970+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086790+0000","last_change":"2024-08-17T17:12:40.336371+0000","last_active":"2024-08-17T17:15:24.086790+0000","last_peered":"2024-08-17T17:15:24.086790+0000","last_clean":"2024-08-17T17:15:24.086790+0000","last_became_active":"2024-08-17T17:12:40.335783+0000","last_became_peered":"2024-08-17T17:12:40.335783+0000","last_unstale":"2024-08-17T17:15:24.086790+0000","last_undegraded":"2024-08-17T17:15:24.086790+0000","last_fullsized":"2024-08-17T17:15:24.086790+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:14.313020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291175+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:16:09.291175+0000","last_peered":"2024-08-17T17:16:09.291175+0000","last_clean":"2024-08-17T17:16:09.291175+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:16:09.291175+0000","last_undegraded":"2024-08-17T17:16:09.291175+0000","last_fullsized":"2024-08-17T17:16:09.291175+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295720+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:16:09.295720+0000","last_peered":"2024-08-17T17:16:09.295720+0000","last_clean":"2024-08-17T17:16:09.295720+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:16:09.295720+0000","last_undegraded":"2024-08-17T17:16:09.295720+0000","last_fullsized":"2024-08-17T17:16:09.295720+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130241+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:15:24.130241+0000","last_peered":"2024-08-17T17:15:24.130241+0000","last_clean":"2024-08-17T17:15:24.130241+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:15:24.130241+0000","last_undegraded":"2024-08-17T17:15:24.130241+0000","last_fullsized":"2024-08-17T17:15:24.130241+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128095+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:15:24.128095+0000","last_peered":"2024-08-17T17:15:24.128095+0000","last_clean":"2024-08-17T17:15:24.128095+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:15:24.128095+0000","last_undegraded":"2024-08-17T17:15:24.128095+0000","last_fullsized":"2024-08-17T17:15:24.128095+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.133692+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:15:24.133692+0000","last_peered":"2024-08-17T17:15:24.133692+0000","last_clean":"2024-08-17T17:15:24.133692+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:15:24.133692+0000","last_undegraded":"2024-08-17T17:15:24.133692+0000","last_fullsized":"2024-08-17T17:15:24.133692+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129775+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:15:24.129775+0000","last_peered":"2024-08-17T17:15:24.129775+0000","last_clean":"2024-08-17T17:15:24.129775+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:15:24.129775+0000","last_undegraded":"2024-08-17T17:15:24.129775+0000","last_fullsized":"2024-08-17T17:15:24.129775+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130215+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:15:24.130215+0000","last_peered":"2024-08-17T17:15:24.130215+0000","last_clean":"2024-08-17T17:15:24.130215+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:15:24.130215+0000","last_undegraded":"2024-08-17T17:15:24.130215+0000","last_fullsized":"2024-08-17T17:15:24.130215+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293324+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:16:09.293324+0000","last_peered":"2024-08-17T17:16:09.293324+0000","last_clean":"2024-08-17T17:16:09.293324+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:16:09.293324+0000","last_undegraded":"2024-08-17T17:16:09.293324+0000","last_fullsized":"2024-08-17T17:16:09.293324+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086551+0000","last_change":"2024-08-17T17:12:40.335001+0000","last_active":"2024-08-17T17:15:24.086551+0000","last_peered":"2024-08-17T17:15:24.086551+0000","last_clean":"2024-08-17T17:15:24.086551+0000","last_became_active":"2024-08-17T17:12:40.334650+0000","last_became_peered":"2024-08-17T17:12:40.334650+0000","last_unstale":"2024-08-17T17:15:24.086551+0000","last_undegraded":"2024-08-17T17:15:24.086551+0000","last_fullsized":"2024-08-17T17:15:24.086551+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:06.876541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":449,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086423+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:15:24.086423+0000","last_peered":"2024-08-17T17:15:24.086423+0000","last_clean":"2024-08-17T17:15:24.086423+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:15:24.086423+0000","last_undegraded":"2024-08-17T17:15:24.086423+0000","last_fullsized":"2024-08-17T17:15:24.086423+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086615+0000","last_change":"2024-08-17T17:10:55.313434+0000","last_active":"2024-08-17T17:15:24.086615+0000","last_peered":"2024-08-17T17:15:24.086615+0000","last_clean":"2024-08-17T17:15:24.086615+0000","last_became_active":"2024-08-17T17:10:55.313091+0000","last_became_peered":"2024-08-17T17:10:55.313091+0000","last_unstale":"2024-08-17T17:15:24.086615+0000","last_undegraded":"2024-08-17T17:15:24.086615+0000","last_fullsized":"2024-08-17T17:15:24.086615+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:17.970517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293055+0000","last_change":"2024-08-17T17:12:40.332647+0000","last_active":"2024-08-17T17:16:09.293055+0000","last_peered":"2024-08-17T17:16:09.293055+0000","last_clean":"2024-08-17T17:16:09.293055+0000","last_became_active":"2024-08-17T17:12:40.332376+0000","last_became_peered":"2024-08-17T17:12:40.332376+0000","last_unstale":"2024-08-17T17:16:09.293055+0000","last_undegraded":"2024-08-17T17:16:09.293055+0000","last_fullsized":"2024-08-17T17:16:09.293055+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:21.624216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128216+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:15:24.128216+0000","last_peered":"2024-08-17T17:15:24.128216+0000","last_clean":"2024-08-17T17:15:24.128216+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:15:24.128216+0000","last_undegraded":"2024-08-17T17:15:24.128216+0000","last_fullsized":"2024-08-17T17:15:24.128216+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086495+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:15:24.086495+0000","last_peered":"2024-08-17T17:15:24.086495+0000","last_clean":"2024-08-17T17:15:24.086495+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:15:24.086495+0000","last_undegraded":"2024-08-17T17:15:24.086495+0000","last_fullsized":"2024-08-17T17:15:24.086495+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291546+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:16:09.291546+0000","last_peered":"2024-08-17T17:16:09.291546+0000","last_clean":"2024-08-17T17:16:09.291546+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:16:09.291546+0000","last_undegraded":"2024-08-17T17:16:09.291546+0000","last_fullsized":"2024-08-17T17:16:09.291546+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295310+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:16:09.295310+0000","last_peered":"2024-08-17T17:16:09.295310+0000","last_clean":"2024-08-17T17:16:09.295310+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:16:09.295310+0000","last_undegraded":"2024-08-17T17:16:09.295310+0000","last_fullsized":"2024-08-17T17:16:09.295310+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":150,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.542779+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:16:08.542779+0000","last_peered":"2024-08-17T17:16:08.542779+0000","last_clean":"2024-08-17T17:16:08.542779+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:16:08.542779+0000","last_undegraded":"2024-08-17T17:16:08.542779+0000","last_fullsized":"2024-08-17T17:16:08.542779+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.546939+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:16:08.546939+0000","last_peered":"2024-08-17T17:16:08.546939+0000","last_clean":"2024-08-17T17:16:08.546939+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:16:08.546939+0000","last_undegraded":"2024-08-17T17:16:08.546939+0000","last_fullsized":"2024-08-17T17:16:08.546939+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.557064+0000","last_change":"2024-08-17T17:07:51.480410+0000","last_active":"2024-08-17T17:16:08.557064+0000","last_peered":"2024-08-17T17:16:08.557064+0000","last_clean":"2024-08-17T17:16:08.557064+0000","last_became_active":"2024-08-17T17:07:51.480083+0000","last_became_peered":"2024-08-17T17:07:51.480083+0000","last_unstale":"2024-08-17T17:16:08.557064+0000","last_undegraded":"2024-08-17T17:16:08.557064+0000","last_fullsized":"2024-08-17T17:16:08.557064+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:39.376320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086589+0000","last_change":"2024-08-17T17:07:51.483538+0000","last_active":"2024-08-17T17:15:24.086589+0000","last_peered":"2024-08-17T17:15:24.086589+0000","last_clean":"2024-08-17T17:15:24.086589+0000","last_became_active":"2024-08-17T17:07:51.482970+0000","last_became_peered":"2024-08-17T17:07:51.482970+0000","last_unstale":"2024-08-17T17:15:24.086589+0000","last_undegraded":"2024-08-17T17:15:24.086589+0000","last_fullsized":"2024-08-17T17:15:24.086589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:37.985321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130448+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:15:24.130448+0000","last_peered":"2024-08-17T17:15:24.130448+0000","last_clean":"2024-08-17T17:15:24.130448+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:15:24.130448+0000","last_undegraded":"2024-08-17T17:15:24.130448+0000","last_fullsized":"2024-08-17T17:15:24.130448+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544455+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:16:08.544455+0000","last_peered":"2024-08-17T17:16:08.544455+0000","last_clean":"2024-08-17T17:16:08.544455+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:16:08.544455+0000","last_undegraded":"2024-08-17T17:16:08.544455+0000","last_fullsized":"2024-08-17T17:16:08.544455+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086721+0000","last_change":"2024-08-17T17:12:40.335196+0000","last_active":"2024-08-17T17:15:24.086721+0000","last_peered":"2024-08-17T17:15:24.086721+0000","last_clean":"2024-08-17T17:15:24.086721+0000","last_became_active":"2024-08-17T17:12:40.334917+0000","last_became_peered":"2024-08-17T17:12:40.334917+0000","last_unstale":"2024-08-17T17:15:24.086721+0000","last_undegraded":"2024-08-17T17:15:24.086721+0000","last_fullsized":"2024-08-17T17:15:24.086721+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:02.633323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":169,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544620+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:16:08.544620+0000","last_peered":"2024-08-17T17:16:08.544620+0000","last_clean":"2024-08-17T17:16:08.544620+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:16:08.544620+0000","last_undegraded":"2024-08-17T17:16:08.544620+0000","last_fullsized":"2024-08-17T17:16:08.544620+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545444+0000","last_change":"2024-08-17T17:07:51.489205+0000","last_active":"2024-08-17T17:16:08.545444+0000","last_peered":"2024-08-17T17:16:08.545444+0000","last_clean":"2024-08-17T17:16:08.545444+0000","last_became_active":"2024-08-17T17:07:51.488783+0000","last_became_peered":"2024-08-17T17:07:51.488783+0000","last_unstale":"2024-08-17T17:16:08.545444+0000","last_undegraded":"2024-08-17T17:16:08.545444+0000","last_fullsized":"2024-08-17T17:16:08.545444+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:22.190780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544540+0000","last_change":"2024-08-17T17:10:55.321862+0000","last_active":"2024-08-17T17:16:08.544540+0000","last_peered":"2024-08-17T17:16:08.544540+0000","last_clean":"2024-08-17T17:16:08.544540+0000","last_became_active":"2024-08-17T17:10:55.321339+0000","last_became_peered":"2024-08-17T17:10:55.321339+0000","last_unstale":"2024-08-17T17:16:08.544540+0000","last_undegraded":"2024-08-17T17:16:08.544540+0000","last_fullsized":"2024-08-17T17:16:08.544540+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:17.662591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087198+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:15:24.087198+0000","last_peered":"2024-08-17T17:15:24.087198+0000","last_clean":"2024-08-17T17:15:24.087198+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:15:24.087198+0000","last_undegraded":"2024-08-17T17:15:24.087198+0000","last_fullsized":"2024-08-17T17:15:24.087198+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291264+0000","last_change":"2024-08-17T17:07:07.424546+0000","last_active":"2024-08-17T17:16:09.291264+0000","last_peered":"2024-08-17T17:16:09.291264+0000","last_clean":"2024-08-17T17:16:09.291264+0000","last_became_active":"2024-08-17T17:07:07.424158+0000","last_became_peered":"2024-08-17T17:07:07.424158+0000","last_unstale":"2024-08-17T17:16:09.291264+0000","last_undegraded":"2024-08-17T17:16:09.291264+0000","last_fullsized":"2024-08-17T17:16:09.291264+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:34.175159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292619+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:16:09.292619+0000","last_peered":"2024-08-17T17:16:09.292619+0000","last_clean":"2024-08-17T17:16:09.292619+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:16:09.292619+0000","last_undegraded":"2024-08-17T17:16:09.292619+0000","last_fullsized":"2024-08-17T17:16:09.292619+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545043+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:16:08.545043+0000","last_peered":"2024-08-17T17:16:08.545043+0000","last_clean":"2024-08-17T17:16:08.545043+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:16:08.545043+0000","last_undegraded":"2024-08-17T17:16:08.545043+0000","last_fullsized":"2024-08-17T17:16:08.545043+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291612+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:16:09.291612+0000","last_peered":"2024-08-17T17:16:09.291612+0000","last_clean":"2024-08-17T17:16:09.291612+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:16:09.291612+0000","last_undegraded":"2024-08-17T17:16:09.291612+0000","last_fullsized":"2024-08-17T17:16:09.291612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295589+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:16:09.295589+0000","last_peered":"2024-08-17T17:16:09.295589+0000","last_clean":"2024-08-17T17:16:09.295589+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:16:09.295589+0000","last_undegraded":"2024-08-17T17:16:09.295589+0000","last_fullsized":"2024-08-17T17:16:09.295589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128317+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:15:24.128317+0000","last_peered":"2024-08-17T17:15:24.128317+0000","last_clean":"2024-08-17T17:15:24.128317+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:15:24.128317+0000","last_undegraded":"2024-08-17T17:15:24.128317+0000","last_fullsized":"2024-08-17T17:15:24.128317+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129742+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T17:15:24.129742+0000","last_peered":"2024-08-17T17:15:24.129742+0000","last_clean":"2024-08-17T17:15:24.129742+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T17:15:24.129742+0000","last_undegraded":"2024-08-17T17:15:24.129742+0000","last_fullsized":"2024-08-17T17:15:24.129742+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130344+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:15:24.130344+0000","last_peered":"2024-08-17T17:15:24.130344+0000","last_clean":"2024-08-17T17:15:24.130344+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:15:24.130344+0000","last_undegraded":"2024-08-17T17:15:24.130344+0000","last_fullsized":"2024-08-17T17:15:24.130344+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":114,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130369+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:15:24.130369+0000","last_peered":"2024-08-17T17:15:24.130369+0000","last_clean":"2024-08-17T17:15:24.130369+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:15:24.130369+0000","last_undegraded":"2024-08-17T17:15:24.130369+0000","last_fullsized":"2024-08-17T17:15:24.130369+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545266+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:16:08.545266+0000","last_peered":"2024-08-17T17:16:08.545266+0000","last_clean":"2024-08-17T17:16:08.545266+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:16:08.545266+0000","last_undegraded":"2024-08-17T17:16:08.545266+0000","last_fullsized":"2024-08-17T17:16:08.545266+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087099+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:15:24.087099+0000","last_peered":"2024-08-17T17:15:24.087099+0000","last_clean":"2024-08-17T17:15:24.087099+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:15:24.087099+0000","last_undegraded":"2024-08-17T17:15:24.087099+0000","last_fullsized":"2024-08-17T17:15:24.087099+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086506+0000","last_change":"2024-08-17T17:07:51.484772+0000","last_active":"2024-08-17T17:15:24.086506+0000","last_peered":"2024-08-17T17:15:24.086506+0000","last_clean":"2024-08-17T17:15:24.086506+0000","last_became_active":"2024-08-17T17:07:51.484397+0000","last_became_peered":"2024-08-17T17:07:51.484397+0000","last_unstale":"2024-08-17T17:15:24.086506+0000","last_undegraded":"2024-08-17T17:15:24.086506+0000","last_fullsized":"2024-08-17T17:15:24.086506+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:21.701624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290999+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:16:09.290999+0000","last_peered":"2024-08-17T17:16:09.290999+0000","last_clean":"2024-08-17T17:16:09.290999+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:16:09.290999+0000","last_undegraded":"2024-08-17T17:16:09.290999+0000","last_fullsized":"2024-08-17T17:16:09.290999+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291726+0000","last_change":"2024-08-17T17:12:40.341757+0000","last_active":"2024-08-17T17:16:09.291726+0000","last_peered":"2024-08-17T17:16:09.291726+0000","last_clean":"2024-08-17T17:16:09.291726+0000","last_became_active":"2024-08-17T17:12:40.341446+0000","last_became_peered":"2024-08-17T17:12:40.341446+0000","last_unstale":"2024-08-17T17:16:09.291726+0000","last_undegraded":"2024-08-17T17:16:09.291726+0000","last_fullsized":"2024-08-17T17:16:09.291726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:51.178405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290988+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:16:09.290988+0000","last_peered":"2024-08-17T17:16:09.290988+0000","last_clean":"2024-08-17T17:16:09.290988+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:16:09.290988+0000","last_undegraded":"2024-08-17T17:16:09.290988+0000","last_fullsized":"2024-08-17T17:16:09.290988+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128292+0000","last_change":"2024-08-17T17:07:51.484291+0000","last_active":"2024-08-17T17:15:24.128292+0000","last_peered":"2024-08-17T17:15:24.128292+0000","last_clean":"2024-08-17T17:15:24.128292+0000","last_became_active":"2024-08-17T17:07:51.483903+0000","last_became_peered":"2024-08-17T17:07:51.483903+0000","last_unstale":"2024-08-17T17:15:24.128292+0000","last_undegraded":"2024-08-17T17:15:24.128292+0000","last_fullsized":"2024-08-17T17:15:24.128292+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:02.755382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291658+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:16:09.291658+0000","last_peered":"2024-08-17T17:16:09.291658+0000","last_clean":"2024-08-17T17:16:09.291658+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:16:09.291658+0000","last_undegraded":"2024-08-17T17:16:09.291658+0000","last_fullsized":"2024-08-17T17:16:09.291658+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295407+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:16:09.295407+0000","last_peered":"2024-08-17T17:16:09.295407+0000","last_clean":"2024-08-17T17:16:09.295407+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:16:09.295407+0000","last_undegraded":"2024-08-17T17:16:09.295407+0000","last_fullsized":"2024-08-17T17:16:09.295407+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544667+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:16:08.544667+0000","last_peered":"2024-08-17T17:16:08.544667+0000","last_clean":"2024-08-17T17:16:08.544667+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:16:08.544667+0000","last_undegraded":"2024-08-17T17:16:08.544667+0000","last_fullsized":"2024-08-17T17:16:08.544667+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291001+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:16:09.291001+0000","last_peered":"2024-08-17T17:16:09.291001+0000","last_clean":"2024-08-17T17:16:09.291001+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:16:09.291001+0000","last_undegraded":"2024-08-17T17:16:09.291001+0000","last_fullsized":"2024-08-17T17:16:09.291001+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129963+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:15:24.129963+0000","last_peered":"2024-08-17T17:15:24.129963+0000","last_clean":"2024-08-17T17:15:24.129963+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:15:24.129963+0000","last_undegraded":"2024-08-17T17:15:24.129963+0000","last_fullsized":"2024-08-17T17:15:24.129963+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545404+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:16:08.545404+0000","last_peered":"2024-08-17T17:16:08.545404+0000","last_clean":"2024-08-17T17:16:08.545404+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:16:08.545404+0000","last_undegraded":"2024-08-17T17:16:08.545404+0000","last_fullsized":"2024-08-17T17:16:08.545404+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291342+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:16:09.291342+0000","last_peered":"2024-08-17T17:16:09.291342+0000","last_clean":"2024-08-17T17:16:09.291342+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:16:09.291342+0000","last_undegraded":"2024-08-17T17:16:09.291342+0000","last_fullsized":"2024-08-17T17:16:09.291342+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130279+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:15:24.130279+0000","last_peered":"2024-08-17T17:15:24.130279+0000","last_clean":"2024-08-17T17:15:24.130279+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:15:24.130279+0000","last_undegraded":"2024-08-17T17:15:24.130279+0000","last_fullsized":"2024-08-17T17:15:24.130279+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086782+0000","last_change":"2024-08-17T17:07:51.482888+0000","last_active":"2024-08-17T17:15:24.086782+0000","last_peered":"2024-08-17T17:15:24.086782+0000","last_clean":"2024-08-17T17:15:24.086782+0000","last_became_active":"2024-08-17T17:07:51.482133+0000","last_became_peered":"2024-08-17T17:07:51.482133+0000","last_unstale":"2024-08-17T17:15:24.086782+0000","last_undegraded":"2024-08-17T17:15:24.086782+0000","last_fullsized":"2024-08-17T17:15:24.086782+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:47.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129880+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:15:24.129880+0000","last_peered":"2024-08-17T17:15:24.129880+0000","last_clean":"2024-08-17T17:15:24.129880+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:15:24.129880+0000","last_undegraded":"2024-08-17T17:15:24.129880+0000","last_fullsized":"2024-08-17T17:15:24.129880+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086698+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T17:15:24.086698+0000","last_peered":"2024-08-17T17:15:24.086698+0000","last_clean":"2024-08-17T17:15:24.086698+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T17:15:24.086698+0000","last_undegraded":"2024-08-17T17:15:24.086698+0000","last_fullsized":"2024-08-17T17:15:24.086698+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295973+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:16:09.295973+0000","last_peered":"2024-08-17T17:16:09.295973+0000","last_clean":"2024-08-17T17:16:09.295973+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:16:09.295973+0000","last_undegraded":"2024-08-17T17:16:09.295973+0000","last_fullsized":"2024-08-17T17:16:09.295973+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293888+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:16:09.293888+0000","last_peered":"2024-08-17T17:16:09.293888+0000","last_clean":"2024-08-17T17:16:09.293888+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:16:09.293888+0000","last_undegraded":"2024-08-17T17:16:09.293888+0000","last_fullsized":"2024-08-17T17:16:09.293888+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129570+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:15:24.129570+0000","last_peered":"2024-08-17T17:15:24.129570+0000","last_clean":"2024-08-17T17:15:24.129570+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:15:24.129570+0000","last_undegraded":"2024-08-17T17:15:24.129570+0000","last_fullsized":"2024-08-17T17:15:24.129570+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291343+0000","last_change":"2024-08-17T17:10:55.313325+0000","last_active":"2024-08-17T17:16:09.291343+0000","last_peered":"2024-08-17T17:16:09.291343+0000","last_clean":"2024-08-17T17:16:09.291343+0000","last_became_active":"2024-08-17T17:10:55.311737+0000","last_became_peered":"2024-08-17T17:10:55.311737+0000","last_unstale":"2024-08-17T17:16:09.291343+0000","last_undegraded":"2024-08-17T17:16:09.291343+0000","last_fullsized":"2024-08-17T17:16:09.291343+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:06.082766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13723,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201539,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299680,"kb_used_data":2324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445472,"statfs":{"total":95995035648,"available":95688163328,"internally_reserved":0,"allocated":2379776,"data_stored":1598688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463175,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312152,"kb_used_data":2316,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93433000,"statfs":{"total":95995035648,"available":95675392000,"internally_reserved":0,"allocated":2371584,"data_stored":1598545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724813,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903176,"kb_used_data":1724,"kb_used_omap":1,"kb_used_meta":901438,"kb_avail":92841976,"statfs":{"total":95995035648,"available":95070183424,"internally_reserved":0,"allocated":1765376,"data_stored":1003680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923072856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986449,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591636,"kb_used_data":1736,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":93153516,"statfs":{"total":95995035648,"available":95389200384,"internally_reserved":0,"allocated":1777664,"data_stored":1007977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225269,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210304,"kb_used_data":1716,"kb_used_omap":4,"kb_used_meta":208571,"kb_avail":93534848,"statfs":{"total":95995035648,"available":95779684352,"internally_reserved":0,"allocated":1757184,"data_stored":1003181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":213577152},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075594,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":225032,"kb_used_data":1724,"kb_used_omap":6,"kb_used_meta":223289,"kb_avail":93520120,"statfs":{"total":95995035648,"available":95764602880,"internally_reserved":0,"allocated":1765376,"data_stored":1003968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":228648379},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533486,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":364480,"kb_used_data":1716,"kb_used_omap":5,"kb_used_meta":362746,"kb_avail":93380672,"statfs":{"total":95995035648,"available":95621808128,"internally_reserved":0,"allocated":1757184,"data_stored":1001971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":371452144},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089463,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":370124,"kb_used_data":2304,"kb_used_omap":5,"kb_used_meta":367802,"kb_avail":93375028,"statfs":{"total":95995035648,"available":95616028672,"internally_reserved":0,"allocated":2359296,"data_stored":1594332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":376629307},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:16:58.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:16:58.284 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:59.120 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:58 smithi142 ceph-mon[106729]: from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:16:59.120 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:16:58 smithi142 ceph-mon[106729]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:59.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:58 smithi142 ceph-mon[103614]: from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:16:59.121 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:16:58 smithi142 ceph-mon[103614]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:16:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:58 smithi152 ceph-mon[93456]: from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:16:59.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:16:58 smithi152 ceph-mon[93456]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:00.190 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:17:00.191 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:17:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:17:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:17:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:17:00.845 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":485,"stamp":"2024-08-17T17:16:59.363929+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3276584,"kb_used_data":15560,"kb_used_omap":29,"kb_used_meta":3260898,"kb_avail":746684632,"statfs":{"total":767960285184,"available":764605063168,"internally_reserved":0,"allocated":15933440,"data_stored":9812342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30202,"internal_metadata":3339160070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002875"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293249+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:16:09.293249+0000","last_peered":"2024-08-17T17:16:09.293249+0000","last_clean":"2024-08-17T17:16:09.293249+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:16:09.293249+0000","last_undegraded":"2024-08-17T17:16:09.293249+0000","last_fullsized":"2024-08-17T17:16:09.293249+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086937+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:15:24.086937+0000","last_peered":"2024-08-17T17:15:24.086937+0000","last_clean":"2024-08-17T17:15:24.086937+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:15:24.086937+0000","last_undegraded":"2024-08-17T17:15:24.086937+0000","last_fullsized":"2024-08-17T17:15:24.086937+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086573+0000","last_change":"2024-08-17T17:12:40.335427+0000","last_active":"2024-08-17T17:15:24.086573+0000","last_peered":"2024-08-17T17:15:24.086573+0000","last_clean":"2024-08-17T17:15:24.086573+0000","last_became_active":"2024-08-17T17:12:40.334535+0000","last_became_peered":"2024-08-17T17:12:40.334535+0000","last_unstale":"2024-08-17T17:15:24.086573+0000","last_undegraded":"2024-08-17T17:15:24.086573+0000","last_fullsized":"2024-08-17T17:15:24.086573+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:47:39.810507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.299324+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:16:09.299324+0000","last_peered":"2024-08-17T17:16:09.299324+0000","last_clean":"2024-08-17T17:16:09.299324+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:16:09.299324+0000","last_undegraded":"2024-08-17T17:16:09.299324+0000","last_fullsized":"2024-08-17T17:16:09.299324+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295526+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:16:09.295526+0000","last_peered":"2024-08-17T17:16:09.295526+0000","last_clean":"2024-08-17T17:16:09.295526+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:16:09.295526+0000","last_undegraded":"2024-08-17T17:16:09.295526+0000","last_fullsized":"2024-08-17T17:16:09.295526+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086438+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:15:24.086438+0000","last_peered":"2024-08-17T17:15:24.086438+0000","last_clean":"2024-08-17T17:15:24.086438+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:15:24.086438+0000","last_undegraded":"2024-08-17T17:15:24.086438+0000","last_fullsized":"2024-08-17T17:15:24.086438+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292483+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:16:09.292483+0000","last_peered":"2024-08-17T17:16:09.292483+0000","last_clean":"2024-08-17T17:16:09.292483+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:16:09.292483+0000","last_undegraded":"2024-08-17T17:16:09.292483+0000","last_fullsized":"2024-08-17T17:16:09.292483+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127961+0000","last_change":"2024-08-17T17:07:07.418480+0000","last_active":"2024-08-17T17:15:24.127961+0000","last_peered":"2024-08-17T17:15:24.127961+0000","last_clean":"2024-08-17T17:15:24.127961+0000","last_became_active":"2024-08-17T17:07:07.418124+0000","last_became_peered":"2024-08-17T17:07:07.418124+0000","last_unstale":"2024-08-17T17:15:24.127961+0000","last_undegraded":"2024-08-17T17:15:24.127961+0000","last_fullsized":"2024-08-17T17:15:24.127961+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:12.020815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087282+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:15:24.087282+0000","last_peered":"2024-08-17T17:15:24.087282+0000","last_clean":"2024-08-17T17:15:24.087282+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:15:24.087282+0000","last_undegraded":"2024-08-17T17:15:24.087282+0000","last_fullsized":"2024-08-17T17:15:24.087282+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.088543+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T17:15:24.088543+0000","last_peered":"2024-08-17T17:15:24.088543+0000","last_clean":"2024-08-17T17:15:24.088543+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T17:15:24.088543+0000","last_undegraded":"2024-08-17T17:15:24.088543+0000","last_fullsized":"2024-08-17T17:15:24.088543+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087299+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:15:24.087299+0000","last_peered":"2024-08-17T17:15:24.087299+0000","last_clean":"2024-08-17T17:15:24.087299+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:15:24.087299+0000","last_undegraded":"2024-08-17T17:15:24.087299+0000","last_fullsized":"2024-08-17T17:15:24.087299+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129675+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:15:24.129675+0000","last_peered":"2024-08-17T17:15:24.129675+0000","last_clean":"2024-08-17T17:15:24.129675+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:15:24.129675+0000","last_undegraded":"2024-08-17T17:15:24.129675+0000","last_fullsized":"2024-08-17T17:15:24.129675+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087183+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T17:15:24.087183+0000","last_peered":"2024-08-17T17:15:24.087183+0000","last_clean":"2024-08-17T17:15:24.087183+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T17:15:24.087183+0000","last_undegraded":"2024-08-17T17:15:24.087183+0000","last_fullsized":"2024-08-17T17:15:24.087183+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127970+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:15:24.127970+0000","last_peered":"2024-08-17T17:15:24.127970+0000","last_clean":"2024-08-17T17:15:24.127970+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:15:24.127970+0000","last_undegraded":"2024-08-17T17:15:24.127970+0000","last_fullsized":"2024-08-17T17:15:24.127970+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086790+0000","last_change":"2024-08-17T17:12:40.336371+0000","last_active":"2024-08-17T17:15:24.086790+0000","last_peered":"2024-08-17T17:15:24.086790+0000","last_clean":"2024-08-17T17:15:24.086790+0000","last_became_active":"2024-08-17T17:12:40.335783+0000","last_became_peered":"2024-08-17T17:12:40.335783+0000","last_unstale":"2024-08-17T17:15:24.086790+0000","last_undegraded":"2024-08-17T17:15:24.086790+0000","last_fullsized":"2024-08-17T17:15:24.086790+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:14.313020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291175+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:16:09.291175+0000","last_peered":"2024-08-17T17:16:09.291175+0000","last_clean":"2024-08-17T17:16:09.291175+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:16:09.291175+0000","last_undegraded":"2024-08-17T17:16:09.291175+0000","last_fullsized":"2024-08-17T17:16:09.291175+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295720+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:16:09.295720+0000","last_peered":"2024-08-17T17:16:09.295720+0000","last_clean":"2024-08-17T17:16:09.295720+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:16:09.295720+0000","last_undegraded":"2024-08-17T17:16:09.295720+0000","last_fullsized":"2024-08-17T17:16:09.295720+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130241+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:15:24.130241+0000","last_peered":"2024-08-17T17:15:24.130241+0000","last_clean":"2024-08-17T17:15:24.130241+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:15:24.130241+0000","last_undegraded":"2024-08-17T17:15:24.130241+0000","last_fullsized":"2024-08-17T17:15:24.130241+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128095+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:15:24.128095+0000","last_peered":"2024-08-17T17:15:24.128095+0000","last_clean":"2024-08-17T17:15:24.128095+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:15:24.128095+0000","last_undegraded":"2024-08-17T17:15:24.128095+0000","last_fullsized":"2024-08-17T17:15:24.128095+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.133692+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:15:24.133692+0000","last_peered":"2024-08-17T17:15:24.133692+0000","last_clean":"2024-08-17T17:15:24.133692+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:15:24.133692+0000","last_undegraded":"2024-08-17T17:15:24.133692+0000","last_fullsized":"2024-08-17T17:15:24.133692+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129775+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:15:24.129775+0000","last_peered":"2024-08-17T17:15:24.129775+0000","last_clean":"2024-08-17T17:15:24.129775+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:15:24.129775+0000","last_undegraded":"2024-08-17T17:15:24.129775+0000","last_fullsized":"2024-08-17T17:15:24.129775+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130215+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:15:24.130215+0000","last_peered":"2024-08-17T17:15:24.130215+0000","last_clean":"2024-08-17T17:15:24.130215+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:15:24.130215+0000","last_undegraded":"2024-08-17T17:15:24.130215+0000","last_fullsized":"2024-08-17T17:15:24.130215+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293371+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:16:09.293371+0000","last_peered":"2024-08-17T17:16:09.293371+0000","last_clean":"2024-08-17T17:16:09.293371+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:16:09.293371+0000","last_undegraded":"2024-08-17T17:16:09.293371+0000","last_fullsized":"2024-08-17T17:16:09.293371+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086551+0000","last_change":"2024-08-17T17:12:40.335001+0000","last_active":"2024-08-17T17:15:24.086551+0000","last_peered":"2024-08-17T17:15:24.086551+0000","last_clean":"2024-08-17T17:15:24.086551+0000","last_became_active":"2024-08-17T17:12:40.334650+0000","last_became_peered":"2024-08-17T17:12:40.334650+0000","last_unstale":"2024-08-17T17:15:24.086551+0000","last_undegraded":"2024-08-17T17:15:24.086551+0000","last_fullsized":"2024-08-17T17:15:24.086551+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:06.876541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":449,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086423+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:15:24.086423+0000","last_peered":"2024-08-17T17:15:24.086423+0000","last_clean":"2024-08-17T17:15:24.086423+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:15:24.086423+0000","last_undegraded":"2024-08-17T17:15:24.086423+0000","last_fullsized":"2024-08-17T17:15:24.086423+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086615+0000","last_change":"2024-08-17T17:10:55.313434+0000","last_active":"2024-08-17T17:15:24.086615+0000","last_peered":"2024-08-17T17:15:24.086615+0000","last_clean":"2024-08-17T17:15:24.086615+0000","last_became_active":"2024-08-17T17:10:55.313091+0000","last_became_peered":"2024-08-17T17:10:55.313091+0000","last_unstale":"2024-08-17T17:15:24.086615+0000","last_undegraded":"2024-08-17T17:15:24.086615+0000","last_fullsized":"2024-08-17T17:15:24.086615+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:17.970517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293055+0000","last_change":"2024-08-17T17:12:40.332647+0000","last_active":"2024-08-17T17:16:09.293055+0000","last_peered":"2024-08-17T17:16:09.293055+0000","last_clean":"2024-08-17T17:16:09.293055+0000","last_became_active":"2024-08-17T17:12:40.332376+0000","last_became_peered":"2024-08-17T17:12:40.332376+0000","last_unstale":"2024-08-17T17:16:09.293055+0000","last_undegraded":"2024-08-17T17:16:09.293055+0000","last_fullsized":"2024-08-17T17:16:09.293055+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:21.624216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128216+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:15:24.128216+0000","last_peered":"2024-08-17T17:15:24.128216+0000","last_clean":"2024-08-17T17:15:24.128216+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:15:24.128216+0000","last_undegraded":"2024-08-17T17:15:24.128216+0000","last_fullsized":"2024-08-17T17:15:24.128216+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086495+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:15:24.086495+0000","last_peered":"2024-08-17T17:15:24.086495+0000","last_clean":"2024-08-17T17:15:24.086495+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:15:24.086495+0000","last_undegraded":"2024-08-17T17:15:24.086495+0000","last_fullsized":"2024-08-17T17:15:24.086495+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291546+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:16:09.291546+0000","last_peered":"2024-08-17T17:16:09.291546+0000","last_clean":"2024-08-17T17:16:09.291546+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:16:09.291546+0000","last_undegraded":"2024-08-17T17:16:09.291546+0000","last_fullsized":"2024-08-17T17:16:09.291546+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295310+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:16:09.295310+0000","last_peered":"2024-08-17T17:16:09.295310+0000","last_clean":"2024-08-17T17:16:09.295310+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:16:09.295310+0000","last_undegraded":"2024-08-17T17:16:09.295310+0000","last_fullsized":"2024-08-17T17:16:09.295310+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":150,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.542779+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:16:08.542779+0000","last_peered":"2024-08-17T17:16:08.542779+0000","last_clean":"2024-08-17T17:16:08.542779+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:16:08.542779+0000","last_undegraded":"2024-08-17T17:16:08.542779+0000","last_fullsized":"2024-08-17T17:16:08.542779+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.546939+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:16:08.546939+0000","last_peered":"2024-08-17T17:16:08.546939+0000","last_clean":"2024-08-17T17:16:08.546939+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:16:08.546939+0000","last_undegraded":"2024-08-17T17:16:08.546939+0000","last_fullsized":"2024-08-17T17:16:08.546939+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.557064+0000","last_change":"2024-08-17T17:07:51.480410+0000","last_active":"2024-08-17T17:16:08.557064+0000","last_peered":"2024-08-17T17:16:08.557064+0000","last_clean":"2024-08-17T17:16:08.557064+0000","last_became_active":"2024-08-17T17:07:51.480083+0000","last_became_peered":"2024-08-17T17:07:51.480083+0000","last_unstale":"2024-08-17T17:16:08.557064+0000","last_undegraded":"2024-08-17T17:16:08.557064+0000","last_fullsized":"2024-08-17T17:16:08.557064+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:39.376320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086589+0000","last_change":"2024-08-17T17:07:51.483538+0000","last_active":"2024-08-17T17:15:24.086589+0000","last_peered":"2024-08-17T17:15:24.086589+0000","last_clean":"2024-08-17T17:15:24.086589+0000","last_became_active":"2024-08-17T17:07:51.482970+0000","last_became_peered":"2024-08-17T17:07:51.482970+0000","last_unstale":"2024-08-17T17:15:24.086589+0000","last_undegraded":"2024-08-17T17:15:24.086589+0000","last_fullsized":"2024-08-17T17:15:24.086589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:37.985321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130448+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:15:24.130448+0000","last_peered":"2024-08-17T17:15:24.130448+0000","last_clean":"2024-08-17T17:15:24.130448+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:15:24.130448+0000","last_undegraded":"2024-08-17T17:15:24.130448+0000","last_fullsized":"2024-08-17T17:15:24.130448+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544455+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:16:08.544455+0000","last_peered":"2024-08-17T17:16:08.544455+0000","last_clean":"2024-08-17T17:16:08.544455+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:16:08.544455+0000","last_undegraded":"2024-08-17T17:16:08.544455+0000","last_fullsized":"2024-08-17T17:16:08.544455+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086721+0000","last_change":"2024-08-17T17:12:40.335196+0000","last_active":"2024-08-17T17:15:24.086721+0000","last_peered":"2024-08-17T17:15:24.086721+0000","last_clean":"2024-08-17T17:15:24.086721+0000","last_became_active":"2024-08-17T17:12:40.334917+0000","last_became_peered":"2024-08-17T17:12:40.334917+0000","last_unstale":"2024-08-17T17:15:24.086721+0000","last_undegraded":"2024-08-17T17:15:24.086721+0000","last_fullsized":"2024-08-17T17:15:24.086721+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:02.633323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":169,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544620+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:16:08.544620+0000","last_peered":"2024-08-17T17:16:08.544620+0000","last_clean":"2024-08-17T17:16:08.544620+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:16:08.544620+0000","last_undegraded":"2024-08-17T17:16:08.544620+0000","last_fullsized":"2024-08-17T17:16:08.544620+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545444+0000","last_change":"2024-08-17T17:07:51.489205+0000","last_active":"2024-08-17T17:16:08.545444+0000","last_peered":"2024-08-17T17:16:08.545444+0000","last_clean":"2024-08-17T17:16:08.545444+0000","last_became_active":"2024-08-17T17:07:51.488783+0000","last_became_peered":"2024-08-17T17:07:51.488783+0000","last_unstale":"2024-08-17T17:16:08.545444+0000","last_undegraded":"2024-08-17T17:16:08.545444+0000","last_fullsized":"2024-08-17T17:16:08.545444+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:22.190780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544540+0000","last_change":"2024-08-17T17:10:55.321862+0000","last_active":"2024-08-17T17:16:08.544540+0000","last_peered":"2024-08-17T17:16:08.544540+0000","last_clean":"2024-08-17T17:16:08.544540+0000","last_became_active":"2024-08-17T17:10:55.321339+0000","last_became_peered":"2024-08-17T17:10:55.321339+0000","last_unstale":"2024-08-17T17:16:08.544540+0000","last_undegraded":"2024-08-17T17:16:08.544540+0000","last_fullsized":"2024-08-17T17:16:08.544540+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:17.662591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087198+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:15:24.087198+0000","last_peered":"2024-08-17T17:15:24.087198+0000","last_clean":"2024-08-17T17:15:24.087198+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:15:24.087198+0000","last_undegraded":"2024-08-17T17:15:24.087198+0000","last_fullsized":"2024-08-17T17:15:24.087198+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291264+0000","last_change":"2024-08-17T17:07:07.424546+0000","last_active":"2024-08-17T17:16:09.291264+0000","last_peered":"2024-08-17T17:16:09.291264+0000","last_clean":"2024-08-17T17:16:09.291264+0000","last_became_active":"2024-08-17T17:07:07.424158+0000","last_became_peered":"2024-08-17T17:07:07.424158+0000","last_unstale":"2024-08-17T17:16:09.291264+0000","last_undegraded":"2024-08-17T17:16:09.291264+0000","last_fullsized":"2024-08-17T17:16:09.291264+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:34.175159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292665+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:16:09.292665+0000","last_peered":"2024-08-17T17:16:09.292665+0000","last_clean":"2024-08-17T17:16:09.292665+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:16:09.292665+0000","last_undegraded":"2024-08-17T17:16:09.292665+0000","last_fullsized":"2024-08-17T17:16:09.292665+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545043+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:16:08.545043+0000","last_peered":"2024-08-17T17:16:08.545043+0000","last_clean":"2024-08-17T17:16:08.545043+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:16:08.545043+0000","last_undegraded":"2024-08-17T17:16:08.545043+0000","last_fullsized":"2024-08-17T17:16:08.545043+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291612+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:16:09.291612+0000","last_peered":"2024-08-17T17:16:09.291612+0000","last_clean":"2024-08-17T17:16:09.291612+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:16:09.291612+0000","last_undegraded":"2024-08-17T17:16:09.291612+0000","last_fullsized":"2024-08-17T17:16:09.291612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295589+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:16:09.295589+0000","last_peered":"2024-08-17T17:16:09.295589+0000","last_clean":"2024-08-17T17:16:09.295589+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:16:09.295589+0000","last_undegraded":"2024-08-17T17:16:09.295589+0000","last_fullsized":"2024-08-17T17:16:09.295589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128317+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:15:24.128317+0000","last_peered":"2024-08-17T17:15:24.128317+0000","last_clean":"2024-08-17T17:15:24.128317+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:15:24.128317+0000","last_undegraded":"2024-08-17T17:15:24.128317+0000","last_fullsized":"2024-08-17T17:15:24.128317+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129742+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T17:15:24.129742+0000","last_peered":"2024-08-17T17:15:24.129742+0000","last_clean":"2024-08-17T17:15:24.129742+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T17:15:24.129742+0000","last_undegraded":"2024-08-17T17:15:24.129742+0000","last_fullsized":"2024-08-17T17:15:24.129742+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130344+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:15:24.130344+0000","last_peered":"2024-08-17T17:15:24.130344+0000","last_clean":"2024-08-17T17:15:24.130344+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:15:24.130344+0000","last_undegraded":"2024-08-17T17:15:24.130344+0000","last_fullsized":"2024-08-17T17:15:24.130344+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":114,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130369+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:15:24.130369+0000","last_peered":"2024-08-17T17:15:24.130369+0000","last_clean":"2024-08-17T17:15:24.130369+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:15:24.130369+0000","last_undegraded":"2024-08-17T17:15:24.130369+0000","last_fullsized":"2024-08-17T17:15:24.130369+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545266+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:16:08.545266+0000","last_peered":"2024-08-17T17:16:08.545266+0000","last_clean":"2024-08-17T17:16:08.545266+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:16:08.545266+0000","last_undegraded":"2024-08-17T17:16:08.545266+0000","last_fullsized":"2024-08-17T17:16:08.545266+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087099+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:15:24.087099+0000","last_peered":"2024-08-17T17:15:24.087099+0000","last_clean":"2024-08-17T17:15:24.087099+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:15:24.087099+0000","last_undegraded":"2024-08-17T17:15:24.087099+0000","last_fullsized":"2024-08-17T17:15:24.087099+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086506+0000","last_change":"2024-08-17T17:07:51.484772+0000","last_active":"2024-08-17T17:15:24.086506+0000","last_peered":"2024-08-17T17:15:24.086506+0000","last_clean":"2024-08-17T17:15:24.086506+0000","last_became_active":"2024-08-17T17:07:51.484397+0000","last_became_peered":"2024-08-17T17:07:51.484397+0000","last_unstale":"2024-08-17T17:15:24.086506+0000","last_undegraded":"2024-08-17T17:15:24.086506+0000","last_fullsized":"2024-08-17T17:15:24.086506+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:21.701624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290999+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:16:09.290999+0000","last_peered":"2024-08-17T17:16:09.290999+0000","last_clean":"2024-08-17T17:16:09.290999+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:16:09.290999+0000","last_undegraded":"2024-08-17T17:16:09.290999+0000","last_fullsized":"2024-08-17T17:16:09.290999+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291726+0000","last_change":"2024-08-17T17:12:40.341757+0000","last_active":"2024-08-17T17:16:09.291726+0000","last_peered":"2024-08-17T17:16:09.291726+0000","last_clean":"2024-08-17T17:16:09.291726+0000","last_became_active":"2024-08-17T17:12:40.341446+0000","last_became_peered":"2024-08-17T17:12:40.341446+0000","last_unstale":"2024-08-17T17:16:09.291726+0000","last_undegraded":"2024-08-17T17:16:09.291726+0000","last_fullsized":"2024-08-17T17:16:09.291726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:51.178405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290988+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:16:09.290988+0000","last_peered":"2024-08-17T17:16:09.290988+0000","last_clean":"2024-08-17T17:16:09.290988+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:16:09.290988+0000","last_undegraded":"2024-08-17T17:16:09.290988+0000","last_fullsized":"2024-08-17T17:16:09.290988+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128292+0000","last_change":"2024-08-17T17:07:51.484291+0000","last_active":"2024-08-17T17:15:24.128292+0000","last_peered":"2024-08-17T17:15:24.128292+0000","last_clean":"2024-08-17T17:15:24.128292+0000","last_became_active":"2024-08-17T17:07:51.483903+0000","last_became_peered":"2024-08-17T17:07:51.483903+0000","last_unstale":"2024-08-17T17:15:24.128292+0000","last_undegraded":"2024-08-17T17:15:24.128292+0000","last_fullsized":"2024-08-17T17:15:24.128292+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:02.755382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291658+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:16:09.291658+0000","last_peered":"2024-08-17T17:16:09.291658+0000","last_clean":"2024-08-17T17:16:09.291658+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:16:09.291658+0000","last_undegraded":"2024-08-17T17:16:09.291658+0000","last_fullsized":"2024-08-17T17:16:09.291658+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295407+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:16:09.295407+0000","last_peered":"2024-08-17T17:16:09.295407+0000","last_clean":"2024-08-17T17:16:09.295407+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:16:09.295407+0000","last_undegraded":"2024-08-17T17:16:09.295407+0000","last_fullsized":"2024-08-17T17:16:09.295407+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544667+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:16:08.544667+0000","last_peered":"2024-08-17T17:16:08.544667+0000","last_clean":"2024-08-17T17:16:08.544667+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:16:08.544667+0000","last_undegraded":"2024-08-17T17:16:08.544667+0000","last_fullsized":"2024-08-17T17:16:08.544667+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291001+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:16:09.291001+0000","last_peered":"2024-08-17T17:16:09.291001+0000","last_clean":"2024-08-17T17:16:09.291001+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:16:09.291001+0000","last_undegraded":"2024-08-17T17:16:09.291001+0000","last_fullsized":"2024-08-17T17:16:09.291001+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129963+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:15:24.129963+0000","last_peered":"2024-08-17T17:15:24.129963+0000","last_clean":"2024-08-17T17:15:24.129963+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:15:24.129963+0000","last_undegraded":"2024-08-17T17:15:24.129963+0000","last_fullsized":"2024-08-17T17:15:24.129963+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545404+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:16:08.545404+0000","last_peered":"2024-08-17T17:16:08.545404+0000","last_clean":"2024-08-17T17:16:08.545404+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:16:08.545404+0000","last_undegraded":"2024-08-17T17:16:08.545404+0000","last_fullsized":"2024-08-17T17:16:08.545404+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291342+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:16:09.291342+0000","last_peered":"2024-08-17T17:16:09.291342+0000","last_clean":"2024-08-17T17:16:09.291342+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:16:09.291342+0000","last_undegraded":"2024-08-17T17:16:09.291342+0000","last_fullsized":"2024-08-17T17:16:09.291342+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130279+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:15:24.130279+0000","last_peered":"2024-08-17T17:15:24.130279+0000","last_clean":"2024-08-17T17:15:24.130279+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:15:24.130279+0000","last_undegraded":"2024-08-17T17:15:24.130279+0000","last_fullsized":"2024-08-17T17:15:24.130279+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086782+0000","last_change":"2024-08-17T17:07:51.482888+0000","last_active":"2024-08-17T17:15:24.086782+0000","last_peered":"2024-08-17T17:15:24.086782+0000","last_clean":"2024-08-17T17:15:24.086782+0000","last_became_active":"2024-08-17T17:07:51.482133+0000","last_became_peered":"2024-08-17T17:07:51.482133+0000","last_unstale":"2024-08-17T17:15:24.086782+0000","last_undegraded":"2024-08-17T17:15:24.086782+0000","last_fullsized":"2024-08-17T17:15:24.086782+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:47.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129880+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:15:24.129880+0000","last_peered":"2024-08-17T17:15:24.129880+0000","last_clean":"2024-08-17T17:15:24.129880+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:15:24.129880+0000","last_undegraded":"2024-08-17T17:15:24.129880+0000","last_fullsized":"2024-08-17T17:15:24.129880+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086698+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T17:15:24.086698+0000","last_peered":"2024-08-17T17:15:24.086698+0000","last_clean":"2024-08-17T17:15:24.086698+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T17:15:24.086698+0000","last_undegraded":"2024-08-17T17:15:24.086698+0000","last_fullsized":"2024-08-17T17:15:24.086698+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295973+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:16:09.295973+0000","last_peered":"2024-08-17T17:16:09.295973+0000","last_clean":"2024-08-17T17:16:09.295973+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:16:09.295973+0000","last_undegraded":"2024-08-17T17:16:09.295973+0000","last_fullsized":"2024-08-17T17:16:09.295973+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293888+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:16:09.293888+0000","last_peered":"2024-08-17T17:16:09.293888+0000","last_clean":"2024-08-17T17:16:09.293888+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:16:09.293888+0000","last_undegraded":"2024-08-17T17:16:09.293888+0000","last_fullsized":"2024-08-17T17:16:09.293888+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129570+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:15:24.129570+0000","last_peered":"2024-08-17T17:15:24.129570+0000","last_clean":"2024-08-17T17:15:24.129570+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:15:24.129570+0000","last_undegraded":"2024-08-17T17:15:24.129570+0000","last_fullsized":"2024-08-17T17:15:24.129570+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291343+0000","last_change":"2024-08-17T17:10:55.313325+0000","last_active":"2024-08-17T17:16:09.291343+0000","last_peered":"2024-08-17T17:16:09.291343+0000","last_clean":"2024-08-17T17:16:09.291343+0000","last_became_active":"2024-08-17T17:10:55.311737+0000","last_became_peered":"2024-08-17T17:10:55.311737+0000","last_unstale":"2024-08-17T17:16:09.291343+0000","last_undegraded":"2024-08-17T17:16:09.291343+0000","last_fullsized":"2024-08-17T17:16:09.291343+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:06.082766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13723,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201540,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299680,"kb_used_data":2324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445472,"statfs":{"total":95995035648,"available":95688163328,"internally_reserved":0,"allocated":2379776,"data_stored":1598688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463176,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312152,"kb_used_data":2316,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93433000,"statfs":{"total":95995035648,"available":95675392000,"internally_reserved":0,"allocated":2371584,"data_stored":1598545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724814,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903176,"kb_used_data":1724,"kb_used_omap":1,"kb_used_meta":901438,"kb_avail":92841976,"statfs":{"total":95995035648,"available":95070183424,"internally_reserved":0,"allocated":1765376,"data_stored":1003680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923072856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986450,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591636,"kb_used_data":1736,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":93153516,"statfs":{"total":95995035648,"available":95389200384,"internally_reserved":0,"allocated":1777664,"data_stored":1007977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225270,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210304,"kb_used_data":1716,"kb_used_omap":4,"kb_used_meta":208571,"kb_avail":93534848,"statfs":{"total":95995035648,"available":95779684352,"internally_reserved":0,"allocated":1757184,"data_stored":1003181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":213577152},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075595,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":225032,"kb_used_data":1724,"kb_used_omap":6,"kb_used_meta":223289,"kb_avail":93520120,"statfs":{"total":95995035648,"available":95764602880,"internally_reserved":0,"allocated":1765376,"data_stored":1003968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":228648379},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533487,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":364480,"kb_used_data":1716,"kb_used_omap":5,"kb_used_meta":362746,"kb_avail":93380672,"statfs":{"total":95995035648,"available":95621808128,"internally_reserved":0,"allocated":1757184,"data_stored":1001971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":371452144},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089464,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":370124,"kb_used_data":2304,"kb_used_omap":5,"kb_used_meta":367802,"kb_avail":93375028,"statfs":{"total":95995035648,"available":95616028672,"internally_reserved":0,"allocated":2359296,"data_stored":1594332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":376629307},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:17:00.849 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:17:00.849 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:17:00.849 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:17:00.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:17:01.103 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:00 smithi142 ceph-mon[103614]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:00 smithi142 ceph-mon[106729]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:00 smithi152 ceph-mon[93456]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:02.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:01 smithi142 ceph-mon[106729]: from='client.35907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:01 smithi142 ceph-mon[103614]: from='client.35907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:02.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:17:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:17:01] "GET /metrics HTTP/1.1" 200 34007 "" "Prometheus/2.51.0" 2024-08-17T17:17:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:01 smithi152 ceph-mon[93456]: from='client.35907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:03.011 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:17:03.012 INFO:teuthology.orchestra.run.smithi142.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:17:03.308 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:02 smithi142 ceph-mon[106729]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:03.309 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:02 smithi142 ceph-mon[103614]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:03.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:02 smithi152 ceph-mon[93456]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:03.651 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:17:03.652 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:17:03.661 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-08-17T17:17:03.661 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T17:17:03.669 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T17:17:03.669 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:17:03.669 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-08-17T17:17:03.672 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:17:03.672 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '91f88e4d132ac08dc53355fa3d9717a53236a3a9'}} 2024-08-17T17:17:03.672 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:17:03.672 INFO:tasks.thrashosds:Beginning thrashosds... 2024-08-17T17:17:03.672 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:17:03.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:17:03.931 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:04.162 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1685291962' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:17:04.163 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1685291962' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:17:04.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1685291962' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:17:04.983 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:04 smithi142 ceph-mon[106729]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:04.983 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:04 smithi142 ceph-mon[103614]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:04 smithi152 ceph-mon[93456]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:05.699 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:17:05.700 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:17:06.425 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":488,"stamp":"2024-08-17T17:17:05.365357+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3276584,"kb_used_data":15560,"kb_used_omap":29,"kb_used_meta":3260898,"kb_avail":746684632,"statfs":{"total":767960285184,"available":764605063168,"internally_reserved":0,"allocated":15933440,"data_stored":9812342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30202,"internal_metadata":3339160070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002607"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293249+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:16:09.293249+0000","last_peered":"2024-08-17T17:16:09.293249+0000","last_clean":"2024-08-17T17:16:09.293249+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:16:09.293249+0000","last_undegraded":"2024-08-17T17:16:09.293249+0000","last_fullsized":"2024-08-17T17:16:09.293249+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086937+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:15:24.086937+0000","last_peered":"2024-08-17T17:15:24.086937+0000","last_clean":"2024-08-17T17:15:24.086937+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:15:24.086937+0000","last_undegraded":"2024-08-17T17:15:24.086937+0000","last_fullsized":"2024-08-17T17:15:24.086937+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086573+0000","last_change":"2024-08-17T17:12:40.335427+0000","last_active":"2024-08-17T17:15:24.086573+0000","last_peered":"2024-08-17T17:15:24.086573+0000","last_clean":"2024-08-17T17:15:24.086573+0000","last_became_active":"2024-08-17T17:12:40.334535+0000","last_became_peered":"2024-08-17T17:12:40.334535+0000","last_unstale":"2024-08-17T17:15:24.086573+0000","last_undegraded":"2024-08-17T17:15:24.086573+0000","last_fullsized":"2024-08-17T17:15:24.086573+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:47:39.810507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.299324+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:16:09.299324+0000","last_peered":"2024-08-17T17:16:09.299324+0000","last_clean":"2024-08-17T17:16:09.299324+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:16:09.299324+0000","last_undegraded":"2024-08-17T17:16:09.299324+0000","last_fullsized":"2024-08-17T17:16:09.299324+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295526+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:16:09.295526+0000","last_peered":"2024-08-17T17:16:09.295526+0000","last_clean":"2024-08-17T17:16:09.295526+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:16:09.295526+0000","last_undegraded":"2024-08-17T17:16:09.295526+0000","last_fullsized":"2024-08-17T17:16:09.295526+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086438+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:15:24.086438+0000","last_peered":"2024-08-17T17:15:24.086438+0000","last_clean":"2024-08-17T17:15:24.086438+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:15:24.086438+0000","last_undegraded":"2024-08-17T17:15:24.086438+0000","last_fullsized":"2024-08-17T17:15:24.086438+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292483+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:16:09.292483+0000","last_peered":"2024-08-17T17:16:09.292483+0000","last_clean":"2024-08-17T17:16:09.292483+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:16:09.292483+0000","last_undegraded":"2024-08-17T17:16:09.292483+0000","last_fullsized":"2024-08-17T17:16:09.292483+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127961+0000","last_change":"2024-08-17T17:07:07.418480+0000","last_active":"2024-08-17T17:15:24.127961+0000","last_peered":"2024-08-17T17:15:24.127961+0000","last_clean":"2024-08-17T17:15:24.127961+0000","last_became_active":"2024-08-17T17:07:07.418124+0000","last_became_peered":"2024-08-17T17:07:07.418124+0000","last_unstale":"2024-08-17T17:15:24.127961+0000","last_undegraded":"2024-08-17T17:15:24.127961+0000","last_fullsized":"2024-08-17T17:15:24.127961+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:12.020815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087282+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:15:24.087282+0000","last_peered":"2024-08-17T17:15:24.087282+0000","last_clean":"2024-08-17T17:15:24.087282+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:15:24.087282+0000","last_undegraded":"2024-08-17T17:15:24.087282+0000","last_fullsized":"2024-08-17T17:15:24.087282+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.088543+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T17:15:24.088543+0000","last_peered":"2024-08-17T17:15:24.088543+0000","last_clean":"2024-08-17T17:15:24.088543+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T17:15:24.088543+0000","last_undegraded":"2024-08-17T17:15:24.088543+0000","last_fullsized":"2024-08-17T17:15:24.088543+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087299+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:15:24.087299+0000","last_peered":"2024-08-17T17:15:24.087299+0000","last_clean":"2024-08-17T17:15:24.087299+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:15:24.087299+0000","last_undegraded":"2024-08-17T17:15:24.087299+0000","last_fullsized":"2024-08-17T17:15:24.087299+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129675+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:15:24.129675+0000","last_peered":"2024-08-17T17:15:24.129675+0000","last_clean":"2024-08-17T17:15:24.129675+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:15:24.129675+0000","last_undegraded":"2024-08-17T17:15:24.129675+0000","last_fullsized":"2024-08-17T17:15:24.129675+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087183+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T17:15:24.087183+0000","last_peered":"2024-08-17T17:15:24.087183+0000","last_clean":"2024-08-17T17:15:24.087183+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T17:15:24.087183+0000","last_undegraded":"2024-08-17T17:15:24.087183+0000","last_fullsized":"2024-08-17T17:15:24.087183+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127970+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:15:24.127970+0000","last_peered":"2024-08-17T17:15:24.127970+0000","last_clean":"2024-08-17T17:15:24.127970+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:15:24.127970+0000","last_undegraded":"2024-08-17T17:15:24.127970+0000","last_fullsized":"2024-08-17T17:15:24.127970+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086790+0000","last_change":"2024-08-17T17:12:40.336371+0000","last_active":"2024-08-17T17:15:24.086790+0000","last_peered":"2024-08-17T17:15:24.086790+0000","last_clean":"2024-08-17T17:15:24.086790+0000","last_became_active":"2024-08-17T17:12:40.335783+0000","last_became_peered":"2024-08-17T17:12:40.335783+0000","last_unstale":"2024-08-17T17:15:24.086790+0000","last_undegraded":"2024-08-17T17:15:24.086790+0000","last_fullsized":"2024-08-17T17:15:24.086790+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:14.313020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291175+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:16:09.291175+0000","last_peered":"2024-08-17T17:16:09.291175+0000","last_clean":"2024-08-17T17:16:09.291175+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:16:09.291175+0000","last_undegraded":"2024-08-17T17:16:09.291175+0000","last_fullsized":"2024-08-17T17:16:09.291175+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295720+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:16:09.295720+0000","last_peered":"2024-08-17T17:16:09.295720+0000","last_clean":"2024-08-17T17:16:09.295720+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:16:09.295720+0000","last_undegraded":"2024-08-17T17:16:09.295720+0000","last_fullsized":"2024-08-17T17:16:09.295720+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130241+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:15:24.130241+0000","last_peered":"2024-08-17T17:15:24.130241+0000","last_clean":"2024-08-17T17:15:24.130241+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:15:24.130241+0000","last_undegraded":"2024-08-17T17:15:24.130241+0000","last_fullsized":"2024-08-17T17:15:24.130241+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128095+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:15:24.128095+0000","last_peered":"2024-08-17T17:15:24.128095+0000","last_clean":"2024-08-17T17:15:24.128095+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:15:24.128095+0000","last_undegraded":"2024-08-17T17:15:24.128095+0000","last_fullsized":"2024-08-17T17:15:24.128095+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.133692+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:15:24.133692+0000","last_peered":"2024-08-17T17:15:24.133692+0000","last_clean":"2024-08-17T17:15:24.133692+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:15:24.133692+0000","last_undegraded":"2024-08-17T17:15:24.133692+0000","last_fullsized":"2024-08-17T17:15:24.133692+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129775+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:15:24.129775+0000","last_peered":"2024-08-17T17:15:24.129775+0000","last_clean":"2024-08-17T17:15:24.129775+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:15:24.129775+0000","last_undegraded":"2024-08-17T17:15:24.129775+0000","last_fullsized":"2024-08-17T17:15:24.129775+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130215+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:15:24.130215+0000","last_peered":"2024-08-17T17:15:24.130215+0000","last_clean":"2024-08-17T17:15:24.130215+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:15:24.130215+0000","last_undegraded":"2024-08-17T17:15:24.130215+0000","last_fullsized":"2024-08-17T17:15:24.130215+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293371+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:16:09.293371+0000","last_peered":"2024-08-17T17:16:09.293371+0000","last_clean":"2024-08-17T17:16:09.293371+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:16:09.293371+0000","last_undegraded":"2024-08-17T17:16:09.293371+0000","last_fullsized":"2024-08-17T17:16:09.293371+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086551+0000","last_change":"2024-08-17T17:12:40.335001+0000","last_active":"2024-08-17T17:15:24.086551+0000","last_peered":"2024-08-17T17:15:24.086551+0000","last_clean":"2024-08-17T17:15:24.086551+0000","last_became_active":"2024-08-17T17:12:40.334650+0000","last_became_peered":"2024-08-17T17:12:40.334650+0000","last_unstale":"2024-08-17T17:15:24.086551+0000","last_undegraded":"2024-08-17T17:15:24.086551+0000","last_fullsized":"2024-08-17T17:15:24.086551+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:06.876541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":449,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086423+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:15:24.086423+0000","last_peered":"2024-08-17T17:15:24.086423+0000","last_clean":"2024-08-17T17:15:24.086423+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:15:24.086423+0000","last_undegraded":"2024-08-17T17:15:24.086423+0000","last_fullsized":"2024-08-17T17:15:24.086423+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086615+0000","last_change":"2024-08-17T17:10:55.313434+0000","last_active":"2024-08-17T17:15:24.086615+0000","last_peered":"2024-08-17T17:15:24.086615+0000","last_clean":"2024-08-17T17:15:24.086615+0000","last_became_active":"2024-08-17T17:10:55.313091+0000","last_became_peered":"2024-08-17T17:10:55.313091+0000","last_unstale":"2024-08-17T17:15:24.086615+0000","last_undegraded":"2024-08-17T17:15:24.086615+0000","last_fullsized":"2024-08-17T17:15:24.086615+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:17.970517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293055+0000","last_change":"2024-08-17T17:12:40.332647+0000","last_active":"2024-08-17T17:16:09.293055+0000","last_peered":"2024-08-17T17:16:09.293055+0000","last_clean":"2024-08-17T17:16:09.293055+0000","last_became_active":"2024-08-17T17:12:40.332376+0000","last_became_peered":"2024-08-17T17:12:40.332376+0000","last_unstale":"2024-08-17T17:16:09.293055+0000","last_undegraded":"2024-08-17T17:16:09.293055+0000","last_fullsized":"2024-08-17T17:16:09.293055+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:21.624216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128216+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:15:24.128216+0000","last_peered":"2024-08-17T17:15:24.128216+0000","last_clean":"2024-08-17T17:15:24.128216+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:15:24.128216+0000","last_undegraded":"2024-08-17T17:15:24.128216+0000","last_fullsized":"2024-08-17T17:15:24.128216+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086495+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:15:24.086495+0000","last_peered":"2024-08-17T17:15:24.086495+0000","last_clean":"2024-08-17T17:15:24.086495+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:15:24.086495+0000","last_undegraded":"2024-08-17T17:15:24.086495+0000","last_fullsized":"2024-08-17T17:15:24.086495+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291546+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:16:09.291546+0000","last_peered":"2024-08-17T17:16:09.291546+0000","last_clean":"2024-08-17T17:16:09.291546+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:16:09.291546+0000","last_undegraded":"2024-08-17T17:16:09.291546+0000","last_fullsized":"2024-08-17T17:16:09.291546+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295310+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:16:09.295310+0000","last_peered":"2024-08-17T17:16:09.295310+0000","last_clean":"2024-08-17T17:16:09.295310+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:16:09.295310+0000","last_undegraded":"2024-08-17T17:16:09.295310+0000","last_fullsized":"2024-08-17T17:16:09.295310+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":150,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.542779+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:16:08.542779+0000","last_peered":"2024-08-17T17:16:08.542779+0000","last_clean":"2024-08-17T17:16:08.542779+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:16:08.542779+0000","last_undegraded":"2024-08-17T17:16:08.542779+0000","last_fullsized":"2024-08-17T17:16:08.542779+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.546939+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:16:08.546939+0000","last_peered":"2024-08-17T17:16:08.546939+0000","last_clean":"2024-08-17T17:16:08.546939+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:16:08.546939+0000","last_undegraded":"2024-08-17T17:16:08.546939+0000","last_fullsized":"2024-08-17T17:16:08.546939+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.557064+0000","last_change":"2024-08-17T17:07:51.480410+0000","last_active":"2024-08-17T17:16:08.557064+0000","last_peered":"2024-08-17T17:16:08.557064+0000","last_clean":"2024-08-17T17:16:08.557064+0000","last_became_active":"2024-08-17T17:07:51.480083+0000","last_became_peered":"2024-08-17T17:07:51.480083+0000","last_unstale":"2024-08-17T17:16:08.557064+0000","last_undegraded":"2024-08-17T17:16:08.557064+0000","last_fullsized":"2024-08-17T17:16:08.557064+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:39.376320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086589+0000","last_change":"2024-08-17T17:07:51.483538+0000","last_active":"2024-08-17T17:15:24.086589+0000","last_peered":"2024-08-17T17:15:24.086589+0000","last_clean":"2024-08-17T17:15:24.086589+0000","last_became_active":"2024-08-17T17:07:51.482970+0000","last_became_peered":"2024-08-17T17:07:51.482970+0000","last_unstale":"2024-08-17T17:15:24.086589+0000","last_undegraded":"2024-08-17T17:15:24.086589+0000","last_fullsized":"2024-08-17T17:15:24.086589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:37.985321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130448+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:15:24.130448+0000","last_peered":"2024-08-17T17:15:24.130448+0000","last_clean":"2024-08-17T17:15:24.130448+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:15:24.130448+0000","last_undegraded":"2024-08-17T17:15:24.130448+0000","last_fullsized":"2024-08-17T17:15:24.130448+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544455+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:16:08.544455+0000","last_peered":"2024-08-17T17:16:08.544455+0000","last_clean":"2024-08-17T17:16:08.544455+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:16:08.544455+0000","last_undegraded":"2024-08-17T17:16:08.544455+0000","last_fullsized":"2024-08-17T17:16:08.544455+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086721+0000","last_change":"2024-08-17T17:12:40.335196+0000","last_active":"2024-08-17T17:15:24.086721+0000","last_peered":"2024-08-17T17:15:24.086721+0000","last_clean":"2024-08-17T17:15:24.086721+0000","last_became_active":"2024-08-17T17:12:40.334917+0000","last_became_peered":"2024-08-17T17:12:40.334917+0000","last_unstale":"2024-08-17T17:15:24.086721+0000","last_undegraded":"2024-08-17T17:15:24.086721+0000","last_fullsized":"2024-08-17T17:15:24.086721+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:02.633323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":169,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544620+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:16:08.544620+0000","last_peered":"2024-08-17T17:16:08.544620+0000","last_clean":"2024-08-17T17:16:08.544620+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:16:08.544620+0000","last_undegraded":"2024-08-17T17:16:08.544620+0000","last_fullsized":"2024-08-17T17:16:08.544620+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545444+0000","last_change":"2024-08-17T17:07:51.489205+0000","last_active":"2024-08-17T17:16:08.545444+0000","last_peered":"2024-08-17T17:16:08.545444+0000","last_clean":"2024-08-17T17:16:08.545444+0000","last_became_active":"2024-08-17T17:07:51.488783+0000","last_became_peered":"2024-08-17T17:07:51.488783+0000","last_unstale":"2024-08-17T17:16:08.545444+0000","last_undegraded":"2024-08-17T17:16:08.545444+0000","last_fullsized":"2024-08-17T17:16:08.545444+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:22.190780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544540+0000","last_change":"2024-08-17T17:10:55.321862+0000","last_active":"2024-08-17T17:16:08.544540+0000","last_peered":"2024-08-17T17:16:08.544540+0000","last_clean":"2024-08-17T17:16:08.544540+0000","last_became_active":"2024-08-17T17:10:55.321339+0000","last_became_peered":"2024-08-17T17:10:55.321339+0000","last_unstale":"2024-08-17T17:16:08.544540+0000","last_undegraded":"2024-08-17T17:16:08.544540+0000","last_fullsized":"2024-08-17T17:16:08.544540+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:17.662591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087198+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:15:24.087198+0000","last_peered":"2024-08-17T17:15:24.087198+0000","last_clean":"2024-08-17T17:15:24.087198+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:15:24.087198+0000","last_undegraded":"2024-08-17T17:15:24.087198+0000","last_fullsized":"2024-08-17T17:15:24.087198+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291264+0000","last_change":"2024-08-17T17:07:07.424546+0000","last_active":"2024-08-17T17:16:09.291264+0000","last_peered":"2024-08-17T17:16:09.291264+0000","last_clean":"2024-08-17T17:16:09.291264+0000","last_became_active":"2024-08-17T17:07:07.424158+0000","last_became_peered":"2024-08-17T17:07:07.424158+0000","last_unstale":"2024-08-17T17:16:09.291264+0000","last_undegraded":"2024-08-17T17:16:09.291264+0000","last_fullsized":"2024-08-17T17:16:09.291264+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:34.175159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292665+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:16:09.292665+0000","last_peered":"2024-08-17T17:16:09.292665+0000","last_clean":"2024-08-17T17:16:09.292665+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:16:09.292665+0000","last_undegraded":"2024-08-17T17:16:09.292665+0000","last_fullsized":"2024-08-17T17:16:09.292665+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545043+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:16:08.545043+0000","last_peered":"2024-08-17T17:16:08.545043+0000","last_clean":"2024-08-17T17:16:08.545043+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:16:08.545043+0000","last_undegraded":"2024-08-17T17:16:08.545043+0000","last_fullsized":"2024-08-17T17:16:08.545043+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291612+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:16:09.291612+0000","last_peered":"2024-08-17T17:16:09.291612+0000","last_clean":"2024-08-17T17:16:09.291612+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:16:09.291612+0000","last_undegraded":"2024-08-17T17:16:09.291612+0000","last_fullsized":"2024-08-17T17:16:09.291612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295589+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:16:09.295589+0000","last_peered":"2024-08-17T17:16:09.295589+0000","last_clean":"2024-08-17T17:16:09.295589+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:16:09.295589+0000","last_undegraded":"2024-08-17T17:16:09.295589+0000","last_fullsized":"2024-08-17T17:16:09.295589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128317+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:15:24.128317+0000","last_peered":"2024-08-17T17:15:24.128317+0000","last_clean":"2024-08-17T17:15:24.128317+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:15:24.128317+0000","last_undegraded":"2024-08-17T17:15:24.128317+0000","last_fullsized":"2024-08-17T17:15:24.128317+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129742+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T17:15:24.129742+0000","last_peered":"2024-08-17T17:15:24.129742+0000","last_clean":"2024-08-17T17:15:24.129742+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T17:15:24.129742+0000","last_undegraded":"2024-08-17T17:15:24.129742+0000","last_fullsized":"2024-08-17T17:15:24.129742+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130344+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:15:24.130344+0000","last_peered":"2024-08-17T17:15:24.130344+0000","last_clean":"2024-08-17T17:15:24.130344+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:15:24.130344+0000","last_undegraded":"2024-08-17T17:15:24.130344+0000","last_fullsized":"2024-08-17T17:15:24.130344+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":114,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130369+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:15:24.130369+0000","last_peered":"2024-08-17T17:15:24.130369+0000","last_clean":"2024-08-17T17:15:24.130369+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:15:24.130369+0000","last_undegraded":"2024-08-17T17:15:24.130369+0000","last_fullsized":"2024-08-17T17:15:24.130369+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545266+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:16:08.545266+0000","last_peered":"2024-08-17T17:16:08.545266+0000","last_clean":"2024-08-17T17:16:08.545266+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:16:08.545266+0000","last_undegraded":"2024-08-17T17:16:08.545266+0000","last_fullsized":"2024-08-17T17:16:08.545266+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087099+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:15:24.087099+0000","last_peered":"2024-08-17T17:15:24.087099+0000","last_clean":"2024-08-17T17:15:24.087099+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:15:24.087099+0000","last_undegraded":"2024-08-17T17:15:24.087099+0000","last_fullsized":"2024-08-17T17:15:24.087099+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086506+0000","last_change":"2024-08-17T17:07:51.484772+0000","last_active":"2024-08-17T17:15:24.086506+0000","last_peered":"2024-08-17T17:15:24.086506+0000","last_clean":"2024-08-17T17:15:24.086506+0000","last_became_active":"2024-08-17T17:07:51.484397+0000","last_became_peered":"2024-08-17T17:07:51.484397+0000","last_unstale":"2024-08-17T17:15:24.086506+0000","last_undegraded":"2024-08-17T17:15:24.086506+0000","last_fullsized":"2024-08-17T17:15:24.086506+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:21.701624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290999+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:16:09.290999+0000","last_peered":"2024-08-17T17:16:09.290999+0000","last_clean":"2024-08-17T17:16:09.290999+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:16:09.290999+0000","last_undegraded":"2024-08-17T17:16:09.290999+0000","last_fullsized":"2024-08-17T17:16:09.290999+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291726+0000","last_change":"2024-08-17T17:12:40.341757+0000","last_active":"2024-08-17T17:16:09.291726+0000","last_peered":"2024-08-17T17:16:09.291726+0000","last_clean":"2024-08-17T17:16:09.291726+0000","last_became_active":"2024-08-17T17:12:40.341446+0000","last_became_peered":"2024-08-17T17:12:40.341446+0000","last_unstale":"2024-08-17T17:16:09.291726+0000","last_undegraded":"2024-08-17T17:16:09.291726+0000","last_fullsized":"2024-08-17T17:16:09.291726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:51.178405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290988+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:16:09.290988+0000","last_peered":"2024-08-17T17:16:09.290988+0000","last_clean":"2024-08-17T17:16:09.290988+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:16:09.290988+0000","last_undegraded":"2024-08-17T17:16:09.290988+0000","last_fullsized":"2024-08-17T17:16:09.290988+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128292+0000","last_change":"2024-08-17T17:07:51.484291+0000","last_active":"2024-08-17T17:15:24.128292+0000","last_peered":"2024-08-17T17:15:24.128292+0000","last_clean":"2024-08-17T17:15:24.128292+0000","last_became_active":"2024-08-17T17:07:51.483903+0000","last_became_peered":"2024-08-17T17:07:51.483903+0000","last_unstale":"2024-08-17T17:15:24.128292+0000","last_undegraded":"2024-08-17T17:15:24.128292+0000","last_fullsized":"2024-08-17T17:15:24.128292+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:02.755382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291658+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:16:09.291658+0000","last_peered":"2024-08-17T17:16:09.291658+0000","last_clean":"2024-08-17T17:16:09.291658+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:16:09.291658+0000","last_undegraded":"2024-08-17T17:16:09.291658+0000","last_fullsized":"2024-08-17T17:16:09.291658+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295407+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:16:09.295407+0000","last_peered":"2024-08-17T17:16:09.295407+0000","last_clean":"2024-08-17T17:16:09.295407+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:16:09.295407+0000","last_undegraded":"2024-08-17T17:16:09.295407+0000","last_fullsized":"2024-08-17T17:16:09.295407+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544667+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:16:08.544667+0000","last_peered":"2024-08-17T17:16:08.544667+0000","last_clean":"2024-08-17T17:16:08.544667+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:16:08.544667+0000","last_undegraded":"2024-08-17T17:16:08.544667+0000","last_fullsized":"2024-08-17T17:16:08.544667+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291001+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:16:09.291001+0000","last_peered":"2024-08-17T17:16:09.291001+0000","last_clean":"2024-08-17T17:16:09.291001+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:16:09.291001+0000","last_undegraded":"2024-08-17T17:16:09.291001+0000","last_fullsized":"2024-08-17T17:16:09.291001+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129963+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:15:24.129963+0000","last_peered":"2024-08-17T17:15:24.129963+0000","last_clean":"2024-08-17T17:15:24.129963+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:15:24.129963+0000","last_undegraded":"2024-08-17T17:15:24.129963+0000","last_fullsized":"2024-08-17T17:15:24.129963+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545404+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:16:08.545404+0000","last_peered":"2024-08-17T17:16:08.545404+0000","last_clean":"2024-08-17T17:16:08.545404+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:16:08.545404+0000","last_undegraded":"2024-08-17T17:16:08.545404+0000","last_fullsized":"2024-08-17T17:16:08.545404+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291342+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:16:09.291342+0000","last_peered":"2024-08-17T17:16:09.291342+0000","last_clean":"2024-08-17T17:16:09.291342+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:16:09.291342+0000","last_undegraded":"2024-08-17T17:16:09.291342+0000","last_fullsized":"2024-08-17T17:16:09.291342+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130279+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:15:24.130279+0000","last_peered":"2024-08-17T17:15:24.130279+0000","last_clean":"2024-08-17T17:15:24.130279+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:15:24.130279+0000","last_undegraded":"2024-08-17T17:15:24.130279+0000","last_fullsized":"2024-08-17T17:15:24.130279+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086782+0000","last_change":"2024-08-17T17:07:51.482888+0000","last_active":"2024-08-17T17:15:24.086782+0000","last_peered":"2024-08-17T17:15:24.086782+0000","last_clean":"2024-08-17T17:15:24.086782+0000","last_became_active":"2024-08-17T17:07:51.482133+0000","last_became_peered":"2024-08-17T17:07:51.482133+0000","last_unstale":"2024-08-17T17:15:24.086782+0000","last_undegraded":"2024-08-17T17:15:24.086782+0000","last_fullsized":"2024-08-17T17:15:24.086782+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:47.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129880+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:15:24.129880+0000","last_peered":"2024-08-17T17:15:24.129880+0000","last_clean":"2024-08-17T17:15:24.129880+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:15:24.129880+0000","last_undegraded":"2024-08-17T17:15:24.129880+0000","last_fullsized":"2024-08-17T17:15:24.129880+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086698+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T17:15:24.086698+0000","last_peered":"2024-08-17T17:15:24.086698+0000","last_clean":"2024-08-17T17:15:24.086698+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T17:15:24.086698+0000","last_undegraded":"2024-08-17T17:15:24.086698+0000","last_fullsized":"2024-08-17T17:15:24.086698+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295973+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:16:09.295973+0000","last_peered":"2024-08-17T17:16:09.295973+0000","last_clean":"2024-08-17T17:16:09.295973+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:16:09.295973+0000","last_undegraded":"2024-08-17T17:16:09.295973+0000","last_fullsized":"2024-08-17T17:16:09.295973+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293888+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:16:09.293888+0000","last_peered":"2024-08-17T17:16:09.293888+0000","last_clean":"2024-08-17T17:16:09.293888+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:16:09.293888+0000","last_undegraded":"2024-08-17T17:16:09.293888+0000","last_fullsized":"2024-08-17T17:16:09.293888+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129570+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:15:24.129570+0000","last_peered":"2024-08-17T17:15:24.129570+0000","last_clean":"2024-08-17T17:15:24.129570+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:15:24.129570+0000","last_undegraded":"2024-08-17T17:15:24.129570+0000","last_fullsized":"2024-08-17T17:15:24.129570+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291343+0000","last_change":"2024-08-17T17:10:55.313325+0000","last_active":"2024-08-17T17:16:09.291343+0000","last_peered":"2024-08-17T17:16:09.291343+0000","last_clean":"2024-08-17T17:16:09.291343+0000","last_became_active":"2024-08-17T17:10:55.311737+0000","last_became_peered":"2024-08-17T17:10:55.311737+0000","last_unstale":"2024-08-17T17:16:09.291343+0000","last_undegraded":"2024-08-17T17:16:09.291343+0000","last_fullsized":"2024-08-17T17:16:09.291343+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:06.082766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13723,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299680,"kb_used_data":2324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445472,"statfs":{"total":95995035648,"available":95688163328,"internally_reserved":0,"allocated":2379776,"data_stored":1598688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463177,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312152,"kb_used_data":2316,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93433000,"statfs":{"total":95995035648,"available":95675392000,"internally_reserved":0,"allocated":2371584,"data_stored":1598545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724815,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903176,"kb_used_data":1724,"kb_used_omap":1,"kb_used_meta":901438,"kb_avail":92841976,"statfs":{"total":95995035648,"available":95070183424,"internally_reserved":0,"allocated":1765376,"data_stored":1003680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923072856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986451,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591636,"kb_used_data":1736,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":93153516,"statfs":{"total":95995035648,"available":95389200384,"internally_reserved":0,"allocated":1777664,"data_stored":1007977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225271,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210304,"kb_used_data":1716,"kb_used_omap":4,"kb_used_meta":208571,"kb_avail":93534848,"statfs":{"total":95995035648,"available":95779684352,"internally_reserved":0,"allocated":1757184,"data_stored":1003181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":213577152},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075596,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":225032,"kb_used_data":1724,"kb_used_omap":6,"kb_used_meta":223289,"kb_avail":93520120,"statfs":{"total":95995035648,"available":95764602880,"internally_reserved":0,"allocated":1765376,"data_stored":1003968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":228648379},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533488,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":364480,"kb_used_data":1716,"kb_used_omap":5,"kb_used_meta":362746,"kb_avail":93380672,"statfs":{"total":95995035648,"available":95621808128,"internally_reserved":0,"allocated":1757184,"data_stored":1001971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":371452144},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089465,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":370124,"kb_used_data":2304,"kb_used_omap":5,"kb_used_meta":367802,"kb_avail":93375028,"statfs":{"total":95995035648,"available":95616028672,"internally_reserved":0,"allocated":2359296,"data_stored":1594332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":376629307},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:17:06.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:17:06.685 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:07.101 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:06 smithi142 ceph-mon[103614]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:07.102 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:06 smithi142 ceph-mon[103614]: from='client.45731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:07.102 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:06 smithi142 ceph-mon[106729]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:07.102 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:06 smithi142 ceph-mon[106729]: from='client.45731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:06 smithi152 ceph-mon[93456]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:07.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:06 smithi152 ceph-mon[93456]: from='client.45731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:08.749 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:17:08.750 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:17:09.051 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:08 smithi142 ceph-mon[106729]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:09.052 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:08 smithi142 ceph-mon[103614]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:09.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:08 smithi152 ceph-mon[93456]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:09.352 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2024-08-17T17:17:07.365922+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3276584,"kb_used_data":15560,"kb_used_omap":29,"kb_used_meta":3260898,"kb_avail":746684632,"statfs":{"total":767960285184,"available":764605063168,"internally_reserved":0,"allocated":15933440,"data_stored":9812342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30202,"internal_metadata":3339160070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002770"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293249+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:16:09.293249+0000","last_peered":"2024-08-17T17:16:09.293249+0000","last_clean":"2024-08-17T17:16:09.293249+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:16:09.293249+0000","last_undegraded":"2024-08-17T17:16:09.293249+0000","last_fullsized":"2024-08-17T17:16:09.293249+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086937+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:15:24.086937+0000","last_peered":"2024-08-17T17:15:24.086937+0000","last_clean":"2024-08-17T17:15:24.086937+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:15:24.086937+0000","last_undegraded":"2024-08-17T17:15:24.086937+0000","last_fullsized":"2024-08-17T17:15:24.086937+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086573+0000","last_change":"2024-08-17T17:12:40.335427+0000","last_active":"2024-08-17T17:15:24.086573+0000","last_peered":"2024-08-17T17:15:24.086573+0000","last_clean":"2024-08-17T17:15:24.086573+0000","last_became_active":"2024-08-17T17:12:40.334535+0000","last_became_peered":"2024-08-17T17:12:40.334535+0000","last_unstale":"2024-08-17T17:15:24.086573+0000","last_undegraded":"2024-08-17T17:15:24.086573+0000","last_fullsized":"2024-08-17T17:15:24.086573+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:47:39.810507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.299324+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:16:09.299324+0000","last_peered":"2024-08-17T17:16:09.299324+0000","last_clean":"2024-08-17T17:16:09.299324+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:16:09.299324+0000","last_undegraded":"2024-08-17T17:16:09.299324+0000","last_fullsized":"2024-08-17T17:16:09.299324+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295526+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:16:09.295526+0000","last_peered":"2024-08-17T17:16:09.295526+0000","last_clean":"2024-08-17T17:16:09.295526+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:16:09.295526+0000","last_undegraded":"2024-08-17T17:16:09.295526+0000","last_fullsized":"2024-08-17T17:16:09.295526+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086438+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:15:24.086438+0000","last_peered":"2024-08-17T17:15:24.086438+0000","last_clean":"2024-08-17T17:15:24.086438+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:15:24.086438+0000","last_undegraded":"2024-08-17T17:15:24.086438+0000","last_fullsized":"2024-08-17T17:15:24.086438+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292483+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:16:09.292483+0000","last_peered":"2024-08-17T17:16:09.292483+0000","last_clean":"2024-08-17T17:16:09.292483+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:16:09.292483+0000","last_undegraded":"2024-08-17T17:16:09.292483+0000","last_fullsized":"2024-08-17T17:16:09.292483+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127961+0000","last_change":"2024-08-17T17:07:07.418480+0000","last_active":"2024-08-17T17:15:24.127961+0000","last_peered":"2024-08-17T17:15:24.127961+0000","last_clean":"2024-08-17T17:15:24.127961+0000","last_became_active":"2024-08-17T17:07:07.418124+0000","last_became_peered":"2024-08-17T17:07:07.418124+0000","last_unstale":"2024-08-17T17:15:24.127961+0000","last_undegraded":"2024-08-17T17:15:24.127961+0000","last_fullsized":"2024-08-17T17:15:24.127961+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:12.020815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087282+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:15:24.087282+0000","last_peered":"2024-08-17T17:15:24.087282+0000","last_clean":"2024-08-17T17:15:24.087282+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:15:24.087282+0000","last_undegraded":"2024-08-17T17:15:24.087282+0000","last_fullsized":"2024-08-17T17:15:24.087282+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.088543+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T17:15:24.088543+0000","last_peered":"2024-08-17T17:15:24.088543+0000","last_clean":"2024-08-17T17:15:24.088543+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T17:15:24.088543+0000","last_undegraded":"2024-08-17T17:15:24.088543+0000","last_fullsized":"2024-08-17T17:15:24.088543+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087299+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:15:24.087299+0000","last_peered":"2024-08-17T17:15:24.087299+0000","last_clean":"2024-08-17T17:15:24.087299+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:15:24.087299+0000","last_undegraded":"2024-08-17T17:15:24.087299+0000","last_fullsized":"2024-08-17T17:15:24.087299+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129675+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:15:24.129675+0000","last_peered":"2024-08-17T17:15:24.129675+0000","last_clean":"2024-08-17T17:15:24.129675+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:15:24.129675+0000","last_undegraded":"2024-08-17T17:15:24.129675+0000","last_fullsized":"2024-08-17T17:15:24.129675+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087183+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T17:15:24.087183+0000","last_peered":"2024-08-17T17:15:24.087183+0000","last_clean":"2024-08-17T17:15:24.087183+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T17:15:24.087183+0000","last_undegraded":"2024-08-17T17:15:24.087183+0000","last_fullsized":"2024-08-17T17:15:24.087183+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.127970+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:15:24.127970+0000","last_peered":"2024-08-17T17:15:24.127970+0000","last_clean":"2024-08-17T17:15:24.127970+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:15:24.127970+0000","last_undegraded":"2024-08-17T17:15:24.127970+0000","last_fullsized":"2024-08-17T17:15:24.127970+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086790+0000","last_change":"2024-08-17T17:12:40.336371+0000","last_active":"2024-08-17T17:15:24.086790+0000","last_peered":"2024-08-17T17:15:24.086790+0000","last_clean":"2024-08-17T17:15:24.086790+0000","last_became_active":"2024-08-17T17:12:40.335783+0000","last_became_peered":"2024-08-17T17:12:40.335783+0000","last_unstale":"2024-08-17T17:15:24.086790+0000","last_undegraded":"2024-08-17T17:15:24.086790+0000","last_fullsized":"2024-08-17T17:15:24.086790+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:14.313020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291175+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:16:09.291175+0000","last_peered":"2024-08-17T17:16:09.291175+0000","last_clean":"2024-08-17T17:16:09.291175+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:16:09.291175+0000","last_undegraded":"2024-08-17T17:16:09.291175+0000","last_fullsized":"2024-08-17T17:16:09.291175+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295720+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:16:09.295720+0000","last_peered":"2024-08-17T17:16:09.295720+0000","last_clean":"2024-08-17T17:16:09.295720+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:16:09.295720+0000","last_undegraded":"2024-08-17T17:16:09.295720+0000","last_fullsized":"2024-08-17T17:16:09.295720+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130241+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:15:24.130241+0000","last_peered":"2024-08-17T17:15:24.130241+0000","last_clean":"2024-08-17T17:15:24.130241+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:15:24.130241+0000","last_undegraded":"2024-08-17T17:15:24.130241+0000","last_fullsized":"2024-08-17T17:15:24.130241+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128095+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:15:24.128095+0000","last_peered":"2024-08-17T17:15:24.128095+0000","last_clean":"2024-08-17T17:15:24.128095+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:15:24.128095+0000","last_undegraded":"2024-08-17T17:15:24.128095+0000","last_fullsized":"2024-08-17T17:15:24.128095+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.133692+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:15:24.133692+0000","last_peered":"2024-08-17T17:15:24.133692+0000","last_clean":"2024-08-17T17:15:24.133692+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:15:24.133692+0000","last_undegraded":"2024-08-17T17:15:24.133692+0000","last_fullsized":"2024-08-17T17:15:24.133692+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129775+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:15:24.129775+0000","last_peered":"2024-08-17T17:15:24.129775+0000","last_clean":"2024-08-17T17:15:24.129775+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:15:24.129775+0000","last_undegraded":"2024-08-17T17:15:24.129775+0000","last_fullsized":"2024-08-17T17:15:24.129775+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130215+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:15:24.130215+0000","last_peered":"2024-08-17T17:15:24.130215+0000","last_clean":"2024-08-17T17:15:24.130215+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:15:24.130215+0000","last_undegraded":"2024-08-17T17:15:24.130215+0000","last_fullsized":"2024-08-17T17:15:24.130215+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293371+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:16:09.293371+0000","last_peered":"2024-08-17T17:16:09.293371+0000","last_clean":"2024-08-17T17:16:09.293371+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:16:09.293371+0000","last_undegraded":"2024-08-17T17:16:09.293371+0000","last_fullsized":"2024-08-17T17:16:09.293371+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086551+0000","last_change":"2024-08-17T17:12:40.335001+0000","last_active":"2024-08-17T17:15:24.086551+0000","last_peered":"2024-08-17T17:15:24.086551+0000","last_clean":"2024-08-17T17:15:24.086551+0000","last_became_active":"2024-08-17T17:12:40.334650+0000","last_became_peered":"2024-08-17T17:12:40.334650+0000","last_unstale":"2024-08-17T17:15:24.086551+0000","last_undegraded":"2024-08-17T17:15:24.086551+0000","last_fullsized":"2024-08-17T17:15:24.086551+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:06.876541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":449,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086423+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:15:24.086423+0000","last_peered":"2024-08-17T17:15:24.086423+0000","last_clean":"2024-08-17T17:15:24.086423+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:15:24.086423+0000","last_undegraded":"2024-08-17T17:15:24.086423+0000","last_fullsized":"2024-08-17T17:15:24.086423+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086615+0000","last_change":"2024-08-17T17:10:55.313434+0000","last_active":"2024-08-17T17:15:24.086615+0000","last_peered":"2024-08-17T17:15:24.086615+0000","last_clean":"2024-08-17T17:15:24.086615+0000","last_became_active":"2024-08-17T17:10:55.313091+0000","last_became_peered":"2024-08-17T17:10:55.313091+0000","last_unstale":"2024-08-17T17:15:24.086615+0000","last_undegraded":"2024-08-17T17:15:24.086615+0000","last_fullsized":"2024-08-17T17:15:24.086615+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:17.970517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293055+0000","last_change":"2024-08-17T17:12:40.332647+0000","last_active":"2024-08-17T17:16:09.293055+0000","last_peered":"2024-08-17T17:16:09.293055+0000","last_clean":"2024-08-17T17:16:09.293055+0000","last_became_active":"2024-08-17T17:12:40.332376+0000","last_became_peered":"2024-08-17T17:12:40.332376+0000","last_unstale":"2024-08-17T17:16:09.293055+0000","last_undegraded":"2024-08-17T17:16:09.293055+0000","last_fullsized":"2024-08-17T17:16:09.293055+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:21.624216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128216+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:15:24.128216+0000","last_peered":"2024-08-17T17:15:24.128216+0000","last_clean":"2024-08-17T17:15:24.128216+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:15:24.128216+0000","last_undegraded":"2024-08-17T17:15:24.128216+0000","last_fullsized":"2024-08-17T17:15:24.128216+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086495+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:15:24.086495+0000","last_peered":"2024-08-17T17:15:24.086495+0000","last_clean":"2024-08-17T17:15:24.086495+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:15:24.086495+0000","last_undegraded":"2024-08-17T17:15:24.086495+0000","last_fullsized":"2024-08-17T17:15:24.086495+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291546+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:16:09.291546+0000","last_peered":"2024-08-17T17:16:09.291546+0000","last_clean":"2024-08-17T17:16:09.291546+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:16:09.291546+0000","last_undegraded":"2024-08-17T17:16:09.291546+0000","last_fullsized":"2024-08-17T17:16:09.291546+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295310+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:16:09.295310+0000","last_peered":"2024-08-17T17:16:09.295310+0000","last_clean":"2024-08-17T17:16:09.295310+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:16:09.295310+0000","last_undegraded":"2024-08-17T17:16:09.295310+0000","last_fullsized":"2024-08-17T17:16:09.295310+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":150,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.542779+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:16:08.542779+0000","last_peered":"2024-08-17T17:16:08.542779+0000","last_clean":"2024-08-17T17:16:08.542779+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:16:08.542779+0000","last_undegraded":"2024-08-17T17:16:08.542779+0000","last_fullsized":"2024-08-17T17:16:08.542779+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.546939+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:16:08.546939+0000","last_peered":"2024-08-17T17:16:08.546939+0000","last_clean":"2024-08-17T17:16:08.546939+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:16:08.546939+0000","last_undegraded":"2024-08-17T17:16:08.546939+0000","last_fullsized":"2024-08-17T17:16:08.546939+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":156,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.557064+0000","last_change":"2024-08-17T17:07:51.480410+0000","last_active":"2024-08-17T17:16:08.557064+0000","last_peered":"2024-08-17T17:16:08.557064+0000","last_clean":"2024-08-17T17:16:08.557064+0000","last_became_active":"2024-08-17T17:07:51.480083+0000","last_became_peered":"2024-08-17T17:07:51.480083+0000","last_unstale":"2024-08-17T17:16:08.557064+0000","last_undegraded":"2024-08-17T17:16:08.557064+0000","last_fullsized":"2024-08-17T17:16:08.557064+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:39.376320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086589+0000","last_change":"2024-08-17T17:07:51.483538+0000","last_active":"2024-08-17T17:15:24.086589+0000","last_peered":"2024-08-17T17:15:24.086589+0000","last_clean":"2024-08-17T17:15:24.086589+0000","last_became_active":"2024-08-17T17:07:51.482970+0000","last_became_peered":"2024-08-17T17:07:51.482970+0000","last_unstale":"2024-08-17T17:15:24.086589+0000","last_undegraded":"2024-08-17T17:15:24.086589+0000","last_fullsized":"2024-08-17T17:15:24.086589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:37.985321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130448+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:15:24.130448+0000","last_peered":"2024-08-17T17:15:24.130448+0000","last_clean":"2024-08-17T17:15:24.130448+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:15:24.130448+0000","last_undegraded":"2024-08-17T17:15:24.130448+0000","last_fullsized":"2024-08-17T17:15:24.130448+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544455+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:16:08.544455+0000","last_peered":"2024-08-17T17:16:08.544455+0000","last_clean":"2024-08-17T17:16:08.544455+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:16:08.544455+0000","last_undegraded":"2024-08-17T17:16:08.544455+0000","last_fullsized":"2024-08-17T17:16:08.544455+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086721+0000","last_change":"2024-08-17T17:12:40.335196+0000","last_active":"2024-08-17T17:15:24.086721+0000","last_peered":"2024-08-17T17:15:24.086721+0000","last_clean":"2024-08-17T17:15:24.086721+0000","last_became_active":"2024-08-17T17:12:40.334917+0000","last_became_peered":"2024-08-17T17:12:40.334917+0000","last_unstale":"2024-08-17T17:15:24.086721+0000","last_undegraded":"2024-08-17T17:15:24.086721+0000","last_fullsized":"2024-08-17T17:15:24.086721+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:02.633323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":169,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544620+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:16:08.544620+0000","last_peered":"2024-08-17T17:16:08.544620+0000","last_clean":"2024-08-17T17:16:08.544620+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:16:08.544620+0000","last_undegraded":"2024-08-17T17:16:08.544620+0000","last_fullsized":"2024-08-17T17:16:08.544620+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545444+0000","last_change":"2024-08-17T17:07:51.489205+0000","last_active":"2024-08-17T17:16:08.545444+0000","last_peered":"2024-08-17T17:16:08.545444+0000","last_clean":"2024-08-17T17:16:08.545444+0000","last_became_active":"2024-08-17T17:07:51.488783+0000","last_became_peered":"2024-08-17T17:07:51.488783+0000","last_unstale":"2024-08-17T17:16:08.545444+0000","last_undegraded":"2024-08-17T17:16:08.545444+0000","last_fullsized":"2024-08-17T17:16:08.545444+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:22.190780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544540+0000","last_change":"2024-08-17T17:10:55.321862+0000","last_active":"2024-08-17T17:16:08.544540+0000","last_peered":"2024-08-17T17:16:08.544540+0000","last_clean":"2024-08-17T17:16:08.544540+0000","last_became_active":"2024-08-17T17:10:55.321339+0000","last_became_peered":"2024-08-17T17:10:55.321339+0000","last_unstale":"2024-08-17T17:16:08.544540+0000","last_undegraded":"2024-08-17T17:16:08.544540+0000","last_fullsized":"2024-08-17T17:16:08.544540+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:17.662591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087198+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:15:24.087198+0000","last_peered":"2024-08-17T17:15:24.087198+0000","last_clean":"2024-08-17T17:15:24.087198+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:15:24.087198+0000","last_undegraded":"2024-08-17T17:15:24.087198+0000","last_fullsized":"2024-08-17T17:15:24.087198+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291264+0000","last_change":"2024-08-17T17:07:07.424546+0000","last_active":"2024-08-17T17:16:09.291264+0000","last_peered":"2024-08-17T17:16:09.291264+0000","last_clean":"2024-08-17T17:16:09.291264+0000","last_became_active":"2024-08-17T17:07:07.424158+0000","last_became_peered":"2024-08-17T17:07:07.424158+0000","last_unstale":"2024-08-17T17:16:09.291264+0000","last_undegraded":"2024-08-17T17:16:09.291264+0000","last_fullsized":"2024-08-17T17:16:09.291264+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:34.175159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.292665+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:16:09.292665+0000","last_peered":"2024-08-17T17:16:09.292665+0000","last_clean":"2024-08-17T17:16:09.292665+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:16:09.292665+0000","last_undegraded":"2024-08-17T17:16:09.292665+0000","last_fullsized":"2024-08-17T17:16:09.292665+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545043+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:16:08.545043+0000","last_peered":"2024-08-17T17:16:08.545043+0000","last_clean":"2024-08-17T17:16:08.545043+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:16:08.545043+0000","last_undegraded":"2024-08-17T17:16:08.545043+0000","last_fullsized":"2024-08-17T17:16:08.545043+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291612+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:16:09.291612+0000","last_peered":"2024-08-17T17:16:09.291612+0000","last_clean":"2024-08-17T17:16:09.291612+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:16:09.291612+0000","last_undegraded":"2024-08-17T17:16:09.291612+0000","last_fullsized":"2024-08-17T17:16:09.291612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295589+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:16:09.295589+0000","last_peered":"2024-08-17T17:16:09.295589+0000","last_clean":"2024-08-17T17:16:09.295589+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:16:09.295589+0000","last_undegraded":"2024-08-17T17:16:09.295589+0000","last_fullsized":"2024-08-17T17:16:09.295589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128317+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:15:24.128317+0000","last_peered":"2024-08-17T17:15:24.128317+0000","last_clean":"2024-08-17T17:15:24.128317+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:15:24.128317+0000","last_undegraded":"2024-08-17T17:15:24.128317+0000","last_fullsized":"2024-08-17T17:15:24.128317+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129742+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T17:15:24.129742+0000","last_peered":"2024-08-17T17:15:24.129742+0000","last_clean":"2024-08-17T17:15:24.129742+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T17:15:24.129742+0000","last_undegraded":"2024-08-17T17:15:24.129742+0000","last_fullsized":"2024-08-17T17:15:24.129742+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130344+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:15:24.130344+0000","last_peered":"2024-08-17T17:15:24.130344+0000","last_clean":"2024-08-17T17:15:24.130344+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:15:24.130344+0000","last_undegraded":"2024-08-17T17:15:24.130344+0000","last_fullsized":"2024-08-17T17:15:24.130344+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":114,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130369+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:15:24.130369+0000","last_peered":"2024-08-17T17:15:24.130369+0000","last_clean":"2024-08-17T17:15:24.130369+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:15:24.130369+0000","last_undegraded":"2024-08-17T17:15:24.130369+0000","last_fullsized":"2024-08-17T17:15:24.130369+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545266+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:16:08.545266+0000","last_peered":"2024-08-17T17:16:08.545266+0000","last_clean":"2024-08-17T17:16:08.545266+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:16:08.545266+0000","last_undegraded":"2024-08-17T17:16:08.545266+0000","last_fullsized":"2024-08-17T17:16:08.545266+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.087099+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:15:24.087099+0000","last_peered":"2024-08-17T17:15:24.087099+0000","last_clean":"2024-08-17T17:15:24.087099+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:15:24.087099+0000","last_undegraded":"2024-08-17T17:15:24.087099+0000","last_fullsized":"2024-08-17T17:15:24.087099+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086506+0000","last_change":"2024-08-17T17:07:51.484772+0000","last_active":"2024-08-17T17:15:24.086506+0000","last_peered":"2024-08-17T17:15:24.086506+0000","last_clean":"2024-08-17T17:15:24.086506+0000","last_became_active":"2024-08-17T17:07:51.484397+0000","last_became_peered":"2024-08-17T17:07:51.484397+0000","last_unstale":"2024-08-17T17:15:24.086506+0000","last_undegraded":"2024-08-17T17:15:24.086506+0000","last_fullsized":"2024-08-17T17:15:24.086506+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:21.701624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290999+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:16:09.290999+0000","last_peered":"2024-08-17T17:16:09.290999+0000","last_clean":"2024-08-17T17:16:09.290999+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:16:09.290999+0000","last_undegraded":"2024-08-17T17:16:09.290999+0000","last_fullsized":"2024-08-17T17:16:09.290999+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":164,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291726+0000","last_change":"2024-08-17T17:12:40.341757+0000","last_active":"2024-08-17T17:16:09.291726+0000","last_peered":"2024-08-17T17:16:09.291726+0000","last_clean":"2024-08-17T17:16:09.291726+0000","last_became_active":"2024-08-17T17:12:40.341446+0000","last_became_peered":"2024-08-17T17:12:40.341446+0000","last_unstale":"2024-08-17T17:16:09.291726+0000","last_undegraded":"2024-08-17T17:16:09.291726+0000","last_fullsized":"2024-08-17T17:16:09.291726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:51.178405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.290988+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:16:09.290988+0000","last_peered":"2024-08-17T17:16:09.290988+0000","last_clean":"2024-08-17T17:16:09.290988+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:16:09.290988+0000","last_undegraded":"2024-08-17T17:16:09.290988+0000","last_fullsized":"2024-08-17T17:16:09.290988+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.128292+0000","last_change":"2024-08-17T17:07:51.484291+0000","last_active":"2024-08-17T17:15:24.128292+0000","last_peered":"2024-08-17T17:15:24.128292+0000","last_clean":"2024-08-17T17:15:24.128292+0000","last_became_active":"2024-08-17T17:07:51.483903+0000","last_became_peered":"2024-08-17T17:07:51.483903+0000","last_unstale":"2024-08-17T17:15:24.128292+0000","last_undegraded":"2024-08-17T17:15:24.128292+0000","last_fullsized":"2024-08-17T17:15:24.128292+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:02.755382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291658+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:16:09.291658+0000","last_peered":"2024-08-17T17:16:09.291658+0000","last_clean":"2024-08-17T17:16:09.291658+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:16:09.291658+0000","last_undegraded":"2024-08-17T17:16:09.291658+0000","last_fullsized":"2024-08-17T17:16:09.291658+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295443+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:16:09.295443+0000","last_peered":"2024-08-17T17:16:09.295443+0000","last_clean":"2024-08-17T17:16:09.295443+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:16:09.295443+0000","last_undegraded":"2024-08-17T17:16:09.295443+0000","last_fullsized":"2024-08-17T17:16:09.295443+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.544667+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:16:08.544667+0000","last_peered":"2024-08-17T17:16:08.544667+0000","last_clean":"2024-08-17T17:16:08.544667+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:16:08.544667+0000","last_undegraded":"2024-08-17T17:16:08.544667+0000","last_fullsized":"2024-08-17T17:16:08.544667+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291001+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:16:09.291001+0000","last_peered":"2024-08-17T17:16:09.291001+0000","last_clean":"2024-08-17T17:16:09.291001+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:16:09.291001+0000","last_undegraded":"2024-08-17T17:16:09.291001+0000","last_fullsized":"2024-08-17T17:16:09.291001+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":127,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129963+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:15:24.129963+0000","last_peered":"2024-08-17T17:15:24.129963+0000","last_clean":"2024-08-17T17:15:24.129963+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:15:24.129963+0000","last_undegraded":"2024-08-17T17:15:24.129963+0000","last_fullsized":"2024-08-17T17:15:24.129963+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:08.545404+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:16:08.545404+0000","last_peered":"2024-08-17T17:16:08.545404+0000","last_clean":"2024-08-17T17:16:08.545404+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:16:08.545404+0000","last_undegraded":"2024-08-17T17:16:08.545404+0000","last_fullsized":"2024-08-17T17:16:08.545404+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":157,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291389+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:16:09.291389+0000","last_peered":"2024-08-17T17:16:09.291389+0000","last_clean":"2024-08-17T17:16:09.291389+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:16:09.291389+0000","last_undegraded":"2024-08-17T17:16:09.291389+0000","last_fullsized":"2024-08-17T17:16:09.291389+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.130279+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:15:24.130279+0000","last_peered":"2024-08-17T17:15:24.130279+0000","last_clean":"2024-08-17T17:15:24.130279+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:15:24.130279+0000","last_undegraded":"2024-08-17T17:15:24.130279+0000","last_fullsized":"2024-08-17T17:15:24.130279+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086782+0000","last_change":"2024-08-17T17:07:51.482888+0000","last_active":"2024-08-17T17:15:24.086782+0000","last_peered":"2024-08-17T17:15:24.086782+0000","last_clean":"2024-08-17T17:15:24.086782+0000","last_became_active":"2024-08-17T17:07:51.482133+0000","last_became_peered":"2024-08-17T17:07:51.482133+0000","last_unstale":"2024-08-17T17:15:24.086782+0000","last_undegraded":"2024-08-17T17:15:24.086782+0000","last_fullsized":"2024-08-17T17:15:24.086782+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:47.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129880+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:15:24.129880+0000","last_peered":"2024-08-17T17:15:24.129880+0000","last_clean":"2024-08-17T17:15:24.129880+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:15:24.129880+0000","last_undegraded":"2024-08-17T17:15:24.129880+0000","last_fullsized":"2024-08-17T17:15:24.129880+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":95,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.086698+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T17:15:24.086698+0000","last_peered":"2024-08-17T17:15:24.086698+0000","last_clean":"2024-08-17T17:15:24.086698+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T17:15:24.086698+0000","last_undegraded":"2024-08-17T17:15:24.086698+0000","last_fullsized":"2024-08-17T17:15:24.086698+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.295973+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:16:09.295973+0000","last_peered":"2024-08-17T17:16:09.295973+0000","last_clean":"2024-08-17T17:16:09.295973+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:16:09.295973+0000","last_undegraded":"2024-08-17T17:16:09.295973+0000","last_fullsized":"2024-08-17T17:16:09.295973+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.293888+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:16:09.293888+0000","last_peered":"2024-08-17T17:16:09.293888+0000","last_clean":"2024-08-17T17:16:09.293888+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:16:09.293888+0000","last_undegraded":"2024-08-17T17:16:09.293888+0000","last_fullsized":"2024-08-17T17:16:09.293888+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-08-17T17:15:24.129570+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:15:24.129570+0000","last_peered":"2024-08-17T17:15:24.129570+0000","last_clean":"2024-08-17T17:15:24.129570+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:15:24.129570+0000","last_undegraded":"2024-08-17T17:15:24.129570+0000","last_fullsized":"2024-08-17T17:15:24.129570+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":168,"reported_epoch":153,"state":"active+clean","last_fresh":"2024-08-17T17:16:09.291343+0000","last_change":"2024-08-17T17:10:55.313325+0000","last_active":"2024-08-17T17:16:09.291343+0000","last_peered":"2024-08-17T17:16:09.291343+0000","last_clean":"2024-08-17T17:16:09.291343+0000","last_became_active":"2024-08-17T17:10:55.311737+0000","last_became_peered":"2024-08-17T17:10:55.311737+0000","last_unstale":"2024-08-17T17:16:09.291343+0000","last_undegraded":"2024-08-17T17:16:09.291343+0000","last_fullsized":"2024-08-17T17:16:09.291343+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:06.082766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13723,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299680,"kb_used_data":2324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445472,"statfs":{"total":95995035648,"available":95688163328,"internally_reserved":0,"allocated":2379776,"data_stored":1598688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463177,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312152,"kb_used_data":2316,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93433000,"statfs":{"total":95995035648,"available":95675392000,"internally_reserved":0,"allocated":2371584,"data_stored":1598545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724815,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903176,"kb_used_data":1724,"kb_used_omap":1,"kb_used_meta":901438,"kb_avail":92841976,"statfs":{"total":95995035648,"available":95070183424,"internally_reserved":0,"allocated":1765376,"data_stored":1003680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923072856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986451,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591636,"kb_used_data":1736,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":93153516,"statfs":{"total":95995035648,"available":95389200384,"internally_reserved":0,"allocated":1777664,"data_stored":1007977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225271,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210304,"kb_used_data":1716,"kb_used_omap":4,"kb_used_meta":208571,"kb_avail":93534848,"statfs":{"total":95995035648,"available":95779684352,"internally_reserved":0,"allocated":1757184,"data_stored":1003181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":213577152},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075596,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":225032,"kb_used_data":1724,"kb_used_omap":6,"kb_used_meta":223289,"kb_avail":93520120,"statfs":{"total":95995035648,"available":95764602880,"internally_reserved":0,"allocated":1765376,"data_stored":1003968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":228648379},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533489,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":364480,"kb_used_data":1716,"kb_used_omap":5,"kb_used_meta":362746,"kb_avail":93380672,"statfs":{"total":95995035648,"available":95621808128,"internally_reserved":0,"allocated":1757184,"data_stored":1001971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":371452144},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089466,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":370124,"kb_used_data":2304,"kb_used_omap":5,"kb_used_meta":367802,"kb_avail":93375028,"statfs":{"total":95995035648,"available":95616028672,"internally_reserved":0,"allocated":2359296,"data_stored":1594332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":376629307},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:17:09.355 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:17:09.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump 2024-08-17T17:17:09.611 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:10.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:09 smithi142 ceph-mon[106729]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:10.014 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:09 smithi142 ceph-mon[103614]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:10.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:09 smithi152 ceph-mon[93456]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:17:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:17:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:17:11.031 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:10 smithi142 ceph-mon[106729]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:11.031 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:10 smithi142 ceph-mon[103614]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:10 smithi152 ceph-mon[93456]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:11.447 INFO:teuthology.orchestra.run.smithi142.stdout:epoch 153 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:created 2024-08-17T16:50:52.038786+0000 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:modified 2024-08-17T17:16:08.533950+0000 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:crush_version 22 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:full_ratio 0.95 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:backfillfull_ratio 0.9 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:nearfull_ratio 0.85 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:require_min_compat_client luminous 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:min_compat_client luminous 2024-08-17T17:17:11.448 INFO:teuthology.orchestra.run.smithi142.stdout:require_osd_release quincy 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:stretch_mode_enabled false 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 65 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:max_osd 8 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0 up in weight 1 up_from 89 up_thru 151 down_at 86 last_clean_interval [8,85) [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512] [v2:172.21.15.142:6804/608035512,v1:172.21.15.142:6805/608035512] exists,up 4a1ac03b-9520-4665-ba56-380cd889d3f0 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 up in weight 1 up_from 106 up_thru 148 down_at 101 last_clean_interval [13,100) [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993] [v2:172.21.15.142:6812/2324159993,v1:172.21.15.142:6813/2324159993] exists,up fa145811-9dfa-468b-a005-75eb2042368d 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2 up in weight 1 up_from 120 up_thru 146 down_at 117 last_clean_interval [18,116) [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937] [v2:172.21.15.142:6820/366314937,v1:172.21.15.142:6821/366314937] exists,up 4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3 up in weight 1 up_from 146 up_thru 149 down_at 143 last_clean_interval [26,142) [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216] [v2:172.21.15.142:6828/195335216,v1:172.21.15.142:6829/195335216] exists,up f9b26ffd-2955-4af8-90c8-5473d3c78fa4 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4 up in weight 1 up_from 31 up_thru 146 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822] [v2:172.21.15.152:6802/3554198822,v1:172.21.15.152:6803/3554198822] exists,up 76eecfff-f5c2-4180-a0c5-26b83e8359e8 2024-08-17T17:17:11.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5 up in weight 1 up_from 39 up_thru 147 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029] [v2:172.21.15.152:6810/1844077029,v1:172.21.15.152:6811/1844077029] exists,up f36620dd-2b34-4322-92c9-59ff43a6799e 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6 up in weight 1 up_from 47 up_thru 143 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576] [v2:172.21.15.152:6818/2120783576,v1:172.21.15.152:6819/2120783576] exists,up 4dc7a5d8-e2a2-4687-a4d8-53799e862d2b 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7 up in weight 1 up_from 55 up_thru 146 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638] [v2:172.21.15.152:6826/1562643638,v1:172.21.15.152:6827/1562643638] exists,up 3a1c6335-e51f-45be-afe7-fd487ca70a0a 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:pg_upmap_items 4.f [1,5] 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:pg_upmap_items 4.14 [1,2] 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:pg_upmap_items 4.1b [1,2] 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.152:0/3235892463 expires 2024-08-18T17:02:18.823112+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6801/3742822359 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/292376296 expires 2024-08-18T16:51:41.645069+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6800/3742822359 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.152:0/3126412235 expires 2024-08-18T17:02:18.823112+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/2627951813 expires 2024-08-18T17:00:33.779952+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6801/3686065451 expires 2024-08-18T16:51:20.134164+0000 2024-08-17T17:17:11.450 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6800/3686065451 expires 2024-08-18T16:51:20.134164+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/3686891819 expires 2024-08-18T16:56:12.454665+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/283129895 expires 2024-08-18T16:51:41.645069+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6801/3356817710 expires 2024-08-18T17:00:33.779952+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/913350931 expires 2024-08-18T16:56:12.454665+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/267249921 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.152:6834/3003700763 expires 2024-08-18T17:02:18.823112+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.152:0/681595170 expires 2024-08-18T17:02:18.823112+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/2588146503 expires 2024-08-18T16:51:20.134164+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6800/3356817710 expires 2024-08-18T17:00:33.779952+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/2543002123 expires 2024-08-18T16:51:20.134164+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/3034915416 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.451 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/298990150 expires 2024-08-18T16:51:41.645069+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.152:0/3133806592 expires 2024-08-18T17:02:18.823112+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6800/508280165 expires 2024-08-18T16:51:41.645069+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/2472242802 expires 2024-08-18T17:00:33.779952+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/639679378 expires 2024-08-18T17:00:33.779952+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.152:6835/3003700763 expires 2024-08-18T17:02:18.823112+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6801/508280165 expires 2024-08-18T16:51:41.645069+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/1828093647 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/528641382 expires 2024-08-18T16:51:20.134164+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6800/744771467 expires 2024-08-18T16:56:12.454665+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:6801/744771467 expires 2024-08-18T16:56:12.454665+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/1253216666 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/1554300251 expires 2024-08-18T17:00:33.779952+0000 2024-08-17T17:17:11.452 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/3319043742 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:11.453 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/54516382 expires 2024-08-18T16:56:12.454665+0000 2024-08-17T17:17:11.453 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/3367745640 expires 2024-08-18T16:56:12.454665+0000 2024-08-17T17:17:11.453 INFO:teuthology.orchestra.run.smithi142.stdout:blocklist 172.21.15.142:0/3716608685 expires 2024-08-18T17:03:11.046184+0000 2024-08-17T17:17:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1999930961' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:17:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1999930961' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:17:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:17:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:17:11] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.51.0" 2024-08-17T17:17:12.193 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 89 up_thru 151 down_at 86 last_clean_interval [8,85) [v2:172.21.15.142:6802/608035512,v1:172.21.15.142:6803/608035512] [v2:172.21.15.142:6804/608035512,v1:172.21.15.142:6805/608035512] exists,up 4a1ac03b-9520-4665-ba56-380cd889d3f0', 'osd.1 up in weight 1 up_from 106 up_thru 148 down_at 101 last_clean_interval [13,100) [v2:172.21.15.142:6810/2324159993,v1:172.21.15.142:6811/2324159993] [v2:172.21.15.142:6812/2324159993,v1:172.21.15.142:6813/2324159993] exists,up fa145811-9dfa-468b-a005-75eb2042368d', 'osd.2 up in weight 1 up_from 120 up_thru 146 down_at 117 last_clean_interval [18,116) [v2:172.21.15.142:6818/366314937,v1:172.21.15.142:6819/366314937] [v2:172.21.15.142:6820/366314937,v1:172.21.15.142:6821/366314937] exists,up 4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63', 'osd.3 up in weight 1 up_from 146 up_thru 149 down_at 143 last_clean_interval [26,142) [v2:172.21.15.142:6826/195335216,v1:172.21.15.142:6827/195335216] [v2:172.21.15.142:6828/195335216,v1:172.21.15.142:6829/195335216] exists,up f9b26ffd-2955-4af8-90c8-5473d3c78fa4', 'osd.4 up in weight 1 up_from 31 up_thru 146 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6800/3554198822,v1:172.21.15.152:6801/3554198822] [v2:172.21.15.152:6802/3554198822,v1:172.21.15.152:6803/3554198822] exists,up 76eecfff-f5c2-4180-a0c5-26b83e8359e8', 'osd.5 up in weight 1 up_from 39 up_thru 147 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6808/1844077029,v1:172.21.15.152:6809/1844077029] [v2:172.21.15.152:6810/1844077029,v1:172.21.15.152:6811/1844077029] exists,up f36620dd-2b34-4322-92c9-59ff43a6799e', 'osd.6 up in weight 1 up_from 47 up_thru 143 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6816/2120783576,v1:172.21.15.152:6817/2120783576] [v2:172.21.15.152:6818/2120783576,v1:172.21.15.152:6819/2120783576] exists,up 4dc7a5d8-e2a2-4687-a4d8-53799e862d2b', 'osd.7 up in weight 1 up_from 55 up_thru 146 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6824/1562643638,v1:172.21.15.152:6825/1562643638] [v2:172.21.15.152:6826/1562643638,v1:172.21.15.152:6827/1562643638] exists,up 3a1c6335-e51f-45be-afe7-fd487ca70a0a'] 2024-08-17T17:17:12.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon mon.a config show 2024-08-17T17:17:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1999930961' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:17:12.448 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:12 smithi152 ceph-mon[93456]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:13.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:12 smithi142 ceph-mon[106729]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:12 smithi142 ceph-mon[103614]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:14.178 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:17:14.178 INFO:teuthology.orchestra.run.smithi142.stdout: "name": "mon.a", 2024-08-17T17:17:14.178 INFO:teuthology.orchestra.run.smithi142.stdout: "cluster": "ceph", 2024-08-17T17:17:14.178 INFO:teuthology.orchestra.run.smithi142.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "admin_socket_mode": "", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_client_required": "cephx, none", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_cluster_required": "cephx", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_debug": "true", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_service_required": "cephx", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_service_ticket_ttl": "240.000000", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "auth_supported": "", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_aio": "true", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_aio_max_queue_depth": "1024", 2024-08-17T17:17:14.179 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_aio_poll_ms": "250", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_aio_reap_max": "16", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_async_discard_threads": "0", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_block_size": "4096", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_debug_aio": "false", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_debug_inflight_ios": "false", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_enable_discard": "false", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_flock_retry": "3", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_flock_retry_interval": "0.100000", 2024-08-17T17:17:14.180 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_inject_crash": "0", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_inject_crash_flush_delay": "2", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_ioring": "false", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_ioring_hipri": "false", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_ioring_sqthread_poll": "false", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_read_buffer_alignment": "4096", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bdev_type": "", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_alloc_size": "1048576", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_allocator": "hybrid", 2024-08-17T17:17:14.181 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_buffered_io": "true", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_check_for_zeros": "false", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_check_volume_selector_often": "false", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_compact_log_sync": "false", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_log_compact_min_size": "16777216", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_log_replay_check_allocations": "true", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_max_log_runway": "4194304", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_max_prefetch": "1048576", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_min_flush_size": "524288", 2024-08-17T17:17:14.182 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_min_log_runway": "1048576", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_replay_recovery": "false", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_shared_alloc_size": "65536", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluefs_sync_write": "false", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_allocation_from_file": "true", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_allocator": "hybrid", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_async_db_compaction": "true", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-08-17T17:17:14.183 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bdev_label_multi": "true", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bdev_label_require_all": "true", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_blobid_prealloc": "10240", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_create": "true", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_db_create": "false", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_db_path": "", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_db_size": "0", 2024-08-17T17:17:14.184 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_path": "", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_preallocate_file": "false", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_size": "107374182400", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_wal_create": "false", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_wal_path": "", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_block_wal_size": "100663296", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bluefs": "true", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bluefs_env_mirror": "false", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-08-17T17:17:14.185 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_autotune": "true", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_size": "0", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-08-17T17:17:14.186 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cache_type": "2q", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_clone_cow": "true", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_algorithm": "snappy", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_max_blob_size": "0", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_min_blob_size": "0", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_mode": "none", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-08-17T17:17:14.187 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_csum_type": "crc32c", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_enforce_settings": "default", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_freelist": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_fsck_abort": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_inject_read_err": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_legacy_omap": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-08-17T17:17:14.188 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_small_allocations": "0", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_default_buffered_read": "true", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_default_buffered_write": "false", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_deferred_batch_ops": "0", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_elastic_shared_blobs": "true", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-08-17T17:17:14.189 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fail_eio": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_on_mkfs": "true", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_on_mount": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_on_umount": "false", 2024-08-17T17:17:14.190 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_ignore_data_csum": "false", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_kvbackend": "rocksdb", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-08-17T17:17:14.191 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_log_op_age": "5.000000", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_max_alloc_size": "0", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_max_blob_size": "0", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_max_defer_interval": "3.000000", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_max_deferred_txc": "32", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_min_alloc_size": "0", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_nid_prealloc": "1024", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_prefer_deferred_size": "0", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-08-17T17:17:14.192 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_qfsck_on_mount": "true", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_retry_disk_reads": "3", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_rocksdb_cf": "true", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_rocksdb_options_annex": "", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_spdk_coremask": "0x1", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_spdk_io_sleep": "5", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_spdk_max_io_completion": "0", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_spdk_mem": "512", 2024-08-17T17:17:14.193 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_sync_submit_transaction": "false", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_throttle_bytes": "67108864", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_throttle_cost_per_io": "0", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_tracing": "false", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_volume_selection_reserved": "0", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-08-17T17:17:14.194 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_write_v2": "false", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_write_v2_random": "false", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "bluestore_zero_block_detection": "false", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_action_update_interval": "2", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-08-17T17:17:14.195 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_mount_timeout": "10", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_cluster_require_signatures": "false", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_cluster_require_version": "2", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_require_signatures": "false", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_require_version": "2", 2024-08-17T17:17:14.196 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_service_require_signatures": "false", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_service_require_version": "2", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "cephx_sign_messages": "true", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "chdir": "", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_acl_type": "", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_asio_thread_count": "2", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_cache_mid": "0.750000", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_cache_size": "16384", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_caps_release_delay": "5", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_check_pool_perm": "true", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_collect_and_send_global_metrics": "false", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_debug_force_sync_read": "false", 2024-08-17T17:17:14.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_debug_getattr_caps": "false", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_debug_inject_features": "", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_debug_inject_tick_delay": "0", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_die_on_failed_remount": "false", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_dirsize_rbytes": "true", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_force_lazyio": "false", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_fs": "", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_inject_fixed_oldest_tid": "false", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_inject_release_failure": "false", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_max_inline_size": "4096", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_max_retries_on_remount_failure": "5", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_mds_namespace": "", 2024-08-17T17:17:14.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_metadata": "", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_mount_gid": "-1", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_mount_timeout": "300", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_mount_uid": "-1", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_mountpoint": "/", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_notify_timeout": "10", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_oc": "true", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_oc_max_dirty": "104857600", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_oc_max_dirty_age": "5.000000", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_oc_max_objects": "1000", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_oc_size": "209715200", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_oc_target_dirty": "8388608", 2024-08-17T17:17:14.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_permissions": "true", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_quota": "true", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_quota_df": "true", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_readahead_max_bytes": "0", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_readahead_max_periods": "4", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_readahead_min": "131072", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_reconnect_stale": "false", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_shutdown_timeout": "30", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_snapdir": ".snap", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_tick_interval": "1", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_trace": "", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_try_dentry_invalidate": "false", 2024-08-17T17:17:14.200 INFO:teuthology.orchestra.run.smithi142.stdout: "client_use_faked_inos": "false", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "client_use_random_mds": "false", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_graylog": "false", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_graylog_port": "12201", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_monitors": "default=true", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_syslog": "false", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "clog_to_syslog_level": "info", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "cluster_addr": "-", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "cluster_network": "", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "cluster_network_interface": "", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "compressor_zlib_isal": "false", 2024-08-17T17:17:14.201 INFO:teuthology.orchestra.run.smithi142.stdout: "compressor_zlib_level": "5", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "compressor_zlib_winsize": "-15", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "compressor_zstd_level": "1", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_alien_op_num_threads": "6", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_alien_thread_cpu_cores": "", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_osd_obc_lru_size": "512", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_osd_stat_interval": "0", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_seastar_cpu_cores": "", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crimson_seastar_num_threads": "0", 2024-08-17T17:17:14.202 INFO:teuthology.orchestra.run.smithi142.stdout: "crush_location": "", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "crush_location_hook": "", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "crush_location_hook_timeout": "10", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "daemonize": "false", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "daos_pool": "tank", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "dbstore_db_name_prefix": "dbstore", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_alienstore": "0/5", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_allow_any_pool_priority": "false", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_asok": "1/5", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_asok_assert_abort": "false", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_asserts_on_shutdown": "true", 2024-08-17T17:17:14.203 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_auth": "1/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_bdev": "1/3", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_bluefs": "1/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_bluestore": "1/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_buffer": "0/1", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_ceph_dedup": "0/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_ceph_exporter": "1/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_cephfs_mirror": "0/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_cephsqlite": "0/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_client": "0/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_compressor": "1/5", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_context": "0/1", 2024-08-17T17:17:14.204 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_crimson_interrupt": "0/5", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_crush": "1/1", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_crypto": "1/5", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_cyanstore": "0/5", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_deliberately_leak_memory": "false", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_disable_randomized_ping": "false", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_dpdk": "1/5", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_eventtrace": "1/5", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_filer": "0/1", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_filestore": "1/3", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_finisher": "1/1", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_fuse": "1/5", 2024-08-17T17:17:14.205 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_heartbeat_testing_span": "0", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_heartbeatmap": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_immutable_obj_cache": "0/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_javaclient": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_journal": "1/3", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_journaler": "0/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_kstore": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_lockdep": "0/1", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mclock": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds_balancer": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds_locker": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds_log": "1/5", 2024-08-17T17:17:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds_log_expire": "1/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds_migrator": "1/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mds_quiesce": "3/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_memstore": "1/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mgr": "2/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mgrc": "1/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_mon": "20/20", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_monc": "0/10", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_ms": "1/1", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_none": "0/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_objclass": "0/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_objectcacher": "0/5", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_objecter": "0/1", 2024-08-17T17:17:14.207 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_optracker": "0/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_osd": "1/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_paxos": "20/20", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_perfcounter": "1/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_prioritycache": "1/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rados": "0/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rbd": "0/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rbd_mirror": "0/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rbd_pwl": "0/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rbd_replay": "0/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_refs": "0/0", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_reserver": "1/1", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw": "1/5", 2024-08-17T17:17:14.208 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_access": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_datacache": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_dbstore": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_flight": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_lifecycle": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_notification": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rgw_sync": "1/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_rocksdb": "4/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore": "0/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_backref": "0/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_cache": "0/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_cleaner": "0/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_device": "0/5", 2024-08-17T17:17:14.209 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_epm": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_journal": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_lba": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_odata": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_omap": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_onode": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_t": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_seastore_tm": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_striper": "0/1", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_test": "0/5", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_throttle": "1/1", 2024-08-17T17:17:14.210 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_timer": "0/1", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_tp": "0/5", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "debug_trace": "1/5", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "device_failure_prediction_mode": "none", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "err_to_graylog": "false", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "err_to_journald": "false", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "err_to_stderr": "true", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "err_to_syslog": "false", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event_tracing": "false", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_addr": "0.0.0.0", 2024-08-17T17:17:14.211 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_cert_file": "", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_http_port": "9926", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_key_file": "", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_prio_limit": "5", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_sort_metrics": "true", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "exporter_stats_period": "5", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "fake_statfs_for_testing": "0", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "fatal_signal_handlers": "true", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filer_max_purge_ops": "10", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filer_max_truncate_ops": "128", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_apply_finisher_threads": "1", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_blackhole": "false", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_btrfs_clone_range": "true", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_btrfs_snap": "true", 2024-08-17T17:17:14.212 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_caller_concurrency": "10", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_collect_device_partition_information": "true", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_commit_timeout": "600.000000", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_debug_inject_read_err": "false", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_debug_omap_check": "false", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_debug_verify_split": "false", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_dump_file": "", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fadvise": "true", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fail_eio": "true", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fd_cache_shards": "16", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fd_cache_size": "128", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fiemap": "false", 2024-08-17T17:17:14.213 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fiemap_threshold": "4096", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_index_retry_probability": "0.000000", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_inject_stall": "0", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_journal_parallel": "false", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_journal_trailing": "false", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_journal_writeahead": "false", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_kill_at": "0", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattr_size": "0", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-08-17T17:17:14.214 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattrs": "0", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattrs_other": "2", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_sync_interval": "5.000000", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_xattr_value_size": "0", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_merge_threshold": "-10", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_min_sync_interval": "0.010000", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_odsync_write": "false", 2024-08-17T17:17:14.215 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_omap_backend": "rocksdb", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_omap_backend_path": "", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_omap_header_cache_size": "1024", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_ondisk_finisher_threads": "1", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_op_thread_timeout": "60", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_op_threads": "2", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_punch_hole": "false", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-08-17T17:17:14.216 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_max_bytes": "104857600", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_queue_max_ops": "50", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_seek_data_hole": "false", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_sloppy_crc": "false", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_splice": "false", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_split_multiple": "2", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_split_rand_factor": "20", 2024-08-17T17:17:14.217 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_update_to": "1000", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_enable": "true", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-08-17T17:17:14.218 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_xfs_extsize": "false", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "filestore_zfs_snap": "false", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fio_dir": "/tmp/fio", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fsid": "bcf463ee-5cb8-11ef-bccf-c7b262605968", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_allow_other": "true", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_atomic_o_trunc": "true", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_big_writes": "true", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_debug": "false", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_default_permissions": "false", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_disable_pagecache": "false", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_max_write": "0", 2024-08-17T17:17:14.219 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_multithreaded": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_require_active_mds": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_set_user_groups": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_splice_move": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_splice_read": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_splice_write": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_syncfs_on_mksnap": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "fuse_use_invalidate_cb": "true", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "gss_target_name": "ceph", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "heartbeat_file": "", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "heartbeat_inject_failure": "0", 2024-08-17T17:17:14.220 INFO:teuthology.orchestra.run.smithi142.stdout: "heartbeat_interval": "5", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "host": "smithi142", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_max_size": "1073741824", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-08-17T17:17:14.221 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "immutable_object_cache_watermark": "0.900000", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "inject_early_sigterm": "false", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "jaeger_agent_port": "6799", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "jaeger_tracing_enable": "false", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_aio": "true", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_align_min_size": "65536", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_block_align": "true", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_block_size": "4096", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_dio": "true", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_discard": "false", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_force_aio": "false", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_ignore_corruption": "false", 2024-08-17T17:17:14.222 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_max_write_bytes": "10485760", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_max_write_entries": "100", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_replay_from": "0", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_throttle_high_multiple": "0.000000", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_throttle_max_multiple": "0.000000", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_write_header_frequency": "0", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journal_zero_on_create": "false", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journaler_prefetch_periods": "10", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journaler_prezero_periods": "5", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "journaler_write_head_interval": "15", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "key": "", 2024-08-17T17:17:14.223 INFO:teuthology.orchestra.run.smithi142.stdout: "keyfile": "", 2024-08-17T17:17:14.224 INFO:teuthology.orchestra.run.smithi142.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-08-17T17:17:14.224 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_backend": "rocksdb", 2024-08-17T17:17:14.224 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_default_stripe_size": "65536", 2024-08-17T17:17:14.224 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_fsck_on_mount": "false", 2024-08-17T17:17:14.225 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_fsck_on_mount_deep": "true", 2024-08-17T17:17:14.225 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_max_bytes": "67108864", 2024-08-17T17:17:14.225 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_max_ops": "512", 2024-08-17T17:17:14.225 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_nid_prealloc": "1024", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_onode_map_size": "1024", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_sync_submit_transaction": "false", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "kstore_sync_transaction": "false", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "librados_thread_count": "2", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "lockdep": "false", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "lockdep_force_backtrace": "false", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "log_coarse_timestamps": "true", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "log_flush_on_exit": "false", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "log_graylog_host": "127.0.0.1", 2024-08-17T17:17:14.226 INFO:teuthology.orchestra.run.smithi142.stdout: "log_graylog_port": "12201", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_max_new": "1000", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_max_recent": "10000", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_stderr_prefix": "", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_stop_at_utilization": "0.970000", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_to_file": "true", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_to_graylog": "false", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_to_journald": "false", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_to_stderr": "false", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "log_to_syslog": "false", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "max_rotating_auth_attempts": "10", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_action_on_write_error": "1", 2024-08-17T17:17:14.227 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_alternate_name_max": "8192", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_asio_thread_count": "2", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_export_pin": "true", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_fragment_dirs": "true", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_fragment_interval": "5", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_fragment_size_max": "100000", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_idle_threshold": "0.000000", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_interval": "10", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_max": "-1", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_max_until": "-1", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_merge_size": "50", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_midchunk": "0.300000", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_min_rebalance": "0.100000", 2024-08-17T17:17:14.228 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_min_start": "0.200000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_minchunk": "0.001000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_mode": "0", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_need_max": "1.200000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_need_min": "0.800000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_overload_epochs": "2", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_sample_interval": "3.000000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_split_bits": "3", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_split_rd": "25000.000000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_split_size": "10000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_split_wr": "10000.000000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_target_decay": "10.000000", 2024-08-17T17:17:14.229 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_beacon_grace": "15.000000", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_beacon_interval": "4.000000", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_beacon_mon_down_grace": "60", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_memory_limit": "4294967296", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_mid": "0.700000", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_quiesce_delay": "0", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_quiesce_sleep": "200", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_quiesce_splitauth": "true", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_quiesce_threshold": "524288", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_release_free_interval": "10", 2024-08-17T17:17:14.230 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_reservation": "0.050000", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_trim_interval": "1", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cache_trim_threshold": "262144", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_client_delegate_inos_pct": "50", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_client_prealloc_inos": "1000", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_connect_bootstrapping": "false", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_damage_table_max_entries": "10000", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_debug_auth_pins": "false", 2024-08-17T17:17:14.231 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_debug_frag": "false", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_debug_scatterstat": "false", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_debug_subtrees": "false", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_decay_halflife": "5.000000", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_default_dir_hash": "2", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_defer_session_stale": "true", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_deny_all_reconnect": "false", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dir_keys_per_op": "16384", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dir_max_commit_size": "10", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dir_max_entries": "0", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dir_prefetch": "true", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dirstat_min_interval": "1.000000", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dump_cache_after_rejoin": "false", 2024-08-17T17:17:14.232 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dump_cache_on_map": "false", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dump_cache_threshold_file": "0", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_early_reply": "true", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_enable_op_tracker": "true", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_enforce_unique_name": "true", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_export_ephemeral_distributed": "true", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_export_ephemeral_random": "true", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_forward_all_requests_to_auth": "false", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-08-17T17:17:14.233 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_health_cache_threshold": "1.500000", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_health_summarize_threshold": "10", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_heartbeat_grace": "15.000000", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_heartbeat_reset_grace": "1000", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_inject_health_dummy": "false", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_inject_migrator_session_race": "false", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-08-17T17:17:14.234 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_join_fs": "", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_journal_format": "1", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_create_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_dirfrag_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_export_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_import_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_journal_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_journal_expire_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_journal_replay_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_link_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_mdstable_at": "0", 2024-08-17T17:17:14.235 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_openc_at": "0", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_rename_at": "0", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_kill_shutdown_at": "0", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_event_large_threshold": "524288", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_events_per_segment": "1024", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_major_segment_event_ratio": "12", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_max_events": "-1", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_max_segments": "128", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_pause": "false", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_segment_size": "0", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_skip_corrupt_events": "false", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_skip_unbounded_events": "false", 2024-08-17T17:17:14.236 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_trim_threshold": "128", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_log_warn_factor": "2.000000", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_caps_per_client": "1048576", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_completed_flushes": "100000", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_completed_requests": "100000", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_export_size": "20971520", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_file_recover": "32", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_purge_files": "64", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_purge_ops": "8192", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_max_snaps_per_dir": "100", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_metrics_update_interval": "2", 2024-08-17T17:17:14.237 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_min_caps_per_client": "100", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_min_caps_working_set": "10000", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_numa_node": "-1", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_op_complaint_time": "30.000000", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_op_history_duration": "600", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_op_history_size": "20", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_op_history_slow_op_size": "20", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_op_history_slow_op_threshold": "10", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_op_log_threshold": "5", 2024-08-17T17:17:14.238 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_ping_grace": "15", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_ping_interval": "5", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_recall_max_caps": "30000", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_recall_max_decay_threshold": "131072", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_recall_warning_threshold": "262144", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_reconnect_timeout": "45.000000", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_replay_interval": "1.000000", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-08-17T17:17:14.239 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_root_ino_gid": "0", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_root_ino_uid": "0", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_blocklist_on_evict": "true", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_blocklist_on_timeout": "true", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-08-17T17:17:14.240 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_session_metadata_threshold": "16777216", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_shutdown_check": "0", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_skip_ino": "0", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_sleep_rank_change": "0.000000", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_snap_max_uid": "4294967294", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_snap_min_uid": "0", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_snap_rstat": "false", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_standby_replay_damaged": "false", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_symlink_recovery": "true", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_task_status_update_interval": "2.000000", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_thrash_exports": "0", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_thrash_fragments": "0", 2024-08-17T17:17:14.241 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_tick_interval": "5.000000", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_valgrind_exit": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_verify_backtrace": "1", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_verify_scatter": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_wipe_ino_prealloc": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mds_wipe_sessions": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mempool_debug": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "memstore_debug_omit_block_device_write": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "memstore_device_bytes": "1073741824", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "memstore_page_set": "false", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "memstore_page_size": "65536", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_client_bytes": "134217728", 2024-08-17T17:17:14.242 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_client_messages": "512", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_connect_retry_interval": "1.000000", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_disabled_modules": "", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_enable_op_tracker": "true", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_max_pg_creating": "1024", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_max_pg_num_change": "128", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_mds_bytes": "134217728", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_mds_messages": "128", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-08-17T17:17:14.243 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_mon_bytes": "134217728", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_mon_messages": "128", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_num_op_tracker_shard": "32", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_op_complaint_time": "30.000000", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_op_history_duration": "600", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_op_history_size": "20", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_op_history_slow_op_size": "20", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_op_log_threshold": "5", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_osd_bytes": "536870912", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_osd_messages": "8192", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_pool": "true", 2024-08-17T17:17:14.244 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_service_beacon_grace": "60.000000", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_standby_modules": "true", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_stats_period": "5", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_stats_threshold": "5", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_tick_period": "2", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_accept_timeout_factor": "2.000000", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_allow_pool_delete": "true", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_allow_pool_size_one": "false", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_auth_validate_all_caps": "true", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_bytes": "104857600", 2024-08-17T17:17:14.245 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_directed_command_retry": "2", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_hunt_interval": "3.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_hunt_parallel": "3", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_log_interval": "1.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_ping_interval": "10.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_ping_timeout": "30.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_client_target_rank": "-1", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_clock_drift_allowed": "1.000000", 2024-08-17T17:17:14.246 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_level": "debug", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_file": "true", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_graylog": "false", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_journald": "true", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_stderr": "false", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_compact_on_bootstrap": "false", 2024-08-17T17:17:14.247 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_compact_on_start": "false", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_compact_on_trim": "true", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_con_tracker_persist_interval": "10", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_con_tracker_score_halflife": "43200", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_config_key_max_entry_size": "65536", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_cpu_threads": "4", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_crush_min_required_version": "hammer", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_daemon_bytes": "419430400", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_data_avail_crit": "5", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_data_avail_warn": "5", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_data_size_warn": "16106127360", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_block_osdmap_trim": "false", 2024-08-17T17:17:14.248 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_dump_json": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_dump_transactions": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_extra_checks": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_no_require_reef": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_no_require_squid": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_delta_reset_interval": "10.000000", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_dns_srv_name": "ceph-mon", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_down_mkfs_grace": "60", 2024-08-17T17:17:14.249 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_down_uptime_grace": "60", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_election_default_strategy": "3", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_election_timeout": "5.000000", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_elector_ping_divisor": "2", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_elector_ping_timeout": "2.000000", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_enable_op_tracker": "true", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_fake_pool_delete": "false", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_force_quorum_join": "false", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_fsmap_prune_threshold": "300", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_globalid_prealloc": "10000", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_health_detail_to_clog": "true", 2024-08-17T17:17:14.250 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_health_log_update_period": "5", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_health_max_detail": "50", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_health_to_clog": "true", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_health_to_clog_interval": "600", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_host": "[v2:172.21.15.142:3300/0,v1:172.21.15.142:6789/0] [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.142:3301/0,v1:172.21.15.142:6790/0]", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_host_override": "", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_initial_members": "", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_keyvaluedb": "rocksdb", 2024-08-17T17:17:14.251 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_lease": "5.000000", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_log_full_interval": "50", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_log_max": "10000", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_log_max_summary": "50", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_log_entries_per_event": "4096", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_log_epochs": "500", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_mdsmap_epochs": "500", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_mgrmap_epochs": "500", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_nvmeof_epochs": "500", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_osd": "10000", 2024-08-17T17:17:14.252 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_pg_per_osd": "10000", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_pool_pg_num": "65536", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mds_force_trim_to": "0", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mds_skip_sanity": "true", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_memory_autotune": "true", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_memory_target": "2147483648", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mgr_beacon_grace": "30", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mgr_digest_period": "5", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mgr_inactive_grace": "60", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mgr_mkfs_grace": "240", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-08-17T17:17:14.253 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_min_osdmap_epochs": "500", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_op_complaint_time": "30", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_op_history_duration": "600", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_op_history_size": "20", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_op_history_slow_op_size": "20", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_op_history_slow_op_threshold": "10", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_op_log_threshold": "5", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-08-17T17:17:14.254 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_auto_mark_in": "false", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_auto_mark_new_in": "true", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_cache_size": "500", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_cache_size_min": "134217728", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_crush_smoke_test": "true", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_destroyed_out_interval": "600", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_down_out_interval": "600", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_force_trim_to": "0", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_full_ratio": "0.950000", 2024-08-17T17:17:14.255 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_laggy_halflife": "3600", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_laggy_max_interval": "300", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_laggy_weight": "0.300000", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_max_initial_pgs": "1024", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_min_down_reporters": "2", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_prime_pg_temp": "true", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-08-17T17:17:14.256 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_report_timeout": "900", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_warn_num_repaired": "10", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osd_warn_op_age": "32.000000", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osdmap_full_prune_interval": "10", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osdmap_full_prune_min": "10000", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pg_stuck_threshold": "60", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-08-17T17:17:14.257 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pg_warn_min_objects": "10000", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pg_warn_min_per_osd": "0", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pool_quota_crit_threshold": "0", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_pool_quota_warn_threshold": "0", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_probe_timeout": "2.000000", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_reweight_max_change": "0.050000", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_reweight_max_osds": "4", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-08-17T17:17:14.258 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_scrub_interval": "86400", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_scrub_max_keys": "100", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_scrub_timeout": "300", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_session_timeout": "300", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_smart_report_timeout": "5", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_stat_smooth_intervals": "6", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_stretch_pool_min_size": "2", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_stretch_pool_size": "4", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_subscribe_interval": "86400.000000", 2024-08-17T17:17:14.259 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_sync_debug": "false", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_sync_max_payload_keys": "2000", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_sync_max_payload_size": "1048576", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_sync_provider_kill_at": "0", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_sync_requester_kill_at": "0", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_sync_timeout": "60.000000", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_target_pg_per_osd": "100", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_tick_interval": "5", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_timecheck_interval": "300.000000", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_use_min_delay_socket": "false", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_older_version_delay": "604800", 2024-08-17T17:17:14.260 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_filestore_osds": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_misplaced": "false", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_older_version": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_pool_no_app": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-08-17T17:17:14.261 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_on_too_few_osds": "false", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "monmap": "", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-08-17T17:17:14.262 INFO:teuthology.orchestra.run.smithi142.stdout: "motr_tracing_enabled": "false", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_op_threads": "3", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_buffer_size": "131072", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_cm": "false", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_device_name": "", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_dscp": "96", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_gid_idx": "0", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_local_gid": "", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_polling_us": "1000", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_port_num": "1", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-08-17T17:17:14.263 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_roce_ver": "1", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_send_buffers": "1024", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_sl": "3", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_support_srq": "true", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_rdma_type": "ib", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_async_reap_threshold": "5", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_before_connect": "false", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_ipv4": "true", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_ipv6": "false", 2024-08-17T17:17:14.264 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_msgr1": "true", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_msgr2": "true", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_port_max": "7568", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_port_min": "6800", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_prefer_ipv4": "false", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_retry_count": "3", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_bind_retry_delay": "5", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_blackhole_client": "false", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_blackhole_mds": "false", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_blackhole_mgr": "false", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_blackhole_mon": "false", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_blackhole_osd": "false", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_client_mode": "crc secure", 2024-08-17T17:17:14.265 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_cluster_mode": "crc secure", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_cluster_type": "", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_compress_secure": "false", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_connection_idle_timeout": "900", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_connection_ready_timeout": "10", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_crc_data": "true", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_crc_header": "true", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_die_on_bad_msg": "false", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_die_on_bug": "true", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_die_on_old_message": "true", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_die_on_skipped_message": "false", 2024-08-17T17:17:14.266 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_die_on_unhandled_msg": "false", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_coremask": "0xF", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_devs_allowlist": "", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_enable_tso": "true", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_hugepages": "", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_hw_flow_control": "true", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_lro": "true", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_memory_channel": "4", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-08-17T17:17:14.267 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_pmd": "", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_port_id": "0", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dump_corrupt_message_level": "1", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_dump_on_send": "false", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_initial_backoff": "0.200000", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_inject_delay_max": "1.000000", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_inject_delay_probability": "0.000000", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_inject_delay_type": "", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_inject_internal_delays": "0.000000", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_inject_network_congestion": "0", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_inject_socket_failures": "0", 2024-08-17T17:17:14.268 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_learn_addr_from_peer": "true", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_max_accept_failures": "4", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_max_backoff": "15.000000", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_mon_client_mode": "secure crc", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_mon_cluster_mode": "secure crc", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_mon_service_mode": "secure crc", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_osd_compress_min_size": "1024", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_osd_compress_mode": "none", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_osd_compression_algorithm": "snappy", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_pq_min_cost": "65536", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_public_type": "", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_service_mode": "crc secure", 2024-08-17T17:17:14.269 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_tcp_listen_backlog": "512", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_tcp_nodelay": "true", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_tcp_rcvbuf": "0", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_time_events_min_wait_interval": "1000", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "ms_type": "async+posix", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "no_config_file": "false", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "nvmeof_mon_client_tick_period": "2", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_completion_locks_per_session": "32", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_debug_inject_relock_delay": "false", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_inflight_op_bytes": "104857600", 2024-08-17T17:17:14.270 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_inflight_ops": "1024", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_inject_no_watch_ping": "false", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_tick_interval": "5.000000", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "objecter_timeout": "10.000000", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "objectstore_blackhole": "false", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "openssl_engine_opts": "", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_delay_time": "5.000000", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_hist_halflife": "1000", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_max_low_ops": "2", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_max_ops": "4", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-08-17T17:17:14.271 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_quantize_effort": "0.100000", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_agent_slop": "0.020000", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_allow_recovery_below_min_size": "true", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_asio_thread_count": "2", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_async_recovery_min_cost": "100", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_auto_mark_unfound_lost": "false", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_backfill_retry_interval": "30.000000", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_backfill_scan_max": "512", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_backfill_scan_min": "64", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_backoff_on_degraded": "false", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_backoff_on_peering": "false", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_backoff_on_unfound": "true", 2024-08-17T17:17:14.272 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_beacon_report_interval": "300", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_bench_duration": "30", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_bench_max_block_size": "67108864", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_bench_small_size_max_iops": "100", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_blkin_trace_all": "false", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_blocked_scrub_grace_period": "120", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_check_for_log_corruption": "false", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:17:14.273 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_class_update_on_start": "true", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_client_message_cap": "256", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_client_message_size_cap": "524288000", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_client_op_priority": "63", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_client_watch_timeout": "30", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_command_max_records": "256", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_compact_on_start": "false", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_crush_chooseleaf_type": "1", 2024-08-17T17:17:14.274 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_crush_initial_weight": "-1.000000", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_crush_update_on_start": "true", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_crush_update_weight_set": "true", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_drop_ping_duration": "0", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_feed_pullee": "-1", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_misdirected_ops": "false", 2024-08-17T17:17:14.275 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_no_acting_change": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_no_purge_strays": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_op_order": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_pg_log_writeout": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_pretend_recovery_active": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_shutdown": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_trim_objects": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_verify_cached_snaps": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_verify_missing_on_start": "false", 2024-08-17T17:17:14.276 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_verify_snaps": "false", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_keys": "1024", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_stride": "524288", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_default_data_pool_replay_window": "45", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_default_notify_timeout": "30", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_delete_sleep": "0.000000", 2024-08-17T17:17:14.277 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_discard_disconnected_ops": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_ec_partial_reads": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_enable_op_tracker": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_extblkdev_plugins": "vdo", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_fast_info": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_fast_shutdown": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-08-17T17:17:14.278 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_fast_shutdown_timeout": "15", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_function_tracing": "false", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_grace": "20", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_interval": "6", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_min_peers": "10", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_min_size": "2000", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_stale": "600", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_hit_set_max_size": "100000", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_hit_set_min_size": "1000", 2024-08-17T17:17:14.279 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_journal_flush_on_shutdown": "true", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_journal_size": "5120", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_kill_backfill_at": "0", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_loop_before_reset_tphandle": "64", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_map_cache_size": "50", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_map_dedup": "true", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_map_message_max": "40", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_map_message_max_bytes": "10485760", 2024-08-17T17:17:14.280 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_map_share_max_epochs": "40", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_attr_name_len": "100", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_attr_size": "0", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_backfills": "1", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_markdown_count": "5", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_markdown_period": "600", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_object_name_len": "2048", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_object_namespace_len": "256", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_object_size": "134217728", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_omap_entries_per_request": "1024", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_pg_blocked_by": "16", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_pg_log_entries": "10000", 2024-08-17T17:17:14.281 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_pgls": "1024", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_push_cost": "8388608", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_push_objects": "10", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_scrubs": "3", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_trimming_pgs": "2", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_write_op_reply_len": "64", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_max_write_size": "90", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-08-17T17:17:14.282 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_override_recovery_settings": "false", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_profile": "balanced", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-08-17T17:17:14.283 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mclock_skip_benchmark": "false", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_base": "805306368", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_cache_min": "134217728", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_target": "4294967296", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_target_autotune": "false", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_min_pg_log_entries": "250", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_min_recovery_priority": "0", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mon_heartbeat_interval": "30", 2024-08-17T17:17:14.284 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mon_report_interval": "5", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mon_report_max_in_flight": "2", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_num_cache_shards": "32", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_num_op_tracker_shard": "32", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_numa_auto_affinity": "true", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_numa_node": "-1", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_numa_prefer_iface": "true", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_objecter_finishers": "1", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_objectstore": "bluestore", 2024-08-17T17:17:14.285 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_objectstore_fuse": "false", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_objectstore_ideal_list_max": "64", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_objectstore_tracing": "false", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_complaint_time": "30.000000", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_history_duration": "600", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_history_size": "20", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_history_slow_op_size": "20", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_log_threshold": "5", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_num_shards": "0", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_num_shards_hdd": "5", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_num_shards_ssd": "8", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_num_threads_per_shard": "0", 2024-08-17T17:17:14.286 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_pq_min_cost": "65536", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_queue": "mclock_scheduler", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_queue_cut_off": "high", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_thread_suicide_timeout": "150", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_op_thread_timeout": "15", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_open_classes_on_start": "true", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_os_flags": "0", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_peering_op_priority": "255", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_delete_cost": "1048576", 2024-08-17T17:17:14.287 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_delete_priority": "5", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_log_dups_tracked": "3000", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_log_trim_max": "10000", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_log_trim_min": "100", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_object_context_cache_count": "64", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-08-17T17:17:14.288 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_crimson": "false", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_crush_rule": "-1", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_ec_fast_read": "false", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_flag_bulk": "false", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_flag_nodelete": "false", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_flags": "0", 2024-08-17T17:17:14.289 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_min_size": "0", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_pg_num": "32", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_pgp_num": "0", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_read_ratio": "70", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_size": "3", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_default_type": "replicated", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_pool_use_gmt_hitset": "true", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_push_per_object_cost": "1000", 2024-08-17T17:17:14.290 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_read_ec_check_for_errors": "false", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recover_clone_overlap": "true", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recover_clone_overlap_limit": "10", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_cost": "20971520", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_delay_start": "0.000000", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_max_active": "0", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_max_active_hdd": "3", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_max_active_ssd": "10", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_max_chunk": "8388608", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_max_single_start": "1", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_op_priority": "3", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_op_warn_multiple": "16", 2024-08-17T17:17:14.291 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_priority": "5", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_retry_interval": "30.000000", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_sleep": "0.000000", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_repair_during_recovery": "false", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_requested_scrub_priority": "120", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_rollback_to_cluster_snap": "", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_auto_repair": "false", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-08-17T17:17:14.292 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_begin_hour": "0", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_begin_week_day": "0", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_chunk_max": "25", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_chunk_min": "5", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_cost": "52428800", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_during_recovery": "false", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_end_hour": "0", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_end_week_day": "0", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_event_cost": "4096", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-08-17T17:17:14.293 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_invalid_stats": "true", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_load_threshold": "0.500000", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_max_interval": "604800.000000", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_max_preemptions": "5", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_min_interval": "86400.000000", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_priority": "5", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_scrub_sleep": "0.000000", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_shutdown_pgref_assert": "false", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_skip_data_digest": "false", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_smart_report_timeout": "5", 2024-08-17T17:17:14.294 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_snap_trim_cost": "1048576", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_snap_trim_priority": "5", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_snap_trim_sleep": "0.000000", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_stats_update_period_scrubbing": "15", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_target_transaction_size": "30", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-08-17T17:17:14.295 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_default_cache_mode": "writeback", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_tracing": "false", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_use_stale_snap": "false", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "osdc_blkin_trace_all": "false", 2024-08-17T17:17:14.296 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_kill_at": "0", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_max_join_drift": "10", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_min": "500", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_min_wait": "0.050000", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_propose_interval": "1.000000", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_service_trim_max": "500", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_service_trim_max_multiplier": "20", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_service_trim_min": "250", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_stash_full_interval": "25", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_trim_max": "500", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "paxos_trim_min": "250", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "perf": "true", 2024-08-17T17:17:14.297 INFO:teuthology.orchestra.run.smithi142.stdout: "pid_file": "", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "public_addr": "-", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "public_addrv": "", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "public_bind_addr": "-", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "public_network": "172.21.0.0/20", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "public_network_interface": "", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "qat_compressor_busy_polling": "false", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "qat_compressor_enabled": "false", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "qat_compressor_session_max_number": "256", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "rados_mon_op_timeout": "0", 2024-08-17T17:17:14.298 INFO:teuthology.orchestra.run.smithi142.stdout: "rados_osd_op_timeout": "0", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rados_tracing": "false", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_atime_update_interval": "60", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_balance_parent_reads": "false", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_balance_snap_reads": "false", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_blkin_trace_all": "false", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_blocklist_expire_seconds": "0", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_blocklist_on_break_lock": "true", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache": "true", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_block_writes_upfront": "false", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_max_dirty": "25165824", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-08-17T17:17:14.299 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_max_dirty_object": "0", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_policy": "writearound", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_size": "33554432", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_target_dirty": "16777216", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_clone_copy_on_read": "false", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_compression_hint": "none", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_concurrent_management_ops": "10", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_clone_format": "auto", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_data_pool": "", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_features": "61", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_format": "2", 2024-08-17T17:17:14.300 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_map_options": "", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_order": "22", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_pool": "rbd", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_stripe_count": "0", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_default_stripe_unit": "0", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_disable_zero_copy_writes": "true", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_discard_granularity_bytes": "65536", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_enable_alloc_hint": "true", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_io_scheduler": "simple", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-08-17T17:17:14.301 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_commit_age": "5.000000", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_object_flush_interval": "0", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_order": "24", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_pool": "", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_journal_splay_width": "4", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_localize_parent_reads": "false", 2024-08-17T17:17:14.302 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_localize_snap_reads": "false", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_image_policy_type": "simple", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-08-17T17:17:14.303 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_autotune": "true", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_base": "805306368", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_memory_target": "4294967296", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-08-17T17:17:14.304 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirroring_delete_delay": "0", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirroring_replay_delay": "0", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_move_to_trash_on_remove": "false", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_mtime_update_interval": "60", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_non_blocking_aio": "true", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_op_thread_timeout": "60", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_op_threads": "1", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_parent_cache_enabled": "false", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_persistent_cache_mode": "disabled", 2024-08-17T17:17:14.305 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_persistent_cache_path": "/tmp", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_persistent_cache_size": "1073741824", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_plugins": "", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_bps_burst": "0", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_bps_limit": "0", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_exclude_ops": "0", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_iops_burst": "0", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_iops_limit": "0", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_read_bps_burst": "0", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-08-17T17:17:14.306 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_read_bps_limit": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_read_iops_burst": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_read_iops_limit": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_schedule_tick_min": "50", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_write_bps_burst": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_write_bps_limit": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_write_iops_burst": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_qos_write_iops_limit": "0", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_quiesce_notification_attempts": "10", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_read_from_replica_policy": "default", 2024-08-17T17:17:14.307 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_readahead_max_bytes": "524288", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_readahead_trigger_requests": "10", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_request_timed_out_seconds": "30", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_skip_partial_discard": "true", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_tracing": "false", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_validate_names": "true", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rbd_validate_pool": "true", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "restapi_base_url": "", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "restapi_log_level": "", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-08-17T17:17:14.308 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_account_default_quota_max_size": "-1", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_acl_grants_max_num": "100", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_admin_entry": "admin", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_asio_assert_yielding": "false", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_backend_store": "rados", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_barbican_url": "", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_beast_enable_async": "true", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_counters_cache": "false", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-08-17T17:17:14.309 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_index_max_aio": "128", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_quota_ttl": "600", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_cache_enabled": "true", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_cache_expiry_interval": "900", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_cache_lru_size": "25000", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_config_store": "rados", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_content_length_compat": "false", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_copy_obj_progress": "true", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_cors_rules_max_num": "100", 2024-08-17T17:17:14.310 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_cross_domain_policy": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_default_encryption_key": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_addr": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_ca_path": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_client_cert": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_client_key": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_password": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_kmip_username": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_require_ssl": "true", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-08-17T17:17:14.311 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_suppress_logs": "true", 2024-08-17T17:17:14.312 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_addr": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_auth": "token", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_namespace": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_prefix": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_token_file": "", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_curl_buffersize": "524288", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_curl_low_speed_limit": "1024", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_curl_low_speed_time": "300", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_curl_tcp_keepalive": "0", 2024-08-17T17:17:14.313 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_l1_fadvise": "4", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-08-17T17:17:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_l1_fadvise": "4", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_log_changes_size": "1000", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_log_num_shards": "128", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_log_window": "30", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_notify_interval_msec": "0", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_sync_poll_interval": "20", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_data_sync_spawn_window": "20", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-08-17T17:17:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_default_data_log_backing": "fifo", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_default_region_info_oid": "default.region", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_defer_to_bucket_acls": "", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_disable_s3select": "false", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-08-17T17:17:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_data_res": "500.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dns_name": "", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dns_s3website_name": "", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dynamic_resharding": "true", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-08-17T17:17:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_gc_threads": "true", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_lc_threads": "true", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_mdsearch": "true", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_ops_log": "false", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_quota_threads": "true", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_static_website": "false", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enable_usage_log": "false", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_enforce_swift_acls": "true", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_exit_timeout_secs": "120", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_expose_bucket": "false", 2024-08-17T17:17:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_extended_http_attrs": "", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_filter": "none", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_frontends": "beast port=7480", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_max_concurrent_io": "10", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_max_deferred": "50", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_max_objs": "32", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_max_queue_size": "134213632", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_max_trim_chunk": "16", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_obj_min_wait": "7200", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_processor_max_time": "3600", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_gc_processor_period": "3600", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-08-17T17:17:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_get_obj_window_size": "16777216", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_healthcheck_disabling_path": "", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ignore_get_invalid_range": "false", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_init_timeout": "300", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_kafka_connection_idle": "300", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_kafka_message_timeout": "5000", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_kafka_sleep_timeout": "10", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_domain": "", 2024-08-17T17:17:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_password": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_password_path": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_project": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_tenant": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_token": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_token_path": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_admin_user": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_api_version": "2", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_barbican_domain": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_barbican_password": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_barbican_project": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_barbican_tenant": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_barbican_user": "", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-08-17T17:17:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_implicit_tenants": "false", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_service_token_enabled": "false", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_token_cache_size": "10000", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_url": "", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_keystone_verify_ssl": "true", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_debug_interval": "-1", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_lock_max_time": "90", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_max_objs": "32", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_max_rules": "1000", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_max_worker": "3", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_max_wp_worker": "3", 2024-08-17T17:17:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lc_thread_delay": "0", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ldap_dnattr": "uid", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ldap_searchfilter": "", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ldap_uri": "", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lfuda_sync_frequency": "60", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_log_http_headers": "", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_log_nonexistent_bucket": "false", 2024-08-17T17:17:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_log_object_name_utc": "false", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_attr_name_len": "0", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_attr_size": "0", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_attrs_num_in_req": "0", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_chunk_size": "4194304", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_concurrent_requests": "1024", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_dynamic_shards": "1999", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_listing_results": "1000", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_notify_retries": "10", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_objs_per_shard": "100000", 2024-08-17T17:17:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_put_param_size": "1048576", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_put_size": "5368709120", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_max_slo_entries": "1000", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_md_log_max_shards": "64", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_md_notify_interval_msec": "200", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_meta_sync_poll_interval": "20", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_meta_sync_spawn_window": "20", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_mp_lock_max_time": "600", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_multipart_min_part_size": "5242880", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-08-17T17:17:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_fhcache_size": "2017", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_lru_lanes": "5", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_max_gc": "300", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_run_gc_threads": "false", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_run_lc_threads": "false", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_run_quota_threads": "false", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_run_sync_thread": "false", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_num_async_rados_threads": "32", 2024-08-17T17:17:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_num_control_oids": "8", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_numa_node": "-1", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_obj_stripe_size": "4194304", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_objexp_chunk_size": "100", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_objexp_gc_interval": "600", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_objexp_hints_num_shards": "127", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_op_thread_timeout": "600", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_op_tracing": "false", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_opa_token": "", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_opa_url": "", 2024-08-17T17:17:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_opa_verify_ssl": "true", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ops_log_rados": "false", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_ops_log_socket_path": "", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_period_push_interval": "2.000000", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_period_push_interval_max": "30.000000", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_period_root_pool": ".rgw.root", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-08-17T17:17:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_posix_cache_lanes": "3", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_posix_cache_max_buckets": "100", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_posix_cache_partitions": "3", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_print_continue": "true", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_print_prohibited_content_length": "false", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_rados_tracing": "false", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_realm": "", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_realm_id": "", 2024-08-17T17:17:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_region": "", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_region_root_pool": ".rgw.root", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_relaxed_region_enforcement": "false", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_request_uri": "", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_reshard_batch_size": "64", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_reshard_debug_interval": "-1", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_reshard_max_aio": "128", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_reshard_num_logs": "16", 2024-08-17T17:17:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_reshard_thread_interval": "600", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_resolve_cname": "false", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_rest_getusage_op_compat": "false", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_run_sync_thread": "true", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_auth_use_keystone": "false", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_auth_use_ldap": "false", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_auth_use_rados": "true", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_auth_use_sts": "false", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_s3_success_create_obj_status": "0", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-08-17T17:17:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_scheduler_type": "throttler", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_script_uri": "", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_service_provider_name": "", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_client_id": "", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_client_secret": "", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_entry": "sts", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_key": "sts", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_max_session_duration": "43200", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_min_session_duration": "900", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sts_token_introspection_url": "", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_account_in_url": "false", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_auth_entry": "auth", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_auth_url": "", 2024-08-17T17:17:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_custom_header": "", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_enforce_content_length": "false", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_need_stats": "true", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_tenant_name": "", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_token_expiration": "86400", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_url": "", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_url_prefix": "swift", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_swift_versioning_enabled": "false", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_lease_period": "120", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_log_trim_interval": "1200", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-08-17T17:17:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_obj_etag_verify": "false", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_trace_history_size": "4096", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_thread_pool_size": "512", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_topic_persistency_max_retries": "0", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_topic_require_publish_policy": "false", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_comment": "", 2024-08-17T17:17:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_createby": "", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_encoding": "", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_flag": "false", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_max_size": "5368709120", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_origin": "", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_sha_unit": "524288", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_torrent_tracker": "", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_trust_forwarded_https": "false", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_usage_log_tick_interval": "30", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_usage_max_shards": "32", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_usage_max_user_shards": "1", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_use_opa_authz": "false", 2024-08-17T17:17:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_counters_cache": "false", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_counters_cache_size": "10000", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_default_quota_max_size": "-1", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_max_buckets": "1000", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_policies_max_num": "100", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_quota_sync_interval": "86400", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_user_unique_email": "true", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_verify_ssl": "true", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_website_routing_rules_max_num": "50", 2024-08-17T17:17:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_zone": "", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_zone_id": "", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_zonegroup": "", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_zonegroup_id": "", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgwlc_auto_session_clear": "true", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rgwlc_skip_bucket_step": "false", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_block_size": "4096", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-08-17T17:17:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cache_shard_bits": "4", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cache_size": "536870912", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cache_type": "binned_lru", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_collect_compaction_stats": "false", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_collect_extended_stats": "false", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_collect_memory_stats": "false", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_index_type": "binary_search", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_log_to_ceph_log": "true", 2024-08-17T17:17:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_metadata_block_size": "4096", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_partition_filters": "false", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_perf": "false", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "rotating_keys_renewal_timeout": "10", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "run_dir": "/var/run/ceph", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_block_create": "true", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_cache_lru_size": "67108864", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_cbjournal_size": "5368709120", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_data_delta_based_overwrite": "0", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_default_max_object_size": "16777216", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-08-17T17:17:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_device_size": "53687091200", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_full_integrity_check": "false", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_journal_batch_capacity": "16", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_journal_iodepth_limit": "5", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_main_device_type": "SSD", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_max_concurrent_transactions": "8", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_max_data_allocation_size": "32768", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-08-17T17:17:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "seastore_segment_size": "67108864", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "set_keepcaps": "false", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "setgroup": "ceph", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "setuser": "ceph", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "setuser_match_path": "", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "target_max_misplaced_ratio": "0.050000", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "thp": "false", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "threadpool_default_timeout": "60", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "threadpool_empty_queue_max_wait": "2", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "throttler_perf_counter": "true", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "tmp_dir": "/tmp", 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX" 2024-08-17T17:17:14.341 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:17:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:14 smithi142 ceph-mon[106729]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:14 smithi142 ceph-mon[103614]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:15.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-08-17T17:17:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:14 smithi152 ceph-mon[93456]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:16.251 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:17.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:16 smithi152 ceph-mon[93456]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:17.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:16 smithi142 ceph-mon[106729]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:17.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:16 smithi142 ceph-mon[103614]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:17.918 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a: {} 2024-08-17T17:17:17.918 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b: {} 2024-08-17T17:17:17.918 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c: {} 2024-08-17T17:17:17.918 INFO:teuthology.orchestra.run.smithi142.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:17:17.918 INFO:teuthology.orchestra.run.smithi142.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:17:17.918 INFO:teuthology.orchestra.run.smithi142.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:17:18.213 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2589419006' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:17:18.213 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2589419006' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:17:18.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2589419006' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:17:18.549 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-08-17T17:17:18.550 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-17T17:17:18.550 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:17:18.550 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:17:18.550 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-08-17T17:17:18.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-08-17T17:17:18.551 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-08-17T17:17:18.551 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:17:18.551 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:17:18.552 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-08-17T17:17:18.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:18.552 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-08-17T17:17:18.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:17:18.553 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-08-17T17:17:18.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:17:18.553 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-08-17T17:17:18.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:17:18.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:18.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:18.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:18.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:18.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:18.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:18.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:19.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:19.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:18 smithi152 ceph-mon[93456]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:19.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:19.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:19.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:19.183 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:18 smithi142 ceph-mon[103614]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:19.183 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:18 smithi142 ceph-mon[106729]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:19.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:19.292 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:19.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:19.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:19.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:19.428 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:19.428 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:19.428 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:19.429 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:19.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:19.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:19.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:19.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:19.662 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:19.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:19.786 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:19.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:19.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:20.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:20.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.033 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:20.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:20.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:20.155 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:20.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:20.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:20.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:20.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:20.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:20.482 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:20.482 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:20.482 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:20 smithi142 ceph-mon[106729]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:20.482 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:20.482 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:20.482 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:20 smithi142 ceph-mon[103614]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:20.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:20.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:20.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:20.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:20 smithi152 ceph-mon[93456]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:20.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:17:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:17:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:17:20.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:20.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:20.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:20.775 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:20.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:20.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:21.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:21.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:21.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:21.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:21.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:21.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:21.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:21.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:21.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:21.625 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:21.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:21.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:21.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:21.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:21.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:22.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:22.024 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:17:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:17:21] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.51.0" 2024-08-17T17:17:22.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:22.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:22.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:22.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:22.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:22.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:22.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:22.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:22.510 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:22 smithi142 ceph-mon[103614]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:22.511 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:22 smithi142 ceph-mon[106729]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:22.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:22.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:22.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:22.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:22.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:22.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:22 smithi152 ceph-mon[93456]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:22.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:22.899 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:22.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:23.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:23.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:23.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:23.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:23.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:23.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:23.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:23.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:23.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:23.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:23.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:23.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:23.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:23.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:23.775 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:23.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:23.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:23.903 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:24.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:24.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:24.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:24.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:24.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:24.226 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:17:24.226 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:17:24.226 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:17:24.226 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:17:24.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:24.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:24.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:24.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:24.409 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:24.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:24.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:24.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:24.658 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:24 smithi142 ceph-mon[106729]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:24.658 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:24 smithi142 ceph-mon[103614]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:24.662 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:24.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:24.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:24 smithi152 ceph-mon[93456]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:24.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:24.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:24.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:25.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:25.039 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:25.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:17:25.141 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:25.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:25.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:25.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:25.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:25.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:25.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:25.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:25.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:25.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:25.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:25.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:25.898 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:25.924 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:26.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:26.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:26.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:26.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:26.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:26.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:26.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:26.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:26.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:26.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:26.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:26.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:26.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:26.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:26.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:26.789 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:26 smithi142 ceph-mon[106729]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:26.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:26 smithi142 ceph-mon[103614]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:26.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:26 smithi152 ceph-mon[93456]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:26.883 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:17:26.884 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:26.899 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:17:26.900 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:26.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:26.916 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:17:26.916 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:26.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:26.932 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:17:26.932 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:26.948 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:17:26.948 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:26.964 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:17:26.964 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:26.979 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:17:26.980 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:26.995 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:17:26.995 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:27.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:27.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:27.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:27.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:27.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:27.281 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:27.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:27.404 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:27.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:27.410 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:17:27.411 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_4' created 2024-08-17T17:17:27.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:27.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:27.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:27.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:27.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:27.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:27.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2238273206' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:17:27.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:27 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:17:27.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3756151938' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:27.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:27 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:27.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4027929137' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2238273206' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3756151938' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4027929137' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:27.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2238273206' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:17:27.755 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:17:27.755 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3756151938' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:27.755 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:27.755 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4027929137' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:27.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:27.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:27.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:27.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:28.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:28.057 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:28.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:28.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:28.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:28.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:28.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:28.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:28.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:28.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:28.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:28.642 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[106729]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[106729]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[103614]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:17:28.643 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:28 smithi142 ceph-mon[103614]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T17:17:28.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:28.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:28.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:28.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:28 smithi152 ceph-mon[93456]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:28 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:17:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:17:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:28 smithi152 ceph-mon[93456]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T17:17:28.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:28.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:29.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:29.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:29.050 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:29.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:29.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:29.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:29.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:29.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:29.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-08-17T17:17:29.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:29.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:29.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:29.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:29.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:29.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:29.711 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:29 smithi142 ceph-mon[106729]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T17:17:29.711 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:29 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:29.711 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:29 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:29.712 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:29 smithi142 ceph-mon[103614]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T17:17:29.712 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:29 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:29.712 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:29 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:29.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:29.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:17:29.790 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:29 smithi152 ceph-mon[93456]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:29 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:29 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:29.892 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:29.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:30.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:30.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:30.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:30.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:30.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:30.199 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:30.235 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:30.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:30.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:30.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:30.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:17:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.407 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:30.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:30.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:30.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:17:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:17:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-mon[93456]: pgmap v502: 89 pgs: 3 creating+activating, 7 creating+peering, 5 unknown, 74 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-mon[93456]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T17:17:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:30.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:30.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:30 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:17:30.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[106729]: pgmap v502: 89 pgs: 3 creating+activating, 7 creating+peering, 5 unknown, 74 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:30.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[106729]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T17:17:30.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:30.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:30.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[103614]: pgmap v502: 89 pgs: 3 creating+activating, 7 creating+peering, 5 unknown, 74 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[103614]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:30.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:30 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:17:30.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:30.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:30.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:30.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:30.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:30.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:30.901 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:31.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:31.027 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:31.129 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:31.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:31.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:31.275 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:31.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:31.398 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:31.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:31.519 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:31.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:31.624 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:31.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:31.748 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:31.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:31.823 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:17:31.823 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:17:31.871 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:31.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:31.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:32.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:32.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:17:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:17:31] "GET /metrics HTTP/1.1" 200 34008 "" "Prometheus/2.51.0" 2024-08-17T17:17:32.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:32.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:32.246 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:32.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:32.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:32.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:32.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:32.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:32.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:32.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:32.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:32.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:32.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:32.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:32 smithi152 ceph-mon[93456]: pgmap v504: 89 pgs: 3 creating+activating, 7 creating+peering, 3 unknown, 76 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:32 smithi142 ceph-mon[103614]: pgmap v504: 89 pgs: 3 creating+activating, 7 creating+peering, 3 unknown, 76 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:32 smithi142 ceph-mon[106729]: pgmap v504: 89 pgs: 3 creating+activating, 7 creating+peering, 3 unknown, 76 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:32.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:32.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:33.000 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:33.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:33.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:33.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:33.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:33.276 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:33.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:33.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:33.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:33.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:33.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:33.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:33.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:33.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:33.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:33.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:33.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:33.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:34.007 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:34.029 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:34.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:34.156 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:34.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:34.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:34.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:34.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:34.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:34.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:34.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:34.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:34.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:34.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:34.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:34.693 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:34 smithi142 ceph-mon[106729]: pgmap v505: 89 pgs: 3 creating+activating, 86 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:34.693 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:34 smithi142 ceph-mon[103614]: pgmap v505: 89 pgs: 3 creating+activating, 86 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:34.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:34.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:34 smithi152 ceph-mon[93456]: pgmap v505: 89 pgs: 3 creating+activating, 86 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:34.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:34.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:35.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:35.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.036 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:35.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:35.162 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:35.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:35.290 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:35.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:35.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:35.434 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-08-17T17:17:35.438 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:17:35.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:35.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1261168120' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2722921719' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:35.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:35 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1261168120' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2722921719' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:35 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:35.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:35.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:35.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:17:35.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:35.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1261168120' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2722921719' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:35 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:35.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:35.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:35.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:36.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:36.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:36.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:36.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:36.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:36.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:36.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:36.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:36.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:36.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:36.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:36.554 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:36.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:36.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[106729]: pgmap v506: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:36.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1261168120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[106729]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[103614]: pgmap v506: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1261168120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:36.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:36 smithi142 ceph-mon[103614]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T17:17:36.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:36.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:36.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:36.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:36 smithi152 ceph-mon[93456]: pgmap v506: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:36 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1261168120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:17:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:36 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:36 smithi152 ceph-mon[93456]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T17:17:36.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:36.927 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:37.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:37.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:37.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:37.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:37.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:37.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:37.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:37.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:37.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:37.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:37.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:37.546 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:37.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:37 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:17:37.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:37 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:17:37.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:37.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:37.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:37.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:37 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:17:37.885 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:37.895 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:17:37.895 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:37.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:37.921 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:17:37.921 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:37.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:37.938 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:17:37.938 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:37.954 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:17:37.954 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:37.969 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:17:37.969 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:37.984 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:17:37.985 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:37.988 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-08-17T17:17:37.989 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2024-08-17T17:17:37.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:17:37.991 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:17:37.991 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:17:38.000 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:17:38.000 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:38.017 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:17:38.017 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:38.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:38.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:38.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:38.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:38.165 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:17:38.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:17:38.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_157452 2024-08-17T17:17:38.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:17:38.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:17:38.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:38.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:38.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:38.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:38.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:38.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:38.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:38.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:38.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:38.738 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:38 smithi142 ceph-mon[106729]: pgmap v508: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:38.738 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2391839284' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:38.739 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:38 smithi142 ceph-mon[103614]: pgmap v508: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:38.739 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2391839284' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:38 smithi152 ceph-mon[93456]: pgmap v508: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:17:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2391839284' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:38.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:38.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:38.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:38.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:38.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:38.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:39.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:39.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:39.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 3173 3157 197.301 197.312 0.00247456 0.00505158 2024-08-17T17:17:39.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:39.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:39.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:39.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:39.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:39.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:39.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:39.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:39.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:17:39.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:39.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:39.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:39.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:39.920 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:39.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:39.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:40.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:40.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:40.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:40.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 6804 6788 212.108 226.938 0.002003 0.00470514 2024-08-17T17:17:40.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:40.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:40.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:40.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:40.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:40.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:40.491 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:17:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:17:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:17:40.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:40.592 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:40.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:40.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:40.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:40.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:40 smithi142 ceph-mon[103614]: pgmap v509: 89 pgs: 89 active+clean; 24 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 37 op/s 2024-08-17T17:17:40.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:40 smithi142 ceph-mon[106729]: pgmap v509: 89 pgs: 89 active+clean; 24 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 37 op/s 2024-08-17T17:17:40.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:40.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:40 smithi152 ceph-mon[93456]: pgmap v509: 89 pgs: 89 active+clean; 24 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 37 op/s 2024-08-17T17:17:40.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:40.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:41.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:41.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:41.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:41.166 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 10327 10311 214.794 220.188 0.00361539 0.00464365 2024-08-17T17:17:41.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:41.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:41.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:41.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:41.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:41.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:17:41.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:41.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:41.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:41.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:41.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:41.827 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:41.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:41.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:41.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:42.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:42.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:42.107 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:17:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:17:41] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.51.0" 2024-08-17T17:17:42.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:42.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:42.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 13452 13436 209.877 195.312 0.00504482 0.00475801 2024-08-17T17:17:42.173 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:17:42.173 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:17:42.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:31216 7.4 7:2ab38f1c:::benchmark_data_smithi142_122928_object31215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e105)", 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:07:50.470269+0000", 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 591.69777820700006, 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1470138009999999, 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.174 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 31216 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.470269+0000", 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.175 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.470269+0000", 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.470270+0000", 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.06e-07 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.470292+0000", 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2398000000000001e-05 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.470301+0000", 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5189999999999992e-06 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.470307+0000", 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9599999999999997e-06 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.471497+0000", 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011907829999999999 2024-08-17T17:17:42.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.471611+0000", 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000113267 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4", 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.471797+0000", 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018680699999999999 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476691+0000", 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048931390000000003 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476705+0000", 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4049000000000001e-05 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:51.474356+0000", 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99765157800000004 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:51.474371+0000", 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5233e-05 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.617171+0000", 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.142799329 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.617283+0000", 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011213300000000001 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.180 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:31225 7.4 7:2485fffd:::benchmark_data_smithi142_122928_object31224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e105)", 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:07:50.476674+0000", 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 591.69137302599995, 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1503873470000001, 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.181 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 31225 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476674+0000", 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476674+0000", 2024-08-17T17:17:42.182 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476676+0000", 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.708e-06 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476706+0000", 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0533000000000001e-05 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476778+0000", 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1826e-05 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.476787+0000", 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5320000000000006e-06 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.477059+0000", 2024-08-17T17:17:42.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00027204300000000002 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.477081+0000", 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2143000000000001e-05 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:51.474422+0000", 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99734061200000002 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:51.474460+0000", 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8077000000000002e-05 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.618377+0000", 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.143917002 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.618627+0000", 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024988199999999999 2024-08-17T17:17:42.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.618809+0000", 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018197800000000001 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.622550+0000", 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037416379999999998 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.622613+0000", 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2426000000000002e-05 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.625826+0000", 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032137929999999999 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.625899+0000", 2024-08-17T17:17:42.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2238999999999997e-05 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.627062+0000", 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001162915 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:31232 7.4 7:279fc1a5:::benchmark_data_smithi142_122928_object31231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-08-17T17:17:42.189 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:07:50.484759+0000", 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 591.68328776400006, 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1539881040000002, 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 31232 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.190 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484759+0000", 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484759+0000", 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.191 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484762+0000", 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0329999999999998e-06 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484795+0000", 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3130000000000003e-05 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484808+0000", 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3356999999999999e-05 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484819+0000", 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0518e-05 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484923+0000", 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000104082 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:50.484935+0000", 2024-08-17T17:17:42.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1939e-05 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:51.474485+0000", 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98955063399999998 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:51.474493+0000", 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9820000000000008e-06 2024-08-17T17:17:42.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.620413+0000", 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1459196229999999 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.620561+0000", 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014774699999999999 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.620808+0000", 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024786399999999998 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.622598+0000", 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001789204 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.622622+0000", 2024-08-17T17:17:42.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4284e-05 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.637717+0000", 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015094648 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.637777+0000", 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0294999999999998e-05 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.197 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:07:52.638748+0000", 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00097076400000000005 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:70556 7.4 7:2194be52:::benchmark_data_smithi142_122928_object70555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:08:23.732840+0000", 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 558.43520681699999, 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99772796399999997, 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.198 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 70556 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732840+0000", 2024-08-17T17:17:42.199 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732840+0000", 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.200 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732841+0000", 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2600000000000002e-07 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732860+0000", 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9001999999999999e-05 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732868+0000", 2024-08-17T17:17:42.201 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9859999999999994e-06 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732875+0000", 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4679999999999997e-06 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732936+0000", 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1390999999999995e-05 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.202 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.732945+0000", 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.935e-06 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.725938+0000", 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99299301399999995 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.726051+0000", 2024-08-17T17:17:42.203 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112717 2024-08-17T17:17:42.204 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.204 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.204 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:17:42.204 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.726150+0000", 2024-08-17T17:17:42.204 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8993999999999994e-05 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730403+0000", 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042533090000000003 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730420+0000", 2024-08-17T17:17:42.205 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6629999999999998e-05 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730492+0000", 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2756999999999995e-05 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730527+0000", 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5191999999999998e-05 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730568+0000", 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0942999999999997e-05 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:70563 7.4 7:2fce9a61:::benchmark_data_smithi142_122928_object70562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:08:23.739997+0000", 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 558.42804981699999, 2024-08-17T17:17:42.207 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0003252709999999, 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 70563 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.208 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.739997+0000", 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.739997+0000", 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.739998+0000", 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9299999999999998e-07 2024-08-17T17:17:42.209 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.740022+0000", 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4139e-05 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.740033+0000", 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.095e-05 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.740039+0000", 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7590000000000003e-06 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.740113+0000", 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4307999999999994e-05 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.740120+0000", 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9580000000000001e-06 2024-08-17T17:17:42.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730625+0000", 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99050484699999997 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730697+0000", 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2477000000000004e-05 2024-08-17T17:17:42.212 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.730780+0000", 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2473999999999997e-05 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.732566+0000", 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001786122 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.737558+0000", 2024-08-17T17:17:42.213 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00499198 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.740234+0000", 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026754769999999999 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.740280+0000", 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6328000000000001e-05 2024-08-17T17:17:42.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.740323+0000", 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2858999999999997e-05 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:70591 7.4 7:2b27ff52:::benchmark_data_smithi142_122928_object70590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:08:23.764333+0000", 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 558.403714734, 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99834404799999998, 2024-08-17T17:17:42.215 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 70591 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764333+0000", 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764333+0000", 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764334+0000", 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3540000000000001e-06 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764357+0000", 2024-08-17T17:17:42.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3291000000000001e-05 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764369+0000", 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1967000000000001e-05 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764376+0000", 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1679999999999998e-06 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764528+0000", 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015202099999999999 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:23.764538+0000", 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3470000000000005e-06 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.740827+0000", 2024-08-17T17:17:42.219 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97628947200000005 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.740958+0000", 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013088299999999999 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.741196+0000", 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00023807400000000001 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.749465+0000", 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0082688699999999993 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.756029+0000", 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0065642679999999998 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.762577+0000", 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0065474720000000004 2024-08-17T17:17:42.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.762637+0000", 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9919e-05 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:08:24.762677+0000", 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9941999999999998e-05 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:190686 7.e 7:797e845a:::benchmark_data_smithi142_122928_object78992:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:10:55.308695+0000", 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 406.85935246600002, 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1191704339999999, 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 190686 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.223 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308695+0000", 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308695+0000", 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308696+0000", 2024-08-17T17:17:42.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6929999999999999e-06 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308703+0000", 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7309999999999996e-06 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308724+0000", 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0871999999999999e-05 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308735+0000", 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.134e-05 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313072+0000", 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043364390000000001 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313083+0000", 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1038e-05 2024-08-17T17:17:42.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.403848+0000", 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.090764681 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.404014+0000", 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016686000000000001 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.427782+0000", 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.023767172 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.427865+0000", 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3608000000000003e-05 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:190690 7.e 7:781123bc:::benchmark_data_smithi142_122928_object78996:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:10:55.308758+0000", 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 406.85928960500001, 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1599029970000001, 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.229 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 190690 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308758+0000", 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.230 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308758+0000", 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308758+0000", 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4300000000000003e-07 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308761+0000", 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8890000000000002e-06 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308770+0000", 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3100000000000006e-06 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308776+0000", 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7790000000000001e-06 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313112+0000", 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043357819999999998 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313120+0000", 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7810000000000005e-06 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.404256+0000", 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.091136372 2024-08-17T17:17:42.233 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.404382+0000", 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012586300000000001 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.468559+0000", 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.064176762999999998 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.468661+0000", 2024-08-17T17:17:42.234 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000101915 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:190698 7.e 7:7142ffe2:::benchmark_data_smithi142_122928_object79004:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:10:55.308785+0000", 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 406.85926243300003, 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.160637769, 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.235 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 190698 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308785+0000", 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.236 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308785+0000", 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308785+0000", 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1899999999999998e-07 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308788+0000", 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3810000000000002e-06 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.237 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308794+0000", 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8109999999999997e-06 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308800+0000", 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3689999999999998e-06 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.238 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313148+0000", 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043477389999999998 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313159+0000", 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1141000000000001e-05 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.404414+0000", 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.091255482 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.239 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.404529+0000", 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115235 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.469339+0000", 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.064809137000000003 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.469423+0000", 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4054999999999994e-05 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:190703 7.e 7:79699c74:::benchmark_data_smithi142_122928_object79009:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:10:55.308807+0000", 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 406.85924042099998, 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1460064970000001, 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.241 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 190703 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308807+0000", 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308807+0000", 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308807+0000", 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4900000000000001e-07 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308809+0000", 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9690000000000001e-06 2024-08-17T17:17:42.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308815+0000", 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0260000000000004e-06 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308821+0000", 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.536e-06 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313186+0000", 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043654030000000003 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313194+0000", 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0369999999999995e-06 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.404560+0000", 2024-08-17T17:17:42.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0913658820000001 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.409057+0000", 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044971480000000003 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.409549+0000", 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00049212500000000005 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.427909+0000", 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018359695999999998 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.442563+0000", 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01465436 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.454479+0000", 2024-08-17T17:17:42.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01191594 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.454591+0000", 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011134299999999999 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.454813+0000", 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022258299999999999 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.248 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:190708 7.e 7:7268f706:::benchmark_data_smithi142_122928_object79014:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:10:55.308828+0000", 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 406.85921957099998, 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1462217180000001, 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.249 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.250 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 190708 2024-08-17T17:17:42.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.250 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:42.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308828+0000", 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308828+0000", 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308828+0000", 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.46e-07 2024-08-17T17:17:42.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308830+0000", 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0030000000000001e-06 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308836+0000", 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2890000000000003e-06 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308842+0000", 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1599999999999997e-06 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313226+0000", 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043847060000000004 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313235+0000", 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2400000000000007e-06 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.422154+0000", 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.108919207 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.422284+0000", 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000130037 2024-08-17T17:17:42.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.422424+0000", 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014023400000000001 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.427945+0000", 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005521008 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.442570+0000", 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014625176 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.454863+0000", 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012292908 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.454924+0000", 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1167000000000006e-05 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.455049+0000", 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012513699999999999 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.44453.0:190710 7.e 7:74a62329:::benchmark_data_smithi142_122928_object79016:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:10:55.308848+0000", 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 406.85919962700001, 2024-08-17T17:17:42.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.146412945, 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.44453", 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2906595730", 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 190710 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308848+0000", 2024-08-17T17:17:42.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308848+0000", 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308848+0000", 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1399999999999997e-07 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308850+0000", 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8899999999999999e-06 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308856+0000", 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2439999999999998e-06 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.308861+0000", 2024-08-17T17:17:42.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.977e-06 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313270+0000", 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044091479999999999 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:55.313279+0000", 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3499999999999997e-06 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.427973+0000", 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1146941539999999 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.428091+0000", 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000118312 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.428226+0000", 2024-08-17T17:17:42.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013439499999999999 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.432816+0000", 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045909790000000002 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.442616+0000", 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0097995419999999996 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.455090+0000", 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012473602 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.455141+0000", 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0909999999999999e-05 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:10:56.455261+0000", 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120028 2024-08-17T17:17:42.265 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:25683 8.c 8:3138c023:::benchmark_data_smithi142_139631_object25682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:10.027988+0000", 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 332.140059732, 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0086011479999999, 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 25683 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.027988+0000", 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.027988+0000", 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.027988+0000", 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2699999999999999e-07 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.028013+0000", 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.493e-05 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.028022+0000", 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6689999999999995e-06 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.028028+0000", 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5819999999999998e-06 2024-08-17T17:17:42.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.039651+0000", 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011622311 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:10.039660+0000", 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1710000000000008e-06 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.033406+0000", 2024-08-17T17:17:42.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99374629400000003 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.033593+0000", 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018659500000000001 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 7", 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.033817+0000", 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000224637 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.035275+0000", 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001458057 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.036443+0000", 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011678909999999999 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.036506+0000", 2024-08-17T17:17:42.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3169000000000005e-05 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:11.036589+0000", 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2214999999999998e-05 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76702 8.c 8:35fbd6a1:::benchmark_data_smithi142_139631_object76701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.357918+0000", 2024-08-17T17:17:42.273 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 302.81012936000002, 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3257934150000001, 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76702 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.357918+0000", 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.357918+0000", 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.357919+0000", 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3399999999999997e-07 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.357942+0000", 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2717999999999998e-05 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.357957+0000", 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5826999999999999e-05 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.357965+0000", 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0490000000000004e-06 2024-08-17T17:17:42.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.358167+0000", 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0002018 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.358177+0000", 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8859999999999999e-06 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.329886+0000", 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97170856100000003 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.329898+0000", 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2081e-05 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.672663+0000", 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3427651309999999 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.672805+0000", 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014232900000000001 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.672933+0000", 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012770699999999999 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.680441+0000", 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0075079719999999999 2024-08-17T17:17:42.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.680492+0000", 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1418999999999997e-05 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.681354+0000", 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00086174899999999998 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.681431+0000", 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6959000000000006e-05 2024-08-17T17:17:42.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.683711+0000", 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0022802930000000001 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76717 8.c 8:3e1d8e28:::benchmark_data_smithi142_139631_object76716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.371165+0000", 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 302.79688198999997, 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.31508468, 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.281 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76717 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371165+0000", 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371165+0000", 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371167+0000", 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.359e-06 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371187+0000", 2024-08-17T17:17:42.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0166999999999999e-05 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371199+0000", 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.243e-05 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371205+0000", 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8030000000000002e-06 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371258+0000", 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2667999999999998e-05 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.371266+0000", 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3180000000000007e-06 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.330298+0000", 2024-08-17T17:17:42.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95903205400000002 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.330308+0000", 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0407e-05 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.677503+0000", 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.347194357 2024-08-17T17:17:42.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.677643+0000", 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013980000000000001 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.677808+0000", 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000164885 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.680913+0000", 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031058510000000002 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.683767+0000", 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028538040000000001 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.683825+0000", 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7411999999999999e-05 2024-08-17T17:17:42.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.683867+0000", 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2484999999999997e-05 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686250+0000", 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00238288 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.289 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:209351 8.c 8:3f53ebe6:::benchmark_data_smithi142_139631_object88055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:15:23.107151+0000", 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 139.06089621699999, 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0190565069999999, 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.290 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209351 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107151+0000", 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107151+0000", 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107153+0000", 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5939999999999999e-06 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107158+0000", 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8490000000000002e-06 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107179+0000", 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0964000000000001e-05 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107187+0000", 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4789999999999999e-06 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107262+0000", 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5486000000000002e-05 2024-08-17T17:17:42.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.107273+0000", 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0808999999999999e-05 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.115096+0000", 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.007822698 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.115182+0000", 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5927000000000006e-05 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.115261+0000", 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9275000000000002e-05 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.119096+0000", 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038349690000000001 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.119110+0000", 2024-08-17T17:17:42.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3521e-05 2024-08-17T17:17:42.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126126+0000", 2024-08-17T17:17:42.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0070163689999999997 2024-08-17T17:17:42.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126166+0000", 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0098999999999997e-05 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126208+0000", 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1467999999999997e-05 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.298 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:209354 8.c 8:32096119:::benchmark_data_smithi142_139631_object88058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:15:23.112112+0000", 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 139.05593524, 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0142893369999999, 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.299 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209354 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112112+0000", 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112112+0000", 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112114+0000", 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9989999999999998e-06 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112120+0000", 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0290000000000002e-06 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112138+0000", 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7770000000000001e-05 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112145+0000", 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4220000000000003e-06 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112215+0000", 2024-08-17T17:17:42.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9288999999999993e-05 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.112225+0000", 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0954e-05 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.119155+0000", 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0069294470000001 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.119250+0000", 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4982000000000006e-05 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.119326+0000", 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6049000000000003e-05 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126261+0000", 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0069345810000000004 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126306+0000", 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5686000000000003e-05 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126345+0000", 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8770999999999999e-05 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126377+0000", 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1995000000000001e-05 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126401+0000", 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4363e-05 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:209363 8.c 8:34d6cc20:::benchmark_data_smithi142_139631_object88067:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:15:23.116459+0000", 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 139.051587926, 2024-08-17T17:17:42.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.01007669, 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209363 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116459+0000", 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116459+0000", 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116461+0000", 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4720000000000001e-06 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116467+0000", 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3339999999999998e-06 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116486+0000", 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8544000000000001e-05 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116501+0000", 2024-08-17T17:17:42.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5339e-05 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116669+0000", 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016826699999999999 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.116692+0000", 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2972999999999998e-05 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.122691+0000", 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.005998725 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.122812+0000", 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012071899999999999 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.122943+0000", 2024-08-17T17:17:42.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013098799999999999 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126445+0000", 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035022650000000001 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126452+0000", 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7070000000000003e-06 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126485+0000", 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3674999999999999e-05 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126511+0000", 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5507000000000002e-05 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126536+0000", 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5174999999999999e-05 2024-08-17T17:17:42.313 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:209380 8.c 8:3cb98fa1:::benchmark_data_smithi142_139631_object88084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:15:23.138328+0000", 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 139.02971951699999, 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0013926070000001, 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.314 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209380 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138328+0000", 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138328+0000", 2024-08-17T17:17:42.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138329+0000", 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.595e-06 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138334+0000", 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.459e-06 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138349+0000", 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4749999999999999e-05 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138356+0000", 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0350000000000004e-06 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138427+0000", 2024-08-17T17:17:42.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1218999999999997e-05 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.138437+0000", 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7820000000000005e-06 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126575+0000", 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98813857800000005 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126659+0000", 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3393000000000006e-05 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.126729+0000", 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0327999999999995e-05 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139613+0000", 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012883970999999999 2024-08-17T17:17:42.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139648+0000", 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5649999999999999e-05 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139655+0000", 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8499999999999996e-06 2024-08-17T17:17:42.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139686+0000", 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1146999999999998e-05 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139720+0000", 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3850000000000003e-05 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:209381 8.c 8:32dc26e3:::benchmark_data_smithi142_139631_object88085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:15:23.141526+0000", 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 139.02652107500001, 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.008162118, 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:17:42.322 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209381 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141526+0000", 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:17:42.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141526+0000", 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141528+0000", 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9570000000000001e-06 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141534+0000", 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6110000000000001e-06 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141547+0000", 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3638000000000001e-05 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141556+0000", 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6659999999999997e-06 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141621+0000", 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5085000000000005e-05 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:23.141631+0000", 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0037999999999999e-05 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.131572+0000", 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98994047100000004 2024-08-17T17:17:42.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.131732+0000", 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016038000000000001 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.131841+0000", 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108881 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139730+0000", 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078887880000000007 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.139786+0000", 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6184000000000003e-05 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.149601+0000", 2024-08-17T17:17:42.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0098151939999999993 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.149637+0000", 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6117e-05 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:15:24.149688+0000", 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1107999999999999e-05 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.329 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.330 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:17:42.330 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:17:42.330 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:17:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.331 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:42.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:42.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:42.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:42.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:42.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:42.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:42.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:42 smithi152 ceph-mon[93456]: pgmap v510: 89 pgs: 89 active+clean; 89 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 MiB/s wr, 141 op/s 2024-08-17T17:17:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:42.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[103614]: pgmap v510: 89 pgs: 89 active+clean; 89 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 MiB/s wr, 141 op/s 2024-08-17T17:17:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[106729]: pgmap v510: 89 pgs: 89 active+clean; 89 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 MiB/s wr, 141 op/s 2024-08-17T17:17:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:17:42.880 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:42.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:42.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:43.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:43.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:43.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:43.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 16464 16448 205.538 188.25 0.004844 0.0048594 2024-08-17T17:17:43.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:43.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:43.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:43.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:43.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:43.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:43.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:43.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:43.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:43.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:43.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:43.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:43.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:43.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:44.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:44.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:44.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 18994 18978 197.634 158.125 0.00455573 0.00505368 2024-08-17T17:17:44.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:44.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:44.323 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:44.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:44.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:44.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:44.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:44.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:44.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:44.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:44 smithi142 ceph-mon[106729]: pgmap v511: 89 pgs: 89 active+clean; 617 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-08-17T17:17:44.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:44 smithi142 ceph-mon[103614]: pgmap v511: 89 pgs: 89 active+clean; 617 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-08-17T17:17:44.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:44.671 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:44.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:44.794 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:17:44.795 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:17:44.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:44 smithi152 ceph-mon[93456]: pgmap v511: 89 pgs: 89 active+clean; 617 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-08-17T17:17:44.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:44.938 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:45.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:45.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:45.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:45.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 21115 21099 188.338 132.562 0.00462641 0.00530348 2024-08-17T17:17:45.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:45.213 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:45.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:45.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:45.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:45.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:45.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:45.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:45.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:45.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:45.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:17:45.692 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:45 smithi142 ceph-mon[106729]: from='client.35994 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:45.692 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:45 smithi142 ceph-mon[103614]: from='client.35994 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:45.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:45.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:45 smithi152 ceph-mon[93456]: from='client.35994 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:17:45.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:45.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:45.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:46.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:46.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:46.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:46.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:46.167 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 23719 23703 185.138 162.75 0.0101455 0.00539638 2024-08-17T17:17:46.267 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:46.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:46.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:46.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:46.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:46.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:46.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:46.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:46.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:46 smithi152 ceph-mon[93456]: pgmap v512: 89 pgs: 89 active+clean; 867 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:17:46.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:46 smithi142 ceph-mon[106729]: pgmap v512: 89 pgs: 89 active+clean; 867 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:17:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:46 smithi142 ceph-mon[103614]: pgmap v512: 89 pgs: 89 active+clean; 867 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:17:46.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:46.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:47.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:47.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:47.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 26198 26182 181.782 154.938 0.00450655 0.00549582 2024-08-17T17:17:47.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:47.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:47.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:47.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:47.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:47.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:47.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:47.669 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:47.771 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:47.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:47.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:47.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:48.043 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:48.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:48.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:48.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:48.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 28678 28662 179.102 155 0.00524584 0.00557828 2024-08-17T17:17:48.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:48.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:48.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:48.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:48.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:48.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:48.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:48.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:48.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:48.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:48.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:48 smithi152 ceph-mon[93456]: pgmap v513: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T17:17:48.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:48.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:48 smithi142 ceph-mon[106729]: pgmap v513: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T17:17:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:48 smithi142 ceph-mon[103614]: pgmap v513: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T17:17:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:48.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:49.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:49.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:49.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:49.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 30586 30570 173.661 119.25 0.00427557 0.00575339 2024-08-17T17:17:49.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:49.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:49.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:49.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:49.430 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":511,"stamp":"2024-08-17T17:17:43.375304+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":646911949,"num_objects":9888,"num_object_clones":0,"num_object_copies":29664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9888,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":10140,"num_write_kb":633615,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":10069,"ondisk_log_size":10069,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5244876,"kb_used_data":1670536,"kb_used_omap":29,"kb_used_meta":3574242,"kb_avail":744716340,"statfs":{"total":767960285184,"available":762589532160,"internally_reserved":0,"allocated":1710628864,"data_stored":1704385782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30202,"internal_metadata":3660024326},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":646316032,"num_objects":9862,"num_object_clones":0,"num_object_copies":29586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9862,"num_write_kb":631168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002962"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.437097+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:17:35.437097+0000","last_peered":"2024-08-17T17:17:35.437097+0000","last_clean":"2024-08-17T17:17:35.437097+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:17:35.437097+0000","last_undegraded":"2024-08-17T17:17:35.437097+0000","last_fullsized":"2024-08-17T17:17:35.437097+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":115,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414562+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:17:29.414562+0000","last_peered":"2024-08-17T17:17:29.414562+0000","last_clean":"2024-08-17T17:17:29.414562+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:17:29.414562+0000","last_undegraded":"2024-08-17T17:17:29.414562+0000","last_fullsized":"2024-08-17T17:17:29.414562+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413783+0000","last_change":"2024-08-17T17:12:40.335427+0000","last_active":"2024-08-17T17:17:29.413783+0000","last_peered":"2024-08-17T17:17:29.413783+0000","last_clean":"2024-08-17T17:17:29.413783+0000","last_became_active":"2024-08-17T17:12:40.334535+0000","last_became_peered":"2024-08-17T17:12:40.334535+0000","last_unstale":"2024-08-17T17:17:29.413783+0000","last_undegraded":"2024-08-17T17:17:29.413783+0000","last_fullsized":"2024-08-17T17:17:29.413783+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:47:39.810507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652607+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:17:35.652607+0000","last_peered":"2024-08-17T17:17:35.652607+0000","last_clean":"2024-08-17T17:17:35.652607+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:17:35.652607+0000","last_undegraded":"2024-08-17T17:17:35.652607+0000","last_fullsized":"2024-08-17T17:17:35.652607+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652904+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:17:35.652904+0000","last_peered":"2024-08-17T17:17:35.652904+0000","last_clean":"2024-08-17T17:17:35.652904+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:17:35.652904+0000","last_undegraded":"2024-08-17T17:17:35.652904+0000","last_fullsized":"2024-08-17T17:17:35.652904+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":96,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414492+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:17:29.414492+0000","last_peered":"2024-08-17T17:17:29.414492+0000","last_clean":"2024-08-17T17:17:29.414492+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:17:29.414492+0000","last_undegraded":"2024-08-17T17:17:29.414492+0000","last_fullsized":"2024-08-17T17:17:29.414492+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":99,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.437246+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:17:35.437246+0000","last_peered":"2024-08-17T17:17:35.437246+0000","last_clean":"2024-08-17T17:17:35.437246+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:17:35.437246+0000","last_undegraded":"2024-08-17T17:17:35.437246+0000","last_fullsized":"2024-08-17T17:17:35.437246+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":123,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.521521+0000","last_change":"2024-08-17T17:07:07.418480+0000","last_active":"2024-08-17T17:17:30.521521+0000","last_peered":"2024-08-17T17:17:30.521521+0000","last_clean":"2024-08-17T17:17:30.521521+0000","last_became_active":"2024-08-17T17:07:07.418124+0000","last_became_peered":"2024-08-17T17:07:07.418124+0000","last_unstale":"2024-08-17T17:17:30.521521+0000","last_undegraded":"2024-08-17T17:17:30.521521+0000","last_fullsized":"2024-08-17T17:17:30.521521+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:12.020815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"157'672","reported_seq":689,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:41.379607+0000","last_change":"2024-08-17T17:17:28.420626+0000","last_active":"2024-08-17T17:17:41.379607+0000","last_peered":"2024-08-17T17:17:41.379607+0000","last_clean":"2024-08-17T17:17:41.379607+0000","last_became_active":"2024-08-17T17:17:28.420361+0000","last_became_peered":"2024-08-17T17:17:28.420361+0000","last_unstale":"2024-08-17T17:17:41.379607+0000","last_undegraded":"2024-08-17T17:17:41.379607+0000","last_fullsized":"2024-08-17T17:17:41.379607+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":672,"log_dups_size":0,"ondisk_log_size":672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:16.092391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44040192,"num_objects":672,"num_object_clones":0,"num_object_copies":2016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":672,"num_write_kb":43008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414915+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:17:29.414915+0000","last_peered":"2024-08-17T17:17:29.414915+0000","last_clean":"2024-08-17T17:17:29.414915+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:17:29.414915+0000","last_undegraded":"2024-08-17T17:17:29.414915+0000","last_fullsized":"2024-08-17T17:17:29.414915+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413711+0000","last_change":"2024-08-17T16:57:28.770580+0000","last_active":"2024-08-17T17:17:29.413711+0000","last_peered":"2024-08-17T17:17:29.413711+0000","last_clean":"2024-08-17T17:17:29.413711+0000","last_became_active":"2024-08-17T16:57:28.770195+0000","last_became_peered":"2024-08-17T16:57:28.770195+0000","last_unstale":"2024-08-17T17:17:29.413711+0000","last_undegraded":"2024-08-17T17:17:29.413711+0000","last_fullsized":"2024-08-17T17:17:29.413711+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:35:09.301682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"157'515","reported_seq":532,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:40.475053+0000","last_change":"2024-08-17T17:17:29.065318+0000","last_active":"2024-08-17T17:17:40.475053+0000","last_peered":"2024-08-17T17:17:40.475053+0000","last_clean":"2024-08-17T17:17:40.475053+0000","last_became_active":"2024-08-17T17:17:29.065028+0000","last_became_peered":"2024-08-17T17:17:29.065028+0000","last_unstale":"2024-08-17T17:17:40.475053+0000","last_undegraded":"2024-08-17T17:17:40.475053+0000","last_fullsized":"2024-08-17T17:17:40.475053+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":515,"log_dups_size":0,"ondisk_log_size":515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:59.792009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33751040,"num_objects":515,"num_object_clones":0,"num_object_copies":1545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":515,"num_write_kb":32960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":115,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414934+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:17:29.414934+0000","last_peered":"2024-08-17T17:17:29.414934+0000","last_clean":"2024-08-17T17:17:29.414934+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:17:29.414934+0000","last_undegraded":"2024-08-17T17:17:29.414934+0000","last_fullsized":"2024-08-17T17:17:29.414934+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":129,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076160+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:17:30.076160+0000","last_peered":"2024-08-17T17:17:30.076160+0000","last_clean":"2024-08-17T17:17:30.076160+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:17:30.076160+0000","last_undegraded":"2024-08-17T17:17:30.076160+0000","last_fullsized":"2024-08-17T17:17:30.076160+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"157'884","reported_seq":895,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.606727+0000","last_change":"2024-08-17T17:17:28.420578+0000","last_active":"2024-08-17T17:17:42.606727+0000","last_peered":"2024-08-17T17:17:42.606727+0000","last_clean":"2024-08-17T17:17:42.606727+0000","last_became_active":"2024-08-17T17:17:28.420349+0000","last_became_peered":"2024-08-17T17:17:28.420349+0000","last_unstale":"2024-08-17T17:17:42.606727+0000","last_undegraded":"2024-08-17T17:17:42.606727+0000","last_fullsized":"2024-08-17T17:17:42.606727+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":884,"log_dups_size":0,"ondisk_log_size":884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:02.270484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":884,"num_write_kb":56576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":97,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414844+0000","last_change":"2024-08-17T16:57:28.389577+0000","last_active":"2024-08-17T17:17:29.414844+0000","last_peered":"2024-08-17T17:17:29.414844+0000","last_clean":"2024-08-17T17:17:29.414844+0000","last_became_active":"2024-08-17T16:57:28.389273+0000","last_became_peered":"2024-08-17T16:57:28.389273+0000","last_unstale":"2024-08-17T17:17:29.414844+0000","last_undegraded":"2024-08-17T17:17:29.414844+0000","last_fullsized":"2024-08-17T17:17:29.414844+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:19.264181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.521490+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:17:30.521490+0000","last_peered":"2024-08-17T17:17:30.521490+0000","last_clean":"2024-08-17T17:17:30.521490+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:17:30.521490+0000","last_undegraded":"2024-08-17T17:17:30.521490+0000","last_fullsized":"2024-08-17T17:17:30.521490+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"157'940","reported_seq":951,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.630181+0000","last_change":"2024-08-17T17:17:28.867515+0000","last_active":"2024-08-17T17:17:42.630181+0000","last_peered":"2024-08-17T17:17:42.630181+0000","last_clean":"2024-08-17T17:17:42.630181+0000","last_became_active":"2024-08-17T17:17:28.867246+0000","last_became_peered":"2024-08-17T17:17:28.867246+0000","last_unstale":"2024-08-17T17:17:42.630181+0000","last_undegraded":"2024-08-17T17:17:42.630181+0000","last_fullsized":"2024-08-17T17:17:42.630181+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":940,"log_dups_size":0,"ondisk_log_size":940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:22.512344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61603840,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":940,"num_write_kb":60160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413755+0000","last_change":"2024-08-17T17:12:40.336371+0000","last_active":"2024-08-17T17:17:29.413755+0000","last_peered":"2024-08-17T17:17:29.413755+0000","last_clean":"2024-08-17T17:17:29.413755+0000","last_became_active":"2024-08-17T17:12:40.335783+0000","last_became_peered":"2024-08-17T17:12:40.335783+0000","last_unstale":"2024-08-17T17:17:29.413755+0000","last_undegraded":"2024-08-17T17:17:29.413755+0000","last_fullsized":"2024-08-17T17:17:29.413755+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:14.313020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":172,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652674+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:17:35.652674+0000","last_peered":"2024-08-17T17:17:35.652674+0000","last_clean":"2024-08-17T17:17:35.652674+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:17:35.652674+0000","last_undegraded":"2024-08-17T17:17:35.652674+0000","last_fullsized":"2024-08-17T17:17:35.652674+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"157'99","reported_seq":116,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:38.514824+0000","last_change":"2024-08-17T17:17:28.688481+0000","last_active":"2024-08-17T17:17:38.514824+0000","last_peered":"2024-08-17T17:17:38.514824+0000","last_clean":"2024-08-17T17:17:38.514824+0000","last_became_active":"2024-08-17T17:17:28.688202+0000","last_became_peered":"2024-08-17T17:17:28.688202+0000","last_unstale":"2024-08-17T17:17:38.514824+0000","last_undegraded":"2024-08-17T17:17:38.514824+0000","last_fullsized":"2024-08-17T17:17:38.514824+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.790595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6488064,"num_objects":99,"num_object_clones":0,"num_object_copies":297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":99,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":99,"num_write_kb":6336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":127,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436912+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:17:35.436912+0000","last_peered":"2024-08-17T17:17:35.436912+0000","last_clean":"2024-08-17T17:17:35.436912+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:17:35.436912+0000","last_undegraded":"2024-08-17T17:17:35.436912+0000","last_fullsized":"2024-08-17T17:17:35.436912+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076269+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:17:30.076269+0000","last_peered":"2024-08-17T17:17:30.076269+0000","last_clean":"2024-08-17T17:17:30.076269+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:17:30.076269+0000","last_undegraded":"2024-08-17T17:17:30.076269+0000","last_fullsized":"2024-08-17T17:17:30.076269+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"157'524","reported_seq":541,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:40.469496+0000","last_change":"2024-08-17T17:17:28.867482+0000","last_active":"2024-08-17T17:17:40.469496+0000","last_peered":"2024-08-17T17:17:40.469496+0000","last_clean":"2024-08-17T17:17:40.469496+0000","last_became_active":"2024-08-17T17:17:28.867155+0000","last_became_peered":"2024-08-17T17:17:28.867155+0000","last_unstale":"2024-08-17T17:17:40.469496+0000","last_undegraded":"2024-08-17T17:17:40.469496+0000","last_fullsized":"2024-08-17T17:17:40.469496+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":524,"log_dups_size":0,"ondisk_log_size":524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:28.128178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34340864,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":524,"num_write_kb":33536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":115,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.521595+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:17:30.521595+0000","last_peered":"2024-08-17T17:17:30.521595+0000","last_clean":"2024-08-17T17:17:30.521595+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:17:30.521595+0000","last_undegraded":"2024-08-17T17:17:30.521595+0000","last_fullsized":"2024-08-17T17:17:30.521595+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":129,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076537+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:17:30.076537+0000","last_peered":"2024-08-17T17:17:30.076537+0000","last_clean":"2024-08-17T17:17:30.076537+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:17:30.076537+0000","last_undegraded":"2024-08-17T17:17:30.076537+0000","last_fullsized":"2024-08-17T17:17:30.076537+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"157'882","reported_seq":893,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.431636+0000","last_change":"2024-08-17T17:17:28.420590+0000","last_active":"2024-08-17T17:17:42.431636+0000","last_peered":"2024-08-17T17:17:42.431636+0000","last_clean":"2024-08-17T17:17:42.431636+0000","last_became_active":"2024-08-17T17:17:28.420209+0000","last_became_peered":"2024-08-17T17:17:28.420209+0000","last_unstale":"2024-08-17T17:17:42.431636+0000","last_undegraded":"2024-08-17T17:17:42.431636+0000","last_fullsized":"2024-08-17T17:17:42.431636+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":882,"log_dups_size":0,"ondisk_log_size":882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:57:27.080321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57802752,"num_objects":882,"num_object_clones":0,"num_object_copies":2646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":882,"num_write_kb":56448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076274+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:17:30.076274+0000","last_peered":"2024-08-17T17:17:30.076274+0000","last_clean":"2024-08-17T17:17:30.076274+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:17:30.076274+0000","last_undegraded":"2024-08-17T17:17:30.076274+0000","last_fullsized":"2024-08-17T17:17:30.076274+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":112,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076566+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:17:30.076566+0000","last_peered":"2024-08-17T17:17:30.076566+0000","last_clean":"2024-08-17T17:17:30.076566+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:17:30.076566+0000","last_undegraded":"2024-08-17T17:17:30.076566+0000","last_fullsized":"2024-08-17T17:17:30.076566+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"157'94","reported_seq":111,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:38.513295+0000","last_change":"2024-08-17T17:17:28.867499+0000","last_active":"2024-08-17T17:17:38.513295+0000","last_peered":"2024-08-17T17:17:38.513295+0000","last_clean":"2024-08-17T17:17:38.513295+0000","last_became_active":"2024-08-17T17:17:28.867215+0000","last_became_peered":"2024-08-17T17:17:28.867215+0000","last_unstale":"2024-08-17T17:17:38.513295+0000","last_undegraded":"2024-08-17T17:17:38.513295+0000","last_fullsized":"2024-08-17T17:17:38.513295+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":94,"log_dups_size":0,"ondisk_log_size":94,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:23:01.125637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":94,"num_write_kb":6016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":100,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436830+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:17:35.436830+0000","last_peered":"2024-08-17T17:17:35.436830+0000","last_clean":"2024-08-17T17:17:35.436830+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:17:35.436830+0000","last_undegraded":"2024-08-17T17:17:35.436830+0000","last_fullsized":"2024-08-17T17:17:35.436830+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":110,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413897+0000","last_change":"2024-08-17T17:12:40.335001+0000","last_active":"2024-08-17T17:17:29.413897+0000","last_peered":"2024-08-17T17:17:29.413897+0000","last_clean":"2024-08-17T17:17:29.413897+0000","last_became_active":"2024-08-17T17:12:40.334650+0000","last_became_peered":"2024-08-17T17:12:40.334650+0000","last_unstale":"2024-08-17T17:17:29.413897+0000","last_undegraded":"2024-08-17T17:17:29.413897+0000","last_fullsized":"2024-08-17T17:17:29.413897+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:06.876541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"157'83","reported_seq":100,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:38.511271+0000","last_change":"2024-08-17T17:17:28.419521+0000","last_active":"2024-08-17T17:17:38.511271+0000","last_peered":"2024-08-17T17:17:38.511271+0000","last_clean":"2024-08-17T17:17:38.511271+0000","last_became_active":"2024-08-17T17:17:28.419100+0000","last_became_peered":"2024-08-17T17:17:28.419100+0000","last_unstale":"2024-08-17T17:17:38.511271+0000","last_undegraded":"2024-08-17T17:17:38.511271+0000","last_fullsized":"2024-08-17T17:17:38.511271+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:39.998376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5439488,"num_objects":83,"num_object_clones":0,"num_object_copies":249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":83,"num_write_kb":5312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":453,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414352+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:17:29.414352+0000","last_peered":"2024-08-17T17:17:29.414352+0000","last_clean":"2024-08-17T17:17:29.414352+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:17:29.414352+0000","last_undegraded":"2024-08-17T17:17:29.414352+0000","last_fullsized":"2024-08-17T17:17:29.414352+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":127,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413656+0000","last_change":"2024-08-17T17:10:55.313434+0000","last_active":"2024-08-17T17:17:29.413656+0000","last_peered":"2024-08-17T17:17:29.413656+0000","last_clean":"2024-08-17T17:17:29.413656+0000","last_became_active":"2024-08-17T17:10:55.313091+0000","last_became_peered":"2024-08-17T17:10:55.313091+0000","last_unstale":"2024-08-17T17:17:29.413656+0000","last_undegraded":"2024-08-17T17:17:29.413656+0000","last_fullsized":"2024-08-17T17:17:29.413656+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:17.970517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":86,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436601+0000","last_change":"2024-08-17T17:12:40.332647+0000","last_active":"2024-08-17T17:17:35.436601+0000","last_peered":"2024-08-17T17:17:35.436601+0000","last_clean":"2024-08-17T17:17:35.436601+0000","last_became_active":"2024-08-17T17:12:40.332376+0000","last_became_peered":"2024-08-17T17:12:40.332376+0000","last_unstale":"2024-08-17T17:17:35.436601+0000","last_undegraded":"2024-08-17T17:17:35.436601+0000","last_fullsized":"2024-08-17T17:17:35.436601+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:21.624216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":97,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.521315+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:17:30.521315+0000","last_peered":"2024-08-17T17:17:30.521315+0000","last_clean":"2024-08-17T17:17:30.521315+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:17:30.521315+0000","last_undegraded":"2024-08-17T17:17:30.521315+0000","last_fullsized":"2024-08-17T17:17:30.521315+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"157'711","reported_seq":728,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:41.390597+0000","last_change":"2024-08-17T17:17:28.688485+0000","last_active":"2024-08-17T17:17:41.390597+0000","last_peered":"2024-08-17T17:17:41.390597+0000","last_clean":"2024-08-17T17:17:41.390597+0000","last_became_active":"2024-08-17T17:17:28.688145+0000","last_became_peered":"2024-08-17T17:17:28.688145+0000","last_unstale":"2024-08-17T17:17:41.390597+0000","last_undegraded":"2024-08-17T17:17:41.390597+0000","last_fullsized":"2024-08-17T17:17:41.390597+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":711,"log_dups_size":0,"ondisk_log_size":711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:16.215934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46596096,"num_objects":711,"num_object_clones":0,"num_object_copies":2133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":711,"num_write_kb":45504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":142,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414302+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:17:29.414302+0000","last_peered":"2024-08-17T17:17:29.414302+0000","last_clean":"2024-08-17T17:17:29.414302+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:17:29.414302+0000","last_undegraded":"2024-08-17T17:17:29.414302+0000","last_fullsized":"2024-08-17T17:17:29.414302+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652549+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:17:35.652549+0000","last_peered":"2024-08-17T17:17:35.652549+0000","last_clean":"2024-08-17T17:17:35.652549+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:17:35.652549+0000","last_undegraded":"2024-08-17T17:17:35.652549+0000","last_fullsized":"2024-08-17T17:17:35.652549+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":183,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652802+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:17:35.652802+0000","last_peered":"2024-08-17T17:17:35.652802+0000","last_clean":"2024-08-17T17:17:35.652802+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:17:35.652802+0000","last_undegraded":"2024-08-17T17:17:35.652802+0000","last_fullsized":"2024-08-17T17:17:35.652802+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"157'979","reported_seq":990,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.624327+0000","last_change":"2024-08-17T17:17:28.419197+0000","last_active":"2024-08-17T17:17:42.624327+0000","last_peered":"2024-08-17T17:17:42.624327+0000","last_clean":"2024-08-17T17:17:42.624327+0000","last_became_active":"2024-08-17T17:17:28.418931+0000","last_became_peered":"2024-08-17T17:17:28.418931+0000","last_unstale":"2024-08-17T17:17:42.624327+0000","last_undegraded":"2024-08-17T17:17:42.624327+0000","last_fullsized":"2024-08-17T17:17:42.624327+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":979,"log_dups_size":0,"ondisk_log_size":979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:13:59.859671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64159744,"num_objects":979,"num_object_clones":0,"num_object_copies":2937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":979,"num_write_kb":62656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":159,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436006+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:17:35.436006+0000","last_peered":"2024-08-17T17:17:35.436006+0000","last_clean":"2024-08-17T17:17:35.436006+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:17:35.436006+0000","last_undegraded":"2024-08-17T17:17:35.436006+0000","last_fullsized":"2024-08-17T17:17:35.436006+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.702330+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:17:35.702330+0000","last_peered":"2024-08-17T17:17:35.702330+0000","last_clean":"2024-08-17T17:17:35.702330+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:17:35.702330+0000","last_undegraded":"2024-08-17T17:17:35.702330+0000","last_fullsized":"2024-08-17T17:17:35.702330+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":164,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.703073+0000","last_change":"2024-08-17T17:07:51.480410+0000","last_active":"2024-08-17T17:17:35.703073+0000","last_peered":"2024-08-17T17:17:35.703073+0000","last_clean":"2024-08-17T17:17:35.703073+0000","last_became_active":"2024-08-17T17:07:51.480083+0000","last_became_peered":"2024-08-17T17:07:51.480083+0000","last_unstale":"2024-08-17T17:17:35.703073+0000","last_undegraded":"2024-08-17T17:17:35.703073+0000","last_fullsized":"2024-08-17T17:17:35.703073+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:39.376320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"157'661","reported_seq":678,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:41.374021+0000","last_change":"2024-08-17T17:17:28.419174+0000","last_active":"2024-08-17T17:17:41.374021+0000","last_peered":"2024-08-17T17:17:41.374021+0000","last_clean":"2024-08-17T17:17:41.374021+0000","last_became_active":"2024-08-17T17:17:28.418856+0000","last_became_peered":"2024-08-17T17:17:28.418856+0000","last_unstale":"2024-08-17T17:17:41.374021+0000","last_undegraded":"2024-08-17T17:17:41.374021+0000","last_fullsized":"2024-08-17T17:17:41.374021+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":661,"log_dups_size":0,"ondisk_log_size":661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:33.384778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43319296,"num_objects":661,"num_object_clones":0,"num_object_copies":1983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":661,"num_write_kb":42304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":128,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413670+0000","last_change":"2024-08-17T17:07:51.483538+0000","last_active":"2024-08-17T17:17:29.413670+0000","last_peered":"2024-08-17T17:17:29.413670+0000","last_clean":"2024-08-17T17:17:29.413670+0000","last_became_active":"2024-08-17T17:07:51.482970+0000","last_became_peered":"2024-08-17T17:07:51.482970+0000","last_unstale":"2024-08-17T17:17:29.413670+0000","last_undegraded":"2024-08-17T17:17:29.413670+0000","last_fullsized":"2024-08-17T17:17:29.413670+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:37.985321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":113,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076404+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:17:30.076404+0000","last_peered":"2024-08-17T17:17:30.076404+0000","last_clean":"2024-08-17T17:17:30.076404+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:17:30.076404+0000","last_undegraded":"2024-08-17T17:17:30.076404+0000","last_fullsized":"2024-08-17T17:17:30.076404+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.702917+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:17:35.702917+0000","last_peered":"2024-08-17T17:17:35.702917+0000","last_clean":"2024-08-17T17:17:35.702917+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:17:35.702917+0000","last_undegraded":"2024-08-17T17:17:35.702917+0000","last_fullsized":"2024-08-17T17:17:35.702917+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"157'939","reported_seq":950,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.620424+0000","last_change":"2024-08-17T17:17:28.420498+0000","last_active":"2024-08-17T17:17:42.620424+0000","last_peered":"2024-08-17T17:17:42.620424+0000","last_clean":"2024-08-17T17:17:42.620424+0000","last_became_active":"2024-08-17T17:17:28.420206+0000","last_became_peered":"2024-08-17T17:17:28.420206+0000","last_unstale":"2024-08-17T17:17:42.620424+0000","last_undegraded":"2024-08-17T17:17:42.620424+0000","last_fullsized":"2024-08-17T17:17:42.620424+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":939,"log_dups_size":0,"ondisk_log_size":939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:19.386499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":939,"num_write_kb":60096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413883+0000","last_change":"2024-08-17T17:12:40.335196+0000","last_active":"2024-08-17T17:17:29.413883+0000","last_peered":"2024-08-17T17:17:29.413883+0000","last_clean":"2024-08-17T17:17:29.413883+0000","last_became_active":"2024-08-17T17:12:40.334917+0000","last_became_peered":"2024-08-17T17:12:40.334917+0000","last_unstale":"2024-08-17T17:17:29.413883+0000","last_undegraded":"2024-08-17T17:17:29.413883+0000","last_fullsized":"2024-08-17T17:17:29.413883+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:02.633323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":177,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.702354+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:17:35.702354+0000","last_peered":"2024-08-17T17:17:35.702354+0000","last_clean":"2024-08-17T17:17:35.702354+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:17:35.702354+0000","last_undegraded":"2024-08-17T17:17:35.702354+0000","last_fullsized":"2024-08-17T17:17:35.702354+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":128,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.702765+0000","last_change":"2024-08-17T17:07:51.489205+0000","last_active":"2024-08-17T17:17:35.702765+0000","last_peered":"2024-08-17T17:17:35.702765+0000","last_clean":"2024-08-17T17:17:35.702765+0000","last_became_active":"2024-08-17T17:07:51.488783+0000","last_became_peered":"2024-08-17T17:07:51.488783+0000","last_unstale":"2024-08-17T17:17:35.702765+0000","last_undegraded":"2024-08-17T17:17:35.702765+0000","last_fullsized":"2024-08-17T17:17:35.702765+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:22.190780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"157'910","reported_seq":921,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.428807+0000","last_change":"2024-08-17T17:17:29.064153+0000","last_active":"2024-08-17T17:17:42.428807+0000","last_peered":"2024-08-17T17:17:42.428807+0000","last_clean":"2024-08-17T17:17:42.428807+0000","last_became_active":"2024-08-17T17:17:29.063850+0000","last_became_peered":"2024-08-17T17:17:29.063850+0000","last_unstale":"2024-08-17T17:17:42.428807+0000","last_undegraded":"2024-08-17T17:17:42.428807+0000","last_fullsized":"2024-08-17T17:17:42.428807+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":910,"log_dups_size":0,"ondisk_log_size":910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:14:10.124651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59637760,"num_objects":910,"num_object_clones":0,"num_object_copies":2730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":910,"num_write_kb":58240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.703006+0000","last_change":"2024-08-17T17:10:55.321862+0000","last_active":"2024-08-17T17:17:35.703006+0000","last_peered":"2024-08-17T17:17:35.703006+0000","last_clean":"2024-08-17T17:17:35.703006+0000","last_became_active":"2024-08-17T17:10:55.321339+0000","last_became_peered":"2024-08-17T17:10:55.321339+0000","last_unstale":"2024-08-17T17:17:35.703006+0000","last_undegraded":"2024-08-17T17:17:35.703006+0000","last_fullsized":"2024-08-17T17:17:35.703006+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:17.662591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":125,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414851+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:17:29.414851+0000","last_peered":"2024-08-17T17:17:29.414851+0000","last_clean":"2024-08-17T17:17:29.414851+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:17:29.414851+0000","last_undegraded":"2024-08-17T17:17:29.414851+0000","last_fullsized":"2024-08-17T17:17:29.414851+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652766+0000","last_change":"2024-08-17T17:07:07.424546+0000","last_active":"2024-08-17T17:17:35.652766+0000","last_peered":"2024-08-17T17:17:35.652766+0000","last_clean":"2024-08-17T17:17:35.652766+0000","last_became_active":"2024-08-17T17:07:07.424158+0000","last_became_peered":"2024-08-17T17:07:07.424158+0000","last_unstale":"2024-08-17T17:17:35.652766+0000","last_undegraded":"2024-08-17T17:17:35.652766+0000","last_fullsized":"2024-08-17T17:17:35.652766+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:34.175159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"157'868","reported_seq":879,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:42.408654+0000","last_change":"2024-08-17T17:17:28.421539+0000","last_active":"2024-08-17T17:17:42.408654+0000","last_peered":"2024-08-17T17:17:42.408654+0000","last_clean":"2024-08-17T17:17:42.408654+0000","last_became_active":"2024-08-17T17:17:28.421249+0000","last_became_peered":"2024-08-17T17:17:28.421249+0000","last_unstale":"2024-08-17T17:17:42.408654+0000","last_undegraded":"2024-08-17T17:17:42.408654+0000","last_fullsized":"2024-08-17T17:17:42.408654+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":868,"log_dups_size":0,"ondisk_log_size":868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:02:04.446948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56885248,"num_objects":868,"num_object_clones":0,"num_object_copies":2604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":868,"num_write_kb":55552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":127,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.437328+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:17:35.437328+0000","last_peered":"2024-08-17T17:17:35.437328+0000","last_clean":"2024-08-17T17:17:35.437328+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:17:35.437328+0000","last_undegraded":"2024-08-17T17:17:35.437328+0000","last_fullsized":"2024-08-17T17:17:35.437328+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.703160+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:17:35.703160+0000","last_peered":"2024-08-17T17:17:35.703160+0000","last_clean":"2024-08-17T17:17:35.703160+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:17:35.703160+0000","last_undegraded":"2024-08-17T17:17:35.703160+0000","last_fullsized":"2024-08-17T17:17:35.703160+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":124,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436510+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:17:35.436510+0000","last_peered":"2024-08-17T17:17:35.436510+0000","last_clean":"2024-08-17T17:17:35.436510+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:17:35.436510+0000","last_undegraded":"2024-08-17T17:17:35.436510+0000","last_fullsized":"2024-08-17T17:17:35.436510+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"157'101","reported_seq":118,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:38.511381+0000","last_change":"2024-08-17T17:17:29.065051+0000","last_active":"2024-08-17T17:17:38.511381+0000","last_peered":"2024-08-17T17:17:38.511381+0000","last_clean":"2024-08-17T17:17:38.511381+0000","last_became_active":"2024-08-17T17:17:29.064687+0000","last_became_peered":"2024-08-17T17:17:29.064687+0000","last_unstale":"2024-08-17T17:17:38.511381+0000","last_undegraded":"2024-08-17T17:17:38.511381+0000","last_fullsized":"2024-08-17T17:17:38.511381+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":154,"last_epoch_clean":155,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:27.399201+0000","last_clean_scrub_stamp":"2024-08-17T17:17:27.399201+0000","objects_scrubbed":0,"log_size":101,"log_dups_size":0,"ondisk_log_size":101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:36.105028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6619136,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":6464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":176,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652954+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:17:35.652954+0000","last_peered":"2024-08-17T17:17:35.652954+0000","last_clean":"2024-08-17T17:17:35.652954+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:17:35.652954+0000","last_undegraded":"2024-08-17T17:17:35.652954+0000","last_fullsized":"2024-08-17T17:17:35.652954+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.521794+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:17:30.521794+0000","last_peered":"2024-08-17T17:17:30.521794+0000","last_clean":"2024-08-17T17:17:30.521794+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:17:30.521794+0000","last_undegraded":"2024-08-17T17:17:30.521794+0000","last_fullsized":"2024-08-17T17:17:30.521794+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076028+0000","last_change":"2024-08-17T16:57:29.404285+0000","last_active":"2024-08-17T17:17:30.076028+0000","last_peered":"2024-08-17T17:17:30.076028+0000","last_clean":"2024-08-17T17:17:30.076028+0000","last_became_active":"2024-08-17T16:57:29.403979+0000","last_became_peered":"2024-08-17T16:57:29.403979+0000","last_unstale":"2024-08-17T17:17:30.076028+0000","last_undegraded":"2024-08-17T17:17:30.076028+0000","last_fullsized":"2024-08-17T17:17:30.076028+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:22.435263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":112,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076317+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:17:30.076317+0000","last_peered":"2024-08-17T17:17:30.076317+0000","last_clean":"2024-08-17T17:17:30.076317+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:17:30.076317+0000","last_undegraded":"2024-08-17T17:17:30.076317+0000","last_fullsized":"2024-08-17T17:17:30.076317+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":118,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076437+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:17:30.076437+0000","last_peered":"2024-08-17T17:17:30.076437+0000","last_clean":"2024-08-17T17:17:30.076437+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:17:30.076437+0000","last_undegraded":"2024-08-17T17:17:30.076437+0000","last_fullsized":"2024-08-17T17:17:30.076437+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":139,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.702682+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:17:35.702682+0000","last_peered":"2024-08-17T17:17:35.702682+0000","last_clean":"2024-08-17T17:17:35.702682+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:17:35.702682+0000","last_undegraded":"2024-08-17T17:17:35.702682+0000","last_fullsized":"2024-08-17T17:17:35.702682+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":116,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.414778+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:17:29.414778+0000","last_peered":"2024-08-17T17:17:29.414778+0000","last_clean":"2024-08-17T17:17:29.414778+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:17:29.414778+0000","last_undegraded":"2024-08-17T17:17:29.414778+0000","last_fullsized":"2024-08-17T17:17:29.414778+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413659+0000","last_change":"2024-08-17T17:07:51.484772+0000","last_active":"2024-08-17T17:17:29.413659+0000","last_peered":"2024-08-17T17:17:29.413659+0000","last_clean":"2024-08-17T17:17:29.413659+0000","last_became_active":"2024-08-17T17:07:51.484397+0000","last_became_peered":"2024-08-17T17:07:51.484397+0000","last_unstale":"2024-08-17T17:17:29.413659+0000","last_undegraded":"2024-08-17T17:17:29.413659+0000","last_fullsized":"2024-08-17T17:17:29.413659+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:21.701624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":87,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436552+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:17:35.436552+0000","last_peered":"2024-08-17T17:17:35.436552+0000","last_clean":"2024-08-17T17:17:35.436552+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:17:35.436552+0000","last_undegraded":"2024-08-17T17:17:35.436552+0000","last_fullsized":"2024-08-17T17:17:35.436552+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":172,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.653060+0000","last_change":"2024-08-17T17:12:40.341757+0000","last_active":"2024-08-17T17:17:35.653060+0000","last_peered":"2024-08-17T17:17:35.653060+0000","last_clean":"2024-08-17T17:17:35.653060+0000","last_became_active":"2024-08-17T17:12:40.341446+0000","last_became_peered":"2024-08-17T17:12:40.341446+0000","last_unstale":"2024-08-17T17:17:35.653060+0000","last_undegraded":"2024-08-17T17:17:35.653060+0000","last_fullsized":"2024-08-17T17:17:35.653060+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:51.178405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":99,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436442+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:17:35.436442+0000","last_peered":"2024-08-17T17:17:35.436442+0000","last_clean":"2024-08-17T17:17:35.436442+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:17:35.436442+0000","last_undegraded":"2024-08-17T17:17:35.436442+0000","last_fullsized":"2024-08-17T17:17:35.436442+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.521787+0000","last_change":"2024-08-17T17:07:51.484291+0000","last_active":"2024-08-17T17:17:30.521787+0000","last_peered":"2024-08-17T17:17:30.521787+0000","last_clean":"2024-08-17T17:17:30.521787+0000","last_became_active":"2024-08-17T17:07:51.483903+0000","last_became_peered":"2024-08-17T17:07:51.483903+0000","last_unstale":"2024-08-17T17:17:30.521787+0000","last_undegraded":"2024-08-17T17:17:30.521787+0000","last_fullsized":"2024-08-17T17:17:30.521787+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:02.755382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":117,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436744+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:17:35.436744+0000","last_peered":"2024-08-17T17:17:35.436744+0000","last_clean":"2024-08-17T17:17:35.436744+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:17:35.436744+0000","last_undegraded":"2024-08-17T17:17:35.436744+0000","last_fullsized":"2024-08-17T17:17:35.436744+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":173,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652836+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:17:35.652836+0000","last_peered":"2024-08-17T17:17:35.652836+0000","last_clean":"2024-08-17T17:17:35.652836+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:17:35.652836+0000","last_undegraded":"2024-08-17T17:17:35.652836+0000","last_fullsized":"2024-08-17T17:17:35.652836+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":138,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.702542+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:17:35.702542+0000","last_peered":"2024-08-17T17:17:35.702542+0000","last_clean":"2024-08-17T17:17:35.702542+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:17:35.702542+0000","last_undegraded":"2024-08-17T17:17:35.702542+0000","last_fullsized":"2024-08-17T17:17:35.702542+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":99,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436962+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:17:35.436962+0000","last_peered":"2024-08-17T17:17:35.436962+0000","last_clean":"2024-08-17T17:17:35.436962+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:17:35.436962+0000","last_undegraded":"2024-08-17T17:17:35.436962+0000","last_fullsized":"2024-08-17T17:17:35.436962+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":131,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076221+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:17:30.076221+0000","last_peered":"2024-08-17T17:17:30.076221+0000","last_clean":"2024-08-17T17:17:30.076221+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:17:30.076221+0000","last_undegraded":"2024-08-17T17:17:30.076221+0000","last_fullsized":"2024-08-17T17:17:30.076221+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436261+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:17:35.436261+0000","last_peered":"2024-08-17T17:17:35.436261+0000","last_clean":"2024-08-17T17:17:35.436261+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:17:35.436261+0000","last_undegraded":"2024-08-17T17:17:35.436261+0000","last_fullsized":"2024-08-17T17:17:35.436261+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":165,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652772+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:17:35.652772+0000","last_peered":"2024-08-17T17:17:35.652772+0000","last_clean":"2024-08-17T17:17:35.652772+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:17:35.652772+0000","last_undegraded":"2024-08-17T17:17:35.652772+0000","last_fullsized":"2024-08-17T17:17:35.652772+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076272+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:17:30.076272+0000","last_peered":"2024-08-17T17:17:30.076272+0000","last_clean":"2024-08-17T17:17:30.076272+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:17:30.076272+0000","last_undegraded":"2024-08-17T17:17:30.076272+0000","last_fullsized":"2024-08-17T17:17:30.076272+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":116,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413631+0000","last_change":"2024-08-17T17:07:51.482888+0000","last_active":"2024-08-17T17:17:29.413631+0000","last_peered":"2024-08-17T17:17:29.413631+0000","last_clean":"2024-08-17T17:17:29.413631+0000","last_became_active":"2024-08-17T17:07:51.482133+0000","last_became_peered":"2024-08-17T17:07:51.482133+0000","last_unstale":"2024-08-17T17:17:29.413631+0000","last_undegraded":"2024-08-17T17:17:29.413631+0000","last_fullsized":"2024-08-17T17:17:29.413631+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:47.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076124+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:17:30.076124+0000","last_peered":"2024-08-17T17:17:30.076124+0000","last_clean":"2024-08-17T17:17:30.076124+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:17:30.076124+0000","last_undegraded":"2024-08-17T17:17:30.076124+0000","last_fullsized":"2024-08-17T17:17:30.076124+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":99,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:29.413838+0000","last_change":"2024-08-17T16:57:28.383937+0000","last_active":"2024-08-17T17:17:29.413838+0000","last_peered":"2024-08-17T17:17:29.413838+0000","last_clean":"2024-08-17T17:17:29.413838+0000","last_became_active":"2024-08-17T16:57:28.382945+0000","last_became_peered":"2024-08-17T16:57:28.382945+0000","last_unstale":"2024-08-17T17:17:29.413838+0000","last_undegraded":"2024-08-17T17:17:29.413838+0000","last_fullsized":"2024-08-17T17:17:29.413838+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:53.363406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":130,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.436852+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:17:35.436852+0000","last_peered":"2024-08-17T17:17:35.436852+0000","last_clean":"2024-08-17T17:17:35.436852+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:17:35.436852+0000","last_undegraded":"2024-08-17T17:17:35.436852+0000","last_fullsized":"2024-08-17T17:17:35.436852+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":115,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.437178+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:17:35.437178+0000","last_peered":"2024-08-17T17:17:35.437178+0000","last_clean":"2024-08-17T17:17:35.437178+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:17:35.437178+0000","last_undegraded":"2024-08-17T17:17:35.437178+0000","last_fullsized":"2024-08-17T17:17:35.437178+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":112,"reported_epoch":156,"state":"active+clean","last_fresh":"2024-08-17T17:17:30.076077+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:17:30.076077+0000","last_peered":"2024-08-17T17:17:30.076077+0000","last_clean":"2024-08-17T17:17:30.076077+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:17:30.076077+0000","last_undegraded":"2024-08-17T17:17:30.076077+0000","last_fullsized":"2024-08-17T17:17:30.076077+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":177,"reported_epoch":157,"state":"active+clean","last_fresh":"2024-08-17T17:17:35.652859+0000","last_change":"2024-08-17T17:10:55.313325+0000","last_active":"2024-08-17T17:17:35.652859+0000","last_peered":"2024-08-17T17:17:35.652859+0000","last_clean":"2024-08-17T17:17:35.652859+0000","last_became_active":"2024-08-17T17:10:55.311737+0000","last_became_peered":"2024-08-17T17:10:55.311737+0000","last_unstale":"2024-08-17T17:17:35.652859+0000","last_undegraded":"2024-08-17T17:17:35.652859+0000","last_fullsized":"2024-08-17T17:17:35.652859+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:06.082766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":646316032,"num_objects":9862,"num_object_clones":0,"num_object_copies":29586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9862,"num_write_kb":631168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1985413120,"data_stored":1985413120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":9862,"ondisk_log_size":9862,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13723,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201549,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1018700,"kb_used_data":424644,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":92726452,"statfs":{"total":95995035648,"available":94951886848,"internally_reserved":0,"allocated":434835456,"data_stored":434039120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463185,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":470340,"kb_used_data":168828,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93274812,"statfs":{"total":95995035648,"available":95513407488,"internally_reserved":0,"allocated":172879872,"data_stored":172091585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724823,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1402996,"kb_used_data":505708,"kb_used_omap":1,"kb_used_meta":897278,"kb_avail":92342156,"statfs":{"total":95995035648,"available":94558367744,"internally_reserved":0,"allocated":517844992,"data_stored":517068048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918813016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986459,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":754560,"kb_used_data":156344,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":92990592,"statfs":{"total":95995035648,"available":95222366208,"internally_reserved":0,"allocated":160096256,"data_stored":159311321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225278,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":243564,"kb_used_data":34980,"kb_used_omap":4,"kb_used_meta":208571,"kb_avail":93501588,"statfs":{"total":95995035648,"available":95745626112,"internally_reserved":0,"allocated":35819520,"data_stored":35050269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":213577152},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075603,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":225080,"kb_used_data":1772,"kb_used_omap":6,"kb_used_meta":223289,"kb_avail":93520072,"statfs":{"total":95995035648,"available":95764553728,"internally_reserved":0,"allocated":1814528,"data_stored":1037872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":228648379},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533496,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618156,"kb_used_data":242916,"kb_used_omap":5,"kb_used_meta":375226,"kb_avail":93126996,"statfs":{"total":95995035648,"available":95362043904,"internally_reserved":0,"allocated":248745984,"data_stored":247975523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":384231664},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089473,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":511480,"kb_used_data":135344,"kb_used_omap":5,"kb_used_meta":376122,"kb_avail":93233672,"statfs":{"total":95995035648,"available":95471280128,"internally_reserved":0,"allocated":138592256,"data_stored":137812044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":385148987},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":181207040,"data_stored":181207040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":299958272,"data_stored":299958272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":33423360,"data_stored":33423360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":45809664,"data_stored":45809664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":172097536,"data_stored":172097536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":548077568,"data_stored":548077568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":183173120,"data_stored":183173120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":521666560,"data_stored":521666560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:17:49.434 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:17:49.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_4 pg_num 26 2024-08-17T17:17:49.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:49.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:49.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:49.649 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:49.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:49.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:49.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:49.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:49.932 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:50.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:50.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:50.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:50.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 32819 32803 170.818 139.562 0.00577498 0.00584874 2024-08-17T17:17:50.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:50.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:50.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:50.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:50.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:50.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:50 smithi152 ceph-mon[93456]: pgmap v514: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:17:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:17:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:17:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:17:50.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:50.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:50.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:50.733 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:50 smithi142 ceph-mon[103614]: pgmap v514: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:17:50.734 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:50 smithi142 ceph-mon[106729]: pgmap v514: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:17:50.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:50.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:50.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:50.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:50.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:50.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:51.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:51.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:51.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:51.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 34937 34921 167.861 132.375 0.00728328 0.00595201 2024-08-17T17:17:51.196 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:51.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:51.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:51.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:51.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:51.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:51.590 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:51.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:51.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:51.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:51.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:51.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:51.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:52.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:52.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:17:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:17:51] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.51.0" 2024-08-17T17:17:52.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:52.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 36744 36728 163.937 112.938 0.00542627 0.0060758 2024-08-17T17:17:52.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:52.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:52.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:52.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:52.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:52.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:52.512 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:52.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:52.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:52.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:52.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:52.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:52 smithi152 ceph-mon[93456]: pgmap v515: 89 pgs: 89 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-08-17T17:17:52.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:52 smithi142 ceph-mon[103614]: pgmap v515: 89 pgs: 89 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-08-17T17:17:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:52 smithi142 ceph-mon[106729]: pgmap v515: 89 pgs: 89 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-08-17T17:17:52.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:52.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:53.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:53.049 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:53.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:53.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 38430 38414 160.033 105.375 0.00220336 0.00624012 2024-08-17T17:17:53.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:53.279 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:53.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:53.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:53.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:53.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:53.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:53.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:53.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:53.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:53.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:53.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:53.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:53.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:53.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:54.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:54.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:54.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:54.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 40162 40146 156.796 108.25 0.00892505 0.00637243 2024-08-17T17:17:54.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:54.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:54.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:54.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:54.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:54.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:54.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:54.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:54.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:54.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:54 smithi142 ceph-mon[106729]: pgmap v516: 89 pgs: 89 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 172 MiB/s wr, 2.75k op/s 2024-08-17T17:17:54.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:54 smithi142 ceph-mon[103614]: pgmap v516: 89 pgs: 89 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 172 MiB/s wr, 2.75k op/s 2024-08-17T17:17:54.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:54.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:54.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:54 smithi152 ceph-mon[93456]: pgmap v516: 89 pgs: 89 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 172 MiB/s wr, 2.75k op/s 2024-08-17T17:17:54.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:54.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:55.007 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:55.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:55.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:55.168 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 15 41911 41896 154.006 109.375 0.00545736 0.00648855 2024-08-17T17:17:55.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:55.279 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:55.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:55.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:55.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:55.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:55.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:55.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:55.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:55.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:55.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:55.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:55.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:55.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:55.978 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:56.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:56.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:56.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:56.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 43389 43373 150.579 92.3125 0.00491676 0.0066363 2024-08-17T17:17:56.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:56.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:56.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:56.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:56.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:56.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:56.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:56.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:56.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:56 smithi142 ceph-mon[103614]: pgmap v517: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:17:56.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:56.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:56 smithi142 ceph-mon[106729]: pgmap v517: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:17:56.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:56.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:56.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:56 smithi152 ceph-mon[93456]: pgmap v517: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:17:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:56.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:56.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:56.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:57.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:57.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:57.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:57.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 44893 44877 147.6 94 0.00255283 0.00671205 2024-08-17T17:17:57.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:57.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:57.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:57.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:57.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:57.549 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:17:57.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:57.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:57.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:57.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2245664243' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:57 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:57.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:57.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2245664243' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:57 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2245664243' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:57 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:57.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:57.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:57.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:58.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:58.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:58.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:17:58.169172+0000 min lat: 0.00105076 max lat: 0.266487 avg lat: 0.00685737 2024-08-17T17:17:58.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:17:58.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 46654 46638 145.723 110.062 0.0151751 0.00685737 2024-08-17T17:17:58.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:58.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:58.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:58.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:58.399 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:17:58.400 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:58.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:58.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:58.434 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:17:58.434 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:58.473 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:17:58.473 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:58.509 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:17:58.509 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:58.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:58.543 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:17:58.543 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:58.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:58.578 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:17:58.578 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:58.607 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:17:58.607 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:58.638 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:17:58.638 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:58.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:58.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:58.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:58.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:58 smithi152 ceph-mon[93456]: pgmap v518: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-08-17T17:17:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:58 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:58 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:58 smithi152 ceph-mon[93456]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T17:17:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:17:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3246021444' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[106729]: pgmap v518: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-08-17T17:17:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[106729]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3246021444' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[103614]: pgmap v518: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[103614]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T17:17:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:17:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3246021444' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:58.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:58.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:59.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:59.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:59.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 48513 48497 144.316 116.188 0.00377514 0.00692413 2024-08-17T17:17:59.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:59.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:59.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:59.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:59.371 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:59.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:59.473 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:59.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:59.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:59.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:59.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.778 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:17:59.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:59.810 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:59.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:59.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:17:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:00.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:00.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:00.080 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:00.080 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:00.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:00.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 50369 50353 143.029 116 0.00508503 0.00698727 2024-08-17T17:18:00.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:00.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:00.297 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:00.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:00.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:00.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:00.488 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:18:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:18:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:18:00.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:00.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:00.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:00.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:00.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:00.744 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:00 smithi142 ceph-mon[103614]: pgmap v520: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:18:00.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:00 smithi142 ceph-mon[106729]: pgmap v520: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:18:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:00 smithi152 ceph-mon[93456]: pgmap v520: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:18:00.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:00.856 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:00.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:00.990 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:01.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:01.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:01.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 51991 51975 141.217 101.375 0.00579417 0.00707667 2024-08-17T17:18:01.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:01.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:01.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:01.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:18:01.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:01.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:01.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:01.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:01.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:01.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:01.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:01.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:01.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:01.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:01.933 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:02.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:02.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:18:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:18:01] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.51.0" 2024-08-17T17:18:02.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 53600 53584 139.523 100.562 0.00955343 0.00716287 2024-08-17T17:18:02.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:02.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:02.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:02.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:02.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:02.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.482 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:02.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:02.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:02.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:02.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:02 smithi142 ceph-mon[103614]: pgmap v521: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:18:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:02 smithi142 ceph-mon[106729]: pgmap v521: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:18:02.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:02.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:02 smithi152 ceph-mon[93456]: pgmap v521: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:18:02.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:02.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:02.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:03.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:03.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:18:03.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:03.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:03.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 55361 55345 138.344 110.062 0.00630655 0.00722372 2024-08-17T17:18:03.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:03.307 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:03.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:03.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:03.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:03.550 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:03.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:03.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:03.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:03.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:03.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:03.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:03.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:03.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:04.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:04.129 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:04.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 57023 57007 137.018 103.875 0.0100226 0.00729282 2024-08-17T17:18:04.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:04.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:04.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:04.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:04.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:04.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:04.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:04.675 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:04.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:04.727 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:04 smithi142 ceph-mon[103614]: pgmap v522: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-17T17:18:04.727 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:04 smithi142 ceph-mon[106729]: pgmap v522: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-17T17:18:04.751 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:04.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:04 smithi152 ceph-mon[93456]: pgmap v522: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-17T17:18:04.823 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:04.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:04.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:04.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:05.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:05.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:05.169 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 58537 58521 135.448 94.625 0.0158639 0.00737833 2024-08-17T17:18:05.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:05.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:05.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:05.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:05.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:05.518 INFO:teuthology.orchestra.run.smithi142.stderr:set pool 9 pg_num to 26 2024-08-17T17:18:05.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:05.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1399439711' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:05 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1399439711' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:05 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:05.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:05.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:05.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.796 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:05.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1399439711' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:05 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:05.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:05.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:05.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:06.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:06.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:06.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:06.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 60084 60068 134.063 96.6875 0.00823732 0.00745423 2024-08-17T17:18:06.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:06.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:18:06.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:06.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:06.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:06.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:06.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:06.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:06.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:06.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:06.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:06.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:06 smithi152 ceph-mon[93456]: pgmap v523: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:18:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:06 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:18:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:06 smithi152 ceph-mon[93456]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T17:18:06.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:06 smithi142 ceph-mon[103614]: pgmap v523: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:18:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:06 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:18:06.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:06 smithi142 ceph-mon[103614]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T17:18:06.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:06 smithi142 ceph-mon[106729]: pgmap v523: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:18:06.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:06 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:18:06.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:06 smithi142 ceph-mon[106729]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T17:18:06.904 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:07.007 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:07.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:07.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:07.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:07.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:07.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 61526 61510 132.548 90.125 0.00302191 0.00752239 2024-08-17T17:18:07.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:07.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:07.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:07.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:07.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.440 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:07.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:07.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:07.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:07.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:18:07.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:18:07.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:07.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:07.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:18:07.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:07.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:07.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:08.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:08.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:08.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 61944 61928 129 26.125 0.00323027 0.0076023 2024-08-17T17:18:08.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:08.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:08.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:08.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:08.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:08.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:08.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:08.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:08.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:08.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:08.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:08.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:08 smithi152 ceph-mon[93456]: pgmap v525: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-08-17T17:18:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:18:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:08 smithi152 ceph-mon[93456]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T17:18:08.835 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:08 smithi142 ceph-mon[106729]: pgmap v525: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-08-17T17:18:08.835 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:18:08.835 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:08 smithi142 ceph-mon[106729]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T17:18:08.836 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:08 smithi142 ceph-mon[103614]: pgmap v525: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-08-17T17:18:08.836 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:18:08.836 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:08 smithi142 ceph-mon[103614]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T17:18:08.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:08.875 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:08.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:09.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:09.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:09.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:09.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 63092 63076 127.154 71.75 0.00473241 0.00785965 2024-08-17T17:18:09.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.296 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:09.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:09.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:09.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:09.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:09.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:09.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[106729]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T17:18:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[106729]: 9.9 scrub starts 2024-08-17T17:18:09.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[106729]: 9.9 scrub ok 2024-08-17T17:18:09.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[106729]: 9.0 scrub starts 2024-08-17T17:18:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[103614]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T17:18:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[103614]: 9.9 scrub starts 2024-08-17T17:18:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[103614]: 9.9 scrub ok 2024-08-17T17:18:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:09 smithi142 ceph-mon[103614]: 9.0 scrub starts 2024-08-17T17:18:09.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:09.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:09.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:09 smithi152 ceph-mon[93456]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T17:18:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:09 smithi152 ceph-mon[93456]: 9.9 scrub starts 2024-08-17T17:18:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:09 smithi152 ceph-mon[93456]: 9.9 scrub ok 2024-08-17T17:18:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:09 smithi152 ceph-mon[93456]: 9.0 scrub starts 2024-08-17T17:18:09.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:09.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:09.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:10.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:10.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:10.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 64759 64743 126.436 104.188 0.0193664 0.00790429 2024-08-17T17:18:10.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:10.270 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:10.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:10.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:10.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:10.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:10.542 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:18:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:18:10.552 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:10.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:10.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:10.696 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:10.797 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: pgmap v528: 99 pgs: 5 peering, 7 unknown, 87 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.2 scrub starts 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.7 scrub starts 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.0 scrub ok 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.5 scrub starts 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.2 scrub ok 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.4 scrub starts 2024-08-17T17:18:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:10 smithi152 ceph-mon[93456]: 9.5 scrub ok 2024-08-17T17:18:10.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: pgmap v528: 99 pgs: 5 peering, 7 unknown, 87 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2024-08-17T17:18:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.2 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.7 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.0 scrub ok 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.5 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.2 scrub ok 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.4 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[106729]: 9.5 scrub ok 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: pgmap v528: 99 pgs: 5 peering, 7 unknown, 87 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.2 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.7 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.0 scrub ok 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.5 scrub starts 2024-08-17T17:18:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.2 scrub ok 2024-08-17T17:18:10.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.4 scrub starts 2024-08-17T17:18:10.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:10 smithi142 ceph-mon[103614]: 9.5 scrub ok 2024-08-17T17:18:10.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:10.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:10.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:11.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:11.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:11.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:11.148 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:11.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 66304 66288 125.53 96.5625 0.0095269 0.00796153 2024-08-17T17:18:11.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:11.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:11.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:11.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:11.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:11.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:11.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:11.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:11.557 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:18:11.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:11.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:11.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:11.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.7 scrub ok 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.6 scrub starts 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.1 scrub starts 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2104831358' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.15 scrub starts 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.4 scrub ok 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.1 scrub ok 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.10 scrub starts 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: 9.15 scrub ok 2024-08-17T17:18:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:11.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:18:11] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:18:11.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.7 scrub ok 2024-08-17T17:18:11.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.6 scrub starts 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.1 scrub starts 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2104831358' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.15 scrub starts 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.4 scrub ok 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.1 scrub ok 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.10 scrub starts 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: 9.15 scrub ok 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.7 scrub ok 2024-08-17T17:18:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.6 scrub starts 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.1 scrub starts 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2104831358' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.15 scrub starts 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.4 scrub ok 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.1 scrub ok 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.10 scrub starts 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: 9.15 scrub ok 2024-08-17T17:18:11.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:11.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:11.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:11.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:12.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:12.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:12.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 67942 67926 124.848 102.375 0.00509676 0.00800514 2024-08-17T17:18:12.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:12.214 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:12.316 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:12.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:12.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:12.478 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:12.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:12.555 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.6 scrub ok 2024-08-17T17:18:12.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:12.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:12.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:12.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.6 scrub ok 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: pgmap v529: 99 pgs: 1 active+clean+scrubbing, 5 peering, 5 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.17 scrub starts 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.8 scrub starts 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2104831358' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.19 scrub starts 2024-08-17T17:18:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.10 scrub ok 2024-08-17T17:18:12.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.14 scrub starts 2024-08-17T17:18:12.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:12 smithi152 ceph-mon[93456]: 9.19 scrub ok 2024-08-17T17:18:12.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: pgmap v529: 99 pgs: 1 active+clean+scrubbing, 5 peering, 5 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.17 scrub starts 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.8 scrub starts 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2104831358' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.19 scrub starts 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.10 scrub ok 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.14 scrub starts 2024-08-17T17:18:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[103614]: 9.19 scrub ok 2024-08-17T17:18:12.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.6 scrub ok 2024-08-17T17:18:12.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: pgmap v529: 99 pgs: 1 active+clean+scrubbing, 5 peering, 5 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.17 scrub starts 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.8 scrub starts 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2104831358' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.19 scrub starts 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.10 scrub ok 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.14 scrub starts 2024-08-17T17:18:12.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:12 smithi142 ceph-mon[106729]: 9.19 scrub ok 2024-08-17T17:18:12.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:12.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:12.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:13.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:13.129 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:13.166 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:13.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:13.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 69759 69743 124.526 113.562 0.0104388 0.00802593 2024-08-17T17:18:13.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:13.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:13.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:13.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:13.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.446 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:13.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:13.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:13.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:13.605 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[103614]: 9.17 scrub ok 2024-08-17T17:18:13.605 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[103614]: 9.8 scrub ok 2024-08-17T17:18:13.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[103614]: 9.12 scrub starts 2024-08-17T17:18:13.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[103614]: 9.14 scrub ok 2024-08-17T17:18:13.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[106729]: 9.17 scrub ok 2024-08-17T17:18:13.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[106729]: 9.8 scrub ok 2024-08-17T17:18:13.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[106729]: 9.12 scrub starts 2024-08-17T17:18:13.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:13 smithi142 ceph-mon[106729]: 9.14 scrub ok 2024-08-17T17:18:13.660 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:13.660 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:13.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:13.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:13.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:13 smithi152 ceph-mon[93456]: 9.17 scrub ok 2024-08-17T17:18:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:13 smithi152 ceph-mon[93456]: 9.8 scrub ok 2024-08-17T17:18:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:13 smithi152 ceph-mon[93456]: 9.12 scrub starts 2024-08-17T17:18:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:13 smithi152 ceph-mon[93456]: 9.14 scrub ok 2024-08-17T17:18:13.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:13.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:13.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:13.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:14.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:14.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:14.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 71360 71344 123.846 100.062 0.00533308 0.00806947 2024-08-17T17:18:14.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:14.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:14.356 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:14.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:18:14.391 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:14.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:14.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:14.626 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:18:14.626 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:18:14.626 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-17T17:18:14.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-17T17:18:14.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:14.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:14.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:18:14.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:14 smithi152 ceph-mon[93456]: 9.12 scrub ok 2024-08-17T17:18:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:14 smithi152 ceph-mon[93456]: pgmap v531: 99 pgs: 1 active+clean+scrubbing, 3 peering, 95 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail 2024-08-17T17:18:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:14 smithi152 ceph-mon[93456]: 9.11 scrub starts 2024-08-17T17:18:14.823 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:14 smithi142 ceph-mon[103614]: 9.12 scrub ok 2024-08-17T17:18:14.823 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:14 smithi142 ceph-mon[103614]: pgmap v531: 99 pgs: 1 active+clean+scrubbing, 3 peering, 95 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail 2024-08-17T17:18:14.823 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:14 smithi142 ceph-mon[103614]: 9.11 scrub starts 2024-08-17T17:18:14.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:14 smithi142 ceph-mon[106729]: 9.12 scrub ok 2024-08-17T17:18:14.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:14 smithi142 ceph-mon[106729]: pgmap v531: 99 pgs: 1 active+clean+scrubbing, 3 peering, 95 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail 2024-08-17T17:18:14.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:14 smithi142 ceph-mon[106729]: 9.11 scrub starts 2024-08-17T17:18:14.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:14.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:14.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:14.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:15.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:15.096 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:18:15.096 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:15.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:15.135 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:18:15.135 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:15.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 72798 72782 122.928 89.875 0.0202915 0.00812889 2024-08-17T17:18:15.181 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:18:15.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:15.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:15.223 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:18:15.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:15.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:15.235 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:15.235 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:15.261 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:18:15.262 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:15.291 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:18:15.291 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:15.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:18:15.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:15.326 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:15.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:15.352 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:18:15.352 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:15.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:15.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:15.524 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[106729]: 9.11 scrub ok 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[106729]: 9.18 scrub starts 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1040963701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[103614]: 9.11 scrub ok 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[103614]: 9.18 scrub starts 2024-08-17T17:18:15.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1040963701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:15.605 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:18:15.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:15.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:15.767 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:15.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:15.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:15.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:16.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:16.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:16.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:15 smithi152 ceph-mon[93456]: 9.11 scrub ok 2024-08-17T17:18:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:15 smithi152 ceph-mon[93456]: 9.18 scrub starts 2024-08-17T17:18:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1040963701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T17:18:16.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:16.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 73890 73874 121.488 68.25 0.00191317 0.00814221 2024-08-17T17:18:16.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:16.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:16.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.300 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:16.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:16.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:16.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:16.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:16.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:16.649 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:16.670 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:16.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:16.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[106729]: 9.18 scrub ok 2024-08-17T17:18:16.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[106729]: pgmap v532: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2024-08-17T17:18:16.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:18:16.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[106729]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T17:18:16.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[103614]: 9.18 scrub ok 2024-08-17T17:18:16.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[103614]: pgmap v532: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2024-08-17T17:18:16.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:18:16.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:16 smithi142 ceph-mon[103614]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T17:18:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:16.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:16.898 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:16.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:16.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:17.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:17.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:16 smithi152 ceph-mon[93456]: 9.18 scrub ok 2024-08-17T17:18:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:16 smithi152 ceph-mon[93456]: pgmap v532: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2024-08-17T17:18:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:16 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T17:18:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:16 smithi152 ceph-mon[93456]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T17:18:17.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:17.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:17.170 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 73890 73874 118.373 0 - 0.00814221 2024-08-17T17:18:17.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:17.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:17.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:17.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:17.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:17.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:17.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:17.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:17.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:17.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:17.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:17 smithi142 ceph-mon[106729]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T17:18:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:17 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:18:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:17 smithi142 ceph-mon[103614]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T17:18:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:18:17.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:17.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:18.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:18.033 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:18.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:17 smithi152 ceph-mon[93456]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T17:18:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:17 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T17:18:18.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:18.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:18:18.171145+0000 min lat: 0.00105076 max lat: 2.38813 avg lat: 0.00863591 2024-08-17T17:18:18.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:18:18.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 74062 74046 115.683 5.375 0.00691805 0.00863591 2024-08-17T17:18:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:18.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:18.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:18.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:18:18.301 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:18.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:18.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:18.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:18.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:18.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:18.789 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:18.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[106729]: 9.3 scrub starts 2024-08-17T17:18:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[106729]: 9.3 scrub ok 2024-08-17T17:18:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[106729]: pgmap v535: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-08-17T17:18:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:18:18.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[106729]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T17:18:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[103614]: 9.3 scrub starts 2024-08-17T17:18:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[103614]: 9.3 scrub ok 2024-08-17T17:18:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[103614]: pgmap v535: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-08-17T17:18:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:18:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:18 smithi142 ceph-mon[103614]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T17:18:18.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:18.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:19.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:18 smithi152 ceph-mon[93456]: 9.3 scrub starts 2024-08-17T17:18:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:18 smithi152 ceph-mon[93456]: 9.3 scrub ok 2024-08-17T17:18:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:18 smithi152 ceph-mon[93456]: pgmap v535: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2024-08-17T17:18:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:18 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T17:18:19.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:18 smithi152 ceph-mon[93456]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T17:18:19.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:19.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:19.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 75178 75162 114.562 69.75 0.00733593 0.00872462 2024-08-17T17:18:19.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:19.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:19.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:19.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:19.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:19.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:19.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:19.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:19.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:19.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:19.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:19.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:19.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:19.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:19.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:19.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:19 smithi142 ceph-mon[106729]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T17:18:19.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:18:19.973 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:19 smithi142 ceph-mon[103614]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T17:18:19.973 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:18:20.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:20.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:19 smithi152 ceph-mon[93456]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T17:18:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-08-17T17:18:20.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:20.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 76351 76335 113.58 73.3125 0.00203027 0.00871867 2024-08-17T17:18:20.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:20.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:20.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:20.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:20.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:20.407 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:18:20.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:20.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:20.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:20.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:18:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:18:20.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:20.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:20.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:20.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:20.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:20.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:20.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:20.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:20.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: pgmap v538: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-08-17T17:18:20.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3125141066' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3125141066' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[103614]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: pgmap v538: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3125141066' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:20.992 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:20.993 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3125141066' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:20.993 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:20 smithi142 ceph-mon[106729]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T17:18:21.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:21.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:21.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: pgmap v538: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3125141066' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3125141066' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:20 smithi152 ceph-mon[93456]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T17:18:21.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:21.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 76351 76335 110.939 0 - 0.00871867 2024-08-17T17:18:21.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:21.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:21.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:21.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:21.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:21.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:21.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:21.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:21.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:21.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:21.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:21.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:21.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-mon[103614]: 9.16 deep-scrub starts 2024-08-17T17:18:21.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-mon[103614]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T17:18:21.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:18:21] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:18:21.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-mon[106729]: 9.16 deep-scrub starts 2024-08-17T17:18:21.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-mon[106729]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T17:18:21.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:21.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:21.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:21.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:21 smithi152 ceph-mon[93456]: 9.16 deep-scrub starts 2024-08-17T17:18:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:21 smithi152 ceph-mon[93456]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T17:18:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:22.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:22.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:22.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:22.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 77242 77226 109.683 27.8438 0.00519952 0.00911229 2024-08-17T17:18:22.203 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76678 8.7 8:e98d2fe7:::benchmark_data_smithi142_139631_object76677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.330434+0000", 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 342.86996404600001, 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3664621810000002, 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.204 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76678 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330434+0000", 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.205 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330434+0000", 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330436+0000", 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3209999999999999e-06 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330459+0000", 2024-08-17T17:18:22.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3008000000000001e-05 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330469+0000", 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0740000000000001e-05 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330477+0000", 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8690000000000004e-06 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.207 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.342648+0000", 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012170729 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.342658+0000", 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8109999999999997e-06 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.332589+0000", 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98993133600000005 2024-08-17T17:18:22.208 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.332599+0000", 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0441e-05 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686590+0000", 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3539904709999999 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.209 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686696+0000", 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105598 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686804+0000", 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010843700000000001 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695010+0000", 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0082063510000000006 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695058+0000", 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7336999999999998e-05 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695141+0000", 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2928000000000003e-05 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695204+0000", 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3542999999999998e-05 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.696896+0000", 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001692261 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.212 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76729 8.7 8:e5dda4d8:::benchmark_data_smithi142_139631_object76728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 342.81682346399998, 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3218559409999999, 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:18:22.213 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76729 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0399999999999996e-07 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383612+0000", 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6918999999999997e-05 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383621+0000", 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1730000000000001e-06 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383627+0000", 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7529999999999998e-06 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383690+0000", 2024-08-17T17:18:22.216 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2704000000000001e-05 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383699+0000", 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0589999999999992e-06 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334127+0000", 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95042768600000005 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334134+0000", 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0670000000000003e-06 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.689412+0000", 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.355278092 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.689538+0000", 2024-08-17T17:18:22.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012647900000000001 2024-08-17T17:18:22.219 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.219 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.219 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:18:22.219 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.689690+0000", 2024-08-17T17:18:22.219 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015227200000000001 2024-08-17T17:18:22.219 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695072+0000", 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005381533 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.696946+0000", 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0018738050000000001 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.703090+0000", 2024-08-17T17:18:22.220 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0061438350000000003 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.703285+0000", 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019538699999999999 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.705431+0000", 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0021456729999999999 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.221 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76731 8.f 8:f7eca805:::benchmark_data_smithi142_139631_object76730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:40.327572+0000", 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 341.87282594999999, 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.404185164, 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:18:22.222 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76731 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327572+0000", 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327572+0000", 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.223 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327573+0000", 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9400000000000004e-07 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327633+0000", 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9722000000000002e-05 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327641+0000", 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1869999999999997e-06 2024-08-17T17:18:22.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327648+0000", 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7299999999999999e-06 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327716+0000", 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7831999999999997e-05 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327721+0000", 2024-08-17T17:18:22.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7610000000000004e-06 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334094+0000", 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063721790000000004 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334101+0000", 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.226 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.706562+0000", 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.372461156 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.706682+0000", 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000119981 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.706840+0000", 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000157236 2024-08-17T17:18:22.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.714756+0000", 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079168780000000005 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.717828+0000", 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030717959999999999 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.729937+0000", 2024-08-17T17:18:22.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012108934 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.730148+0000", 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000210924 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.731757+0000", 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001609425 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.229 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76744 8.f 8:f39abda7:::benchmark_data_smithi142_139631_object76743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:40.327674+0000", 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 341.87272385900002, 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4061216009999999, 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:18:22.230 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76744 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327674+0000", 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327674+0000", 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327675+0000", 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0999999999999998e-07 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327798+0000", 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012282499999999999 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327807+0000", 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9979999999999992e-06 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327814+0000", 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7190000000000004e-06 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327924+0000", 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000110155 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327929+0000", 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7729999999999996e-06 2024-08-17T17:18:22.233 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334270+0000", 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063409240000000004 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334280+0000", 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2369999999999998e-06 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.234 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.711398+0000", 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3771184869999999 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.711547+0000", 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014860000000000001 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.711697+0000", 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014978999999999999 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.235 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.717702+0000", 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060053709999999998 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.731817+0000", 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014114658 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.732016+0000", 2024-08-17T17:18:22.236 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019955799999999999 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.732183+0000", 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016642999999999999 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.733796+0000", 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001613466 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.237 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73423 9.10 9:0e0af26c:::benchmark_data_smithi142_157452_object73422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5930872340000004, 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4164945229999999, 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.238 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73423 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607312+0000", 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4619999999999999e-06 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607340+0000", 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7764999999999999e-05 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607353+0000", 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2471e-05 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607364+0000", 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1392999999999999e-05 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607428+0000", 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3658000000000006e-05 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.241 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607439+0000", 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0777e-05 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013664+0000", 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4062256989999999 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.242 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013801+0000", 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000136594 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013929+0000", 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012783400000000001 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018572+0000", 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046436580000000002 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018584+0000", 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1574e-05 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018612+0000", 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7915e-05 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022687+0000", 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.004075543 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022742+0000", 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4341e-05 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023806+0000", 2024-08-17T17:18:22.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001063839 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73452 9.10 9:08329d80:::benchmark_data_smithi142_157452_object73451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5930184729999999, 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4217852020000001, 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.246 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73452 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.15e-07 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607410+0000", 2024-08-17T17:18:22.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9464000000000001e-05 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607416+0000", 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2319999999999998e-06 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607420+0000", 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2649999999999998e-06 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607541+0000", 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012065699999999999 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607548+0000", 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1559999999999998e-06 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016317+0000", 2024-08-17T17:18:22.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.408768746 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016455+0000", 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013853699999999999 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016639+0000", 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018399900000000001 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018680+0000", 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020403840000000001 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023861+0000", 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051812050000000004 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027763+0000", 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039021400000000001 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.028010+0000", 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024739199999999998 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.028049+0000", 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9082000000000003e-05 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029165+0000", 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001115528 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73460 9.10 9:08db73f6:::benchmark_data_smithi142_157452_object73459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5929577469999998, 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4227749750000001, 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.254 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73460 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6199999999999999e-07 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607459+0000", 2024-08-17T17:18:22.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7660999999999999e-05 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607463+0000", 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7570000000000004e-06 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607468+0000", 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.34e-06 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607628+0000", 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016045799999999999 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607638+0000", 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8360000000000004e-06 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017660+0000", 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.410022291 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017755+0000", 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4372999999999999e-05 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017851+0000", 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6256999999999997e-05 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020381+0000", 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025300259999999999 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027826+0000", 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0074454389999999999 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027858+0000", 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1591000000000003e-05 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029231+0000", 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001372877 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029290+0000", 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9687e-05 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030215+0000", 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00092502000000000003 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73489 9.10 9:0a567102:::benchmark_data_smithi142_157452_object73488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.262 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:18:22.263 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5929251339999997, 2024-08-17T17:18:22.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4276421909999999, 2024-08-17T17:18:22.263 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.263 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.263 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.263 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:22.264 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73489 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8599999999999999e-07 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607502+0000", 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8651000000000001e-05 2024-08-17T17:18:22.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607511+0000", 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7800000000000006e-06 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607517+0000", 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8259999999999998e-06 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607995+0000", 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00047819600000000001 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608003+0000", 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4479999999999994e-06 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018702+0000", 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4106989999999999 2024-08-17T17:18:22.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018795+0000", 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2928000000000002e-05 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018901+0000", 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105565 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022627+0000", 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037258590000000002 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027930+0000", 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053030780000000001 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033448+0000", 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005518406 2024-08-17T17:18:22.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033579+0000", 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013044700000000001 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033635+0000", 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5965999999999999e-05 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035115+0000", 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001480755 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73517 9.10 9:09edc6df:::benchmark_data_smithi142_157452_object73516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5928546690000003, 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.428877572, 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.272 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73517 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5499999999999998e-07 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607565+0000", 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0472e-05 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607569+0000", 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4059999999999998e-06 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607572+0000", 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3560000000000001e-06 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608033+0000", 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046030099999999998 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608037+0000", 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9820000000000002e-06 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019497+0000", 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.411460553 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019597+0000", 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9899e-05 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019700+0000", 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000102571 2024-08-17T17:18:22.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023871+0000", 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041716840000000002 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027970+0000", 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040992240000000003 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033519+0000", 2024-08-17T17:18:22.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0055483060000000002 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035184+0000", 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0016652030000000001 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035240+0000", 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5546e-05 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036421+0000", 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011816140000000001 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73531 9.10 9:0aa603eb:::benchmark_data_smithi142_157452_object73530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5928200190000004, 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4341237490000001, 2024-08-17T17:18:22.280 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73531 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607579+0000", 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.03e-07 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607597+0000", 2024-08-17T17:18:22.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8376000000000002e-05 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607601+0000", 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.963e-06 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607604+0000", 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8870000000000001e-06 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608070+0000", 2024-08-17T17:18:22.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046594300000000002 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608077+0000", 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2099999999999996e-06 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020422+0000", 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4123452620000001 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020558+0000", 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135914 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020719+0000", 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016086799999999999 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023882+0000", 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003162621 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036562+0000", 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012679953000000001 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040088+0000", 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035266020000000002 2024-08-17T17:18:22.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040135+0000", 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6859e-05 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040193+0000", 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8128000000000003e-05 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041702+0000", 2024-08-17T17:18:22.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0015088600000000001 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73543 9.10 9:0a994c9f:::benchmark_data_smithi142_157452_object73542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5927780519999999, 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4357055299999999, 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.288 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73543 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607621+0000", 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3599999999999999e-07 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607638+0000", 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.702e-05 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607644+0000", 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.037e-06 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607647+0000", 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0410000000000002e-06 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608107+0000", 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045997900000000001 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608110+0000", 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3220000000000001e-06 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021537+0000", 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.413427129 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021655+0000", 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117779 2024-08-17T17:18:22.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021811+0000", 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015619200000000001 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026398+0000", 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045870349999999997 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036616+0000", 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010218213 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041785+0000", 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051685489999999997 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041907+0000", 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012203500000000001 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041969+0000", 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2238999999999998e-05 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043326+0000", 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013566240000000001 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.295 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73564 9.10 9:0c98c0a8:::benchmark_data_smithi142_157452_object73563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5927469299999997, 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.441867732, 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73564 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607652+0000", 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1199999999999999e-07 2024-08-17T17:18:22.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607715+0000", 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3386000000000006e-05 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607725+0000", 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8679999999999994e-06 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607730+0000", 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.04e-06 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608144+0000", 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00041373799999999997 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608152+0000", 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4060000000000005e-06 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023936+0000", 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.415784218 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.024030+0000", 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3970999999999999e-05 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.024123+0000", 2024-08-17T17:18:22.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2508000000000003e-05 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027775+0000", 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003651774 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036678+0000", 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0089036219999999999 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043422+0000", 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067443060000000003 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.048000+0000", 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045772770000000003 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.048076+0000", 2024-08-17T17:18:22.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6469000000000003e-05 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049519+0000", 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001442837 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73583 9.10 9:0b96fbe3:::benchmark_data_smithi142_157452_object73582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:18:22.303 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5926636529999998, 2024-08-17T17:18:22.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.442907784, 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.305 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73583 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8999999999999998e-07 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607756+0000", 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1569e-05 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607761+0000", 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5689999999999998e-06 2024-08-17T17:18:22.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607764+0000", 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9529999999999999e-06 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608180+0000", 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000415847 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608183+0000", 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1930000000000002e-06 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025148+0000", 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4169648399999999 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025286+0000", 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138417 2024-08-17T17:18:22.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025447+0000", 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000160946 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027868+0000", 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00242055 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041848+0000", 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013980726000000001 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047049+0000", 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052002410000000004 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049578+0000", 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025293820000000002 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049642+0000", 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3837000000000006e-05 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050642+0000", 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001000424 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73678 9.10 9:08769dcd:::benchmark_data_smithi142_157452_object73677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.592630185, 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.44418486, 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.314 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73678 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.65e-07 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607782+0000", 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3886999999999999e-05 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607787+0000", 2024-08-17T17:18:22.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6739999999999996e-06 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607790+0000", 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7970000000000001e-06 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608215+0000", 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00042531699999999998 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608224+0000", 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9339999999999995e-06 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026442+0000", 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.418218049 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026597+0000", 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015451500000000001 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026767+0000", 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170335 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027979+0000", 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012120810000000001 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050711+0000", 2024-08-17T17:18:22.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.022731950000000001 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050763+0000", 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2281999999999999e-05 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050966+0000", 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020286300000000001 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051021+0000", 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5359000000000003e-05 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051953+0000", 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00093155199999999999 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73695 9.10 9:0bc067c9:::benchmark_data_smithi142_157452_object73694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.321 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5926049400000002, 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4639758309999999, 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73695 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607794+0000", 2024-08-17T17:18:22.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.96e-07 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607808+0000", 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4695000000000001e-05 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607813+0000", 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5129999999999998e-06 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607816+0000", 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7240000000000001e-06 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608255+0000", 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00043993199999999997 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608260+0000", 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9010000000000002e-06 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030261+0000", 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4220003640000001 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030373+0000", 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011251000000000001 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030485+0000", 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111542 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033537+0000", 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030526939999999999 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050876+0000", 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017338675000000001 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051992+0000", 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011160079999999999 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.070213+0000", 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018221029 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.070274+0000", 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0436999999999998e-05 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071769+0000", 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014955109999999999 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73736 9.10 9:0f48bdcd:::benchmark_data_smithi142_157452_object73735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5925790700000002, 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4653223989999999, 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.329 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73736 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4999999999999999e-07 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607834+0000", 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4226000000000001e-05 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607841+0000", 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4939999999999997e-06 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607847+0000", 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3850000000000001e-06 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608291+0000", 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000444864 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608300+0000", 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6119999999999999e-06 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032028+0000", 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4237279540000001 2024-08-17T17:18:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032170+0000", 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014252899999999999 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032268+0000", 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7950999999999998e-05 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036506+0000", 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042373360000000004 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054725+0000", 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018219227000000001 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055000+0000", 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00027527500000000001 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071822+0000", 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016821922999999999 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071885+0000", 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2649000000000003e-05 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073142+0000", 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012567240000000001 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.336 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73777 9.10 9:08e4faeb:::benchmark_data_smithi142_157452_object73776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5925470449999999, 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4667512089999999, 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73777 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:18:22.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4900000000000002e-07 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607866+0000", 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4863e-05 2024-08-17T17:18:22.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607870+0000", 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0010000000000003e-06 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607876+0000", 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8320000000000002e-06 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608327+0000", 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045121099999999997 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608331+0000", 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9169999999999999e-06 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036717+0000", 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4283860289999999 2024-08-17T17:18:22.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036865+0000", 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014740099999999999 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.037067+0000", 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020239499999999999 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043346+0000", 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062784049999999999 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054765+0000", 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011419155 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073198+0000", 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018433105000000002 2024-08-17T17:18:22.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073365+0000", 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016761400000000001 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073424+0000", 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.876e-05 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074602+0000", 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001178272 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73834 9.10 9:0efa507b:::benchmark_data_smithi142_157452_object73833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5925043829999996, 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.467875166, 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.345 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73834 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0999999999999999e-07 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607911+0000", 2024-08-17T17:18:22.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7147999999999999e-05 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607916+0000", 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9400000000000001e-06 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607921+0000", 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1590000000000003e-06 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608364+0000", 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00044320200000000002 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608372+0000", 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8730000000000007e-06 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038589+0000", 2024-08-17T17:18:22.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.430217689 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038737+0000", 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014733899999999999 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038912+0000", 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017523699999999999 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043352+0000", 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044401780000000004 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054813+0000", 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011461135000000001 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073240+0000", 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018426973999999999 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074657+0000", 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014169829999999999 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074711+0000", 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4185999999999999e-05 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075769+0000", 2024-08-17T17:18:22.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010577130000000001 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73885 9.10 9:08993fba:::benchmark_data_smithi142_157452_object73884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5924735559999998, 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.468984944, 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.353 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73885 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5499999999999999e-07 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607940+0000", 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4739999999999999e-05 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607944+0000", 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5630000000000002e-06 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607947+0000", 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7439999999999999e-06 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608397+0000", 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045033600000000002 2024-08-17T17:18:22.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608401+0000", 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5870000000000001e-06 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043476+0000", 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4350746729999999 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043632+0000", 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000156176 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043817+0000", 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000184754 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047067+0000", 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032502579999999998 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055046+0000", 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079788489999999997 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073280+0000", 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018234017000000002 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075815+0000", 2024-08-17T17:18:22.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025350659999999999 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075873+0000", 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8557000000000002e-05 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.076910+0000", 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010363690000000001 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.360 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73890 9.10 9:0c2319f1:::benchmark_data_smithi142_157452_object73889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 5.5924476939999996, 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5039437520000001, 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:18:22.361 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73890 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4900000000000002e-07 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607965+0000", 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3899e-05 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607969+0000", 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5879999999999999e-06 2024-08-17T17:18:22.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607972+0000", 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7520000000000001e-06 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608429+0000", 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045679500000000002 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:18:22.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608437+0000", 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3999999999999992e-06 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045264+0000", 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.436826288 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045402+0000", 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138221 2024-08-17T17:18:22.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045554+0000", 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000152585 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047073+0000", 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001518637 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055084+0000", 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080111230000000002 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073321+0000", 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018236588000000001 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.077006+0000", 2024-08-17T17:18:22.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0036852550000000001 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.077063+0000", 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6827999999999999e-05 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.111894+0000", 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034831543999999999 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.368 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.369 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.369 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:18:22.369 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:18:22.369 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:18:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:22.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:22.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:22.501 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:22.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:22.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:22.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:22.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:22.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:18:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:18:22.815 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:18:22.815 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:18:22.815 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:22 smithi142 ceph-mon[103614]: pgmap v541: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6528/222708 objects misplaced (2.931%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:18:22.815 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:22 smithi142 ceph-mon[106729]: pgmap v541: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6528/222708 objects misplaced (2.931%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:18:22.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:22.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:23.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:23.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:22 smithi152 ceph-mon[93456]: pgmap v541: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6528/222708 objects misplaced (2.931%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:18:23.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:23.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 78601 78585 109.133 84.9375 0.00815661 0.00915842 2024-08-17T17:18:23.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:23.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:23.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:23.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:23.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:23.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:23.595 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:23.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:23.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:23.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:23.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:23.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:18:23.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.032 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:24.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:24.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:24.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:24.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 79938 79922 108.577 83.5625 0.00225349 0.00920159 2024-08-17T17:18:24.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:24.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:24.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:24.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:24.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:24.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:24.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:24.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:24.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:24.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:24.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:24.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:24.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:25.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:25.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:24 smithi152 ceph-mon[93456]: pgmap v543: 99 pgs: 1 active+clean+scrubbing+deep, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4.4 MiB/s wr, 71 op/s; 6528/229197 objects misplaced (2.848%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:18:25.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:24 smithi142 ceph-mon[106729]: pgmap v543: 99 pgs: 1 active+clean+scrubbing+deep, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4.4 MiB/s wr, 71 op/s; 6528/229197 objects misplaced (2.848%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:18:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:24 smithi142 ceph-mon[103614]: pgmap v543: 99 pgs: 1 active+clean+scrubbing+deep, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4.4 MiB/s wr, 71 op/s; 6528/229197 objects misplaced (2.848%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:18:25.146 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:25.171 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 81369 81353 108.169 89.4375 0.00283938 0.00923922 2024-08-17T17:18:25.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:25.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:25.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:25.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:25.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:25.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:25.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:25.692 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:25.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:25.818 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:25.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:25.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:25.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:25.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:25 smithi152 ceph-mon[93456]: 9.16 deep-scrub ok 2024-08-17T17:18:26.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:26.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:26.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:26.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:25 smithi142 ceph-mon[103614]: 9.16 deep-scrub ok 2024-08-17T17:18:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:25 smithi142 ceph-mon[106729]: 9.16 deep-scrub ok 2024-08-17T17:18:26.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 82734 82718 107.693 85.3125 0.00221455 0.00926644 2024-08-17T17:18:26.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:26.236 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:26.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:26.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:26.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:26.510 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:26.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:26.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:26.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:26.748 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:26.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:26.894 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:26.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:26.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:27.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:27.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:27.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:26 smithi152 ceph-mon[93456]: pgmap v544: 99 pgs: 1 active+clean+scrubbing+deep, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 13 MiB/s wr, 213 op/s; 6528/232020 objects misplaced (2.814%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:18:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:27.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:26 smithi142 ceph-mon[103614]: pgmap v544: 99 pgs: 1 active+clean+scrubbing+deep, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 13 MiB/s wr, 213 op/s; 6528/232020 objects misplaced (2.814%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:18:27.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:26 smithi142 ceph-mon[106729]: pgmap v544: 99 pgs: 1 active+clean+scrubbing+deep, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 13 MiB/s wr, 213 op/s; 6528/232020 objects misplaced (2.814%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:18:27.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:27.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:27.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 84046 84030 107.169 82 0.00966015 0.00932625 2024-08-17T17:18:27.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:27.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:27.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:27.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:27.347 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:18:27.347 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:18:27.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-08-17T17:18:27.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:27.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:27.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:27.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:27.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:27.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:27.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:27.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:27.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:27.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:27.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:27.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:27.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:27 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 4694/243552 objects degraded (1.927%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:27.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:27.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:28.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:27 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 4694/243552 objects degraded (1.927%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:28.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:27 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 4694/243552 objects degraded (1.927%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:28.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:28.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:28.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 85185 85169 106.449 71.1875 0.0114972 0.00938945 2024-08-17T17:18:28.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:28.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:28.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:28.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:28.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:28.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:28.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:28.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:28.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:28.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:28.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:28.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:28.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:28.925 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:29.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:29.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:28 smithi152 ceph-mon[93456]: pgmap v545: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 22 MiB/s wr, 354 op/s; 4694/243552 objects degraded (1.927%); 5916/243552 objects misplaced (2.429%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:18:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:28 smithi142 ceph-mon[106729]: pgmap v545: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 22 MiB/s wr, 354 op/s; 4694/243552 objects degraded (1.927%); 5916/243552 objects misplaced (2.429%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:18:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:28 smithi142 ceph-mon[103614]: pgmap v545: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 22 MiB/s wr, 354 op/s; 4694/243552 objects degraded (1.927%); 5916/243552 objects misplaced (2.429%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:18:29.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:29.172 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 86398 86382 105.846 75.8125 0.0060343 0.00944128 2024-08-17T17:18:29.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:29.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:29.317 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:29.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:29.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:29.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:29.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:29.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:29.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:29.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:29.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:29.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:29.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:29.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:29.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:30.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:30.122 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:30.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:30.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 87486 87470 105.119 68 0.00502902 0.00950824 2024-08-17T17:18:30.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:30.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:30.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:30.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:30.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:30.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:30.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:30.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:18:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:18:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:18:30.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:30.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:30.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:30.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:30.801 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:30.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:30.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:30.996 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:30 smithi152 ceph-mon[93456]: pgmap v546: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 4694/251571 objects degraded (1.866%); 5916/251571 objects misplaced (2.352%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:18:30.996 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:31.039 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:31.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:30 smithi142 ceph-mon[106729]: pgmap v546: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 4694/251571 objects degraded (1.866%); 5916/251571 objects misplaced (2.352%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:18:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:30 smithi142 ceph-mon[103614]: pgmap v546: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 603 op/s; 4694/251571 objects degraded (1.866%); 5916/251571 objects misplaced (2.352%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:18:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:31.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 88744 88728 104.619 78.625 0.00502506 0.00955397 2024-08-17T17:18:31.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:31.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:31.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:31.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:31.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:31.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:31.496 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:31.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:31.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:31.755 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:31.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:31.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:32.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:32.057 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:32.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:18:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:18:31] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-08-17T17:18:32.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:32.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 89933 89917 104.057 74.3125 0.00443351 0.0096057 2024-08-17T17:18:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:32.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:32.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:32.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:32.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:32.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:32.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:32.597 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:32.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:32.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:32.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:32.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:32.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:32.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:32.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:18:32.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:32.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:32 smithi152 ceph-mon[93456]: pgmap v547: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 4694/254451 objects degraded (1.845%); 5484/254451 objects misplaced (2.155%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:18:33.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:32 smithi142 ceph-mon[106729]: pgmap v547: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 4694/254451 objects degraded (1.845%); 5484/254451 objects misplaced (2.155%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:18:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:32 smithi142 ceph-mon[103614]: pgmap v547: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s; 4694/254451 objects degraded (1.845%); 5484/254451 objects misplaced (2.155%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:18:33.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:33.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 91008 90992 103.387 67.1875 0.0174749 0.00966808 2024-08-17T17:18:33.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:33.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:33.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:33.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:33.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:33.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:33.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:33.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:33.671 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:33.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:33.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:33.816 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:33.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:33.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:34.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:33 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/266064 objects degraded (1.764%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:34.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:34.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:33 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/266064 objects degraded (1.764%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:34.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:33 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/266064 objects degraded (1.764%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:34.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 92127 92111 102.789 69.9375 0.00882186 0.00971471 2024-08-17T17:18:34.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:34.229 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:34.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:34.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:34.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:34.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:34.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:34.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:34.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:34.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:34.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:34.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:34.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:34.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:35.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:35.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:35.053 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:34 smithi142 ceph-mon[103614]: pgmap v548: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 4694/266064 objects degraded (1.764%); 5484/266064 objects misplaced (2.061%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:18:35.053 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:34 smithi142 ceph-mon[106729]: pgmap v548: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 4694/266064 objects degraded (1.764%); 5484/266064 objects misplaced (2.061%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:18:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:34 smithi152 ceph-mon[93456]: pgmap v548: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 4694/266064 objects degraded (1.764%); 5484/266064 objects misplaced (2.061%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:18:35.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:35.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:35.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 93238 93222 102.204 69.4375 0.00503709 0.00977729 2024-08-17T17:18:35.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:35.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:35.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:35.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:35.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:35.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:35.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:35.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:35.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:35.745 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:35.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:35.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:35.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:35.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:36.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:36.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:36.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:36.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 94386 94370 101.679 71.75 0.00708876 0.00982548 2024-08-17T17:18:36.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:36.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:36.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:36.394 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:36.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:36.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:36.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:36.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:36.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:36.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:36.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:36.932 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:37.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:37.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:36 smithi152 ceph-mon[93456]: pgmap v549: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 4694/269397 objects degraded (1.742%); 5484/269397 objects misplaced (2.036%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:18:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:37.099 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:36 smithi142 ceph-mon[106729]: pgmap v549: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 4694/269397 objects degraded (1.742%); 5484/269397 objects misplaced (2.036%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:18:37.099 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:36 smithi142 ceph-mon[103614]: pgmap v549: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s; 4694/269397 objects degraded (1.742%); 5484/269397 objects misplaced (2.036%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:18:37.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 95411 95395 101.041 64.0625 0.00898859 0.00989203 2024-08-17T17:18:37.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:37.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:37.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:37.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:37.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:37.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:37.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:37.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:37.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:37.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:37.744 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:37.846 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:37.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:37.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:37.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:38.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:38.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:38.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:38.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:18:38.173813+0000 min lat: 0.00105076 max lat: 2.38813 avg lat: 0.00992711 2024-08-17T17:18:38.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:18:38.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 96687 96671 100.686 79.75 0.0139486 0.00992711 2024-08-17T17:18:38.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:38.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:38.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:38.439 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:38.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:38.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:38.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:38.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:38.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:38.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:38.858 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:38.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:38.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:39.007 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:39.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:38 smithi152 ceph-mon[93456]: pgmap v550: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 4694/279213 objects degraded (1.681%); 4983/279213 objects misplaced (1.785%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:18:39.092 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:38 smithi142 ceph-mon[106729]: pgmap v550: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 4694/279213 objects degraded (1.681%); 4983/279213 objects misplaced (1.785%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:18:39.092 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:38 smithi142 ceph-mon[103614]: pgmap v550: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 4694/279213 objects degraded (1.681%); 4983/279213 objects misplaced (1.785%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:18:39.109 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:39.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:39.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 97987 97971 100.368 81.25 0.020812 0.00995832 2024-08-17T17:18:39.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:39.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:39.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:39.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:39.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:39.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:39.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:39.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:39.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:39.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:39.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:39.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:39.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:40.014 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:40.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:40.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:40.173 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 99283 99267 100.055 81 0.011493 0.00998971 2024-08-17T17:18:40.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.285 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:40.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:40.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:40.404 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:40.404 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:40.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:40.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:40.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:40.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:18:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:18:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:18:40.585 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:18:40.585 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:40.630 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:18:40.631 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:40.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:40.664 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:18:40.664 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:40.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:40.701 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:18:40.701 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:40.730 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:18:40.730 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:40.760 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:18:40.760 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:40.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:40.805 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:18:40.805 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:40.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:40.849 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:18:40.849 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:40.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:40.977 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[106729]: pgmap v551: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 4694/285987 objects degraded (1.641%); 4983/285987 objects misplaced (1.742%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/285987 objects degraded (1.641%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1592336913' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[103614]: pgmap v551: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 4694/285987 objects degraded (1.641%); 4983/285987 objects misplaced (1.742%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/285987 objects degraded (1.641%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:40.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1592336913' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:40.981 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:40 smithi152 ceph-mon[93456]: pgmap v551: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 4694/285987 objects degraded (1.641%); 4983/285987 objects misplaced (1.742%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:18:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/285987 objects degraded (1.641%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:41.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1592336913' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:41.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:41.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:41.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 100532 100516 99.706 78.0625 0.0117274 0.0100243 2024-08-17T17:18:41.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.266 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:41.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:41.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:41.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:41.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:41.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:41.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:41.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:41.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:41.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:41.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:41.928 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:41.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:42.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:42.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:18:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:18:41] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-08-17T17:18:42.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 101906 101890 99.4898 85.875 0.00398913 0.0100465 2024-08-17T17:18:42.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:42.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:42.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:42.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:42.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:42.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:42.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:42.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:42.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:42.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:42 smithi142 ceph-mon[106729]: pgmap v552: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 4694/288777 objects degraded (1.625%); 4323/288777 objects misplaced (1.497%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:18:42.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:42.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:42.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:42 smithi142 ceph-mon[103614]: pgmap v552: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 4694/288777 objects degraded (1.625%); 4323/288777 objects misplaced (1.497%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:18:42.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:42.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:42.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:42.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:42 smithi152 ceph-mon[93456]: pgmap v552: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 4694/288777 objects degraded (1.625%); 4323/288777 objects misplaced (1.497%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:18:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:18:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:42.914 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:42.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:43.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:43.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:43.174 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 103281 103265 99.2802 85.9375 0.0125175 0.0100678 2024-08-17T17:18:43.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:43.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:43.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:43.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:43.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:43.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:43.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:43.609 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:43.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:43.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:43.752 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:43.782 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:43.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:43.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:44.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:44.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:44.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:44.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:44.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 104452 104436 98.8843 73.1875 0.0169288 0.0101083 2024-08-17T17:18:44.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:44.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:44.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:44.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:44.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:44.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:44.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:44.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:44 smithi142 ceph-mon[106729]: pgmap v553: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 4694/301494 objects degraded (1.557%); 4323/301494 objects misplaced (1.434%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:18:44.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:44 smithi142 ceph-mon[103614]: pgmap v553: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 4694/301494 objects degraded (1.557%); 4323/301494 objects misplaced (1.434%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:18:44.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:18:44.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:44.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:44 smithi152 ceph-mon[93456]: pgmap v553: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 4694/301494 objects degraded (1.557%); 4323/301494 objects misplaced (1.434%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:18:44.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:44.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:44.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:44.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:45.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:45.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:45.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:45.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 105583 105567 98.4634 70.6875 0.00633076 0.0101515 2024-08-17T17:18:45.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:45.286 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:45.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:45.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:45.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:45.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:45.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:45.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:45.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/305301 objects degraded (1.537%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:45.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:45.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/305301 objects degraded (1.537%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/305301 objects degraded (1.537%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:45.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:45.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:46.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:46.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:46.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 106893 106877 98.2193 81.875 0.0125006 0.0101769 2024-08-17T17:18:46.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:46.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:46.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:46.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:46.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:46.544 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:46.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:46.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:46.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:46.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:46 smithi152 ceph-mon[93456]: pgmap v554: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 4694/305301 objects degraded (1.537%); 4323/305301 objects misplaced (1.416%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:18:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:46 smithi142 ceph-mon[103614]: pgmap v554: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 4694/305301 objects degraded (1.537%); 4323/305301 objects misplaced (1.416%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:18:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:46 smithi142 ceph-mon[106729]: pgmap v554: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 4694/305301 objects degraded (1.537%); 4323/305301 objects misplaced (1.416%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:18:46.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:46.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:47.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:47.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:47.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:47.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 108113 108097 97.9008 76.25 0.00252588 0.0102088 2024-08-17T17:18:47.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:47.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:47.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:47.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:47.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:47.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:47.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:47.477 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:47.578 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:47.617 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:47.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:47.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:47.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:47.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:47.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:47.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:47.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:48.028 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:48.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:48.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:48.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:48.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 109335 109319 97.5932 76.375 0.00515447 0.010242 2024-08-17T17:18:48.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:48.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:48.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:48.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:48.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:48.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:48.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:48.726 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:48 smithi142 ceph-mon[103614]: pgmap v555: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 4694/316524 objects degraded (1.483%); 3702/316524 objects misplaced (1.170%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:18:48.727 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:48 smithi142 ceph-mon[106729]: pgmap v555: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 4694/316524 objects degraded (1.483%); 3702/316524 objects misplaced (1.170%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:18:48.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:48 smithi152 ceph-mon[93456]: pgmap v555: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 4694/316524 objects degraded (1.483%); 3702/316524 objects misplaced (1.170%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:18:48.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:48.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:48.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:49.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:49.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:49.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:49.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 110560 110544 97.2969 76.5625 0.024754 0.0102726 2024-08-17T17:18:49.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:49.282 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:49.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:49.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:49.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:18:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:18:49.478 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:18:49.478 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:18:49.478 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:18:49.478 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:18:49.497 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:18:49.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:49.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:49.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:49.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:49.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:49.810 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3909114164' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:49.810 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:49.810 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3909114164' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:49.810 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3909114164' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:49.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:49.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:49.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:50.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:50.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:50.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 111813 111797 97.0332 78.3125 0.0174518 0.0103011 2024-08-17T17:18:50.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:50.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:50.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:50.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:50.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:50.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:50.491 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:18:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:18:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:18:50.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:50.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:50.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:50.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:50.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:50.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:50 smithi152 ceph-mon[93456]: pgmap v556: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 4694/323988 objects degraded (1.449%); 3702/323988 objects misplaced (1.143%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:18:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:50 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:50 smithi152 ceph-mon[93456]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T17:18:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/323988 objects degraded (1.449%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[106729]: pgmap v556: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 4694/323988 objects degraded (1.449%); 3702/323988 objects misplaced (1.143%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:18:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[106729]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T17:18:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/323988 objects degraded (1.449%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[103614]: pgmap v556: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 4694/323988 objects degraded (1.449%); 3702/323988 objects misplaced (1.143%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:18:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[103614]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T17:18:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/323988 objects degraded (1.449%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:50.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:50.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:51.043 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:51.078 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:51.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:51.093 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:18:51.093 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:18:51.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-17T17:18:51.175 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 113104 113088 96.8092 80.6875 0.00524835 0.0103244 2024-08-17T17:18:51.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:51.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:51.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:51.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:51.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:51.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:51.592 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:51.593 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:51.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:51.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:51.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:51.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:51.810 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:51.893 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:51.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:51.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:52.039 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:52.064 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:18:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:18:51] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-08-17T17:18:52.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:52.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 114275 114259 96.4899 73.1875 0.0101462 0.0103593 2024-08-17T17:18:52.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:52.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:52.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:52.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:52.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:52.384 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:52.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:18:52.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:52.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:52.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:18:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:52.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:52 smithi142 ceph-mon[106729]: pgmap v558: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 4694/326892 objects degraded (1.436%); 3005/326892 objects misplaced (0.919%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:52 smithi142 ceph-mon[103614]: pgmap v558: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 4694/326892 objects degraded (1.436%); 3005/326892 objects misplaced (0.919%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:52.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:52.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:52.761 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:52 smithi152 ceph-mon[93456]: pgmap v558: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 4694/326892 objects degraded (1.436%); 3005/326892 objects misplaced (0.919%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:52.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.912 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:52.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:53.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:53.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:53.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:53.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 115353 115337 96.1016 67.375 0.00580058 0.0104014 2024-08-17T17:18:53.186 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:53.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:53.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:53.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:53.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:53.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:53.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:53.688 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:53.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:53.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:53.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:53.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:53.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:54.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:54.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:54.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:54.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:54.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 116545 116529 95.8173 74.5 0.0086243 0.0104321 2024-08-17T17:18:54.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.296 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:54.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:54.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:54.430 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:54.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:54.554 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:54.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:54.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.682 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:54.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:54.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:54 smithi152 ceph-mon[93456]: pgmap v559: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 4694/339030 objects degraded (1.385%); 3005/339030 objects misplaced (0.886%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:54.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:54 smithi142 ceph-mon[106729]: pgmap v559: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 4694/339030 objects degraded (1.385%); 3005/339030 objects misplaced (0.886%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:54 smithi142 ceph-mon[103614]: pgmap v559: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 4694/339030 objects degraded (1.385%); 3005/339030 objects misplaced (0.886%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:54.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:54.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:55.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:55.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:55.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 117576 117560 95.4097 64.4375 0.0138082 0.0104757 2024-08-17T17:18:55.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:55.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:55.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:55.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:55.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:55.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:55.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:55.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:55.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:55.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:55.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/339030 objects degraded (1.385%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/339030 objects degraded (1.385%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/339030 objects degraded (1.385%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:55.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:55.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:56.035 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:56.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:56.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:56.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 118770 118754 95.1431 74.625 0.0039508 0.0105057 2024-08-17T17:18:56.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:56.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:56.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:56.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:56.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:56.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:56.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:56.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:56.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:56 smithi142 ceph-mon[106729]: pgmap v560: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 4694/342303 objects degraded (1.371%); 3005/342303 objects misplaced (0.878%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:56 smithi142 ceph-mon[103614]: pgmap v560: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 4694/342303 objects degraded (1.371%); 3005/342303 objects misplaced (0.878%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:56.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:56.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:56.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:56 smithi152 ceph-mon[93456]: pgmap v560: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 4694/342303 objects degraded (1.371%); 3005/342303 objects misplaced (0.878%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:18:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:56.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:56.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:56.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:57.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:57.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:57.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:57.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 119972 119956 94.8897 75.125 0.00932989 0.0105341 2024-08-17T17:18:57.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:57.298 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:57.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:57.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:57.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:57.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:57.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:57.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:57.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:57.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:57.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:57.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:57.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:57.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:58.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:58.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:58.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:18:58.176596+0000 min lat: 0.00105076 max lat: 2.38813 avg lat: 0.010558 2024-08-17T17:18:58.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:18:58.176 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 121216 121200 94.6754 77.75 0.00785893 0.010558 2024-08-17T17:18:58.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:58.265 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:58.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:58.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:58.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:58.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:58.642 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.696 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:58.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:58.790 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:18:58 smithi142 ceph-mon[103614]: pgmap v561: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 4694/352683 objects degraded (1.331%); 2208/352683 objects misplaced (0.626%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:18:58.791 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:18:58 smithi142 ceph-mon[106729]: pgmap v561: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 4694/352683 objects degraded (1.331%); 2208/352683 objects misplaced (0.626%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:18:58.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:18:58 smithi152 ceph-mon[93456]: pgmap v561: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 4694/352683 objects degraded (1.331%); 2208/352683 objects misplaced (0.626%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:18:58.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:58.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:58.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:59.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:59.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:59.177 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 122416 122400 94.4321 75 0.0098771 0.0105842 2024-08-17T17:18:59.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:59.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:59.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:59.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:59.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:59.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:59.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:59.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:18:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:59.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:59.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:59.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:59.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:18:59.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:00.044 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:19:00.044 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:00.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:00.077 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:19:00.078 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:00.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:00.112 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:19:00.112 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:00.177 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 123471 123455 94.0845 65.9375 0.021495 0.010624 2024-08-17T17:19:00.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:00.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:00.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:00.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:00.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:00.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:00.545 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:19:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:19:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:19:00.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:00.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:00.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:00.739 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:00 smithi142 ceph-mon[106729]: pgmap v562: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 4694/359742 objects degraded (1.305%); 2208/359742 objects misplaced (0.614%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:19:00.739 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/146881605' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:00.739 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/359742 objects degraded (1.305%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:00.739 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:00 smithi142 ceph-mon[103614]: pgmap v562: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 4694/359742 objects degraded (1.305%); 2208/359742 objects misplaced (0.614%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:19:00.739 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/146881605' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:00.739 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/359742 objects degraded (1.305%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:00.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:00.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:00 smithi152 ceph-mon[93456]: pgmap v562: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 4694/359742 objects degraded (1.305%); 2208/359742 objects misplaced (0.614%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:19:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/146881605' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/359742 objects degraded (1.305%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:00.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:00.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:00.975 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:01.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:01.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:01.177 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 124578 124562 93.7845 69.1875 0.00388288 0.0106577 2024-08-17T17:19:01.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:01.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:01.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:01.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:01.441 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:01.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:01.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:01.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:01.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:01.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:01.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:01.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:01.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:01.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:01.966 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:19:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:19:01] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.51.0" 2024-08-17T17:19:02.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:02.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:02.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:02.181 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 125772 125756 93.5517 74.625 0.0200265 0.0106844 2024-08-17T17:19:02.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:02.302 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:02.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:02.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:02.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:02.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:02.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:02.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:02.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:02.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:02 smithi152 ceph-mon[93456]: pgmap v563: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 4694/362571 objects degraded (1.295%); 1265/362571 objects misplaced (0.349%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:19:02.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:02 smithi142 ceph-mon[103614]: pgmap v563: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 4694/362571 objects degraded (1.295%); 1265/362571 objects misplaced (0.349%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:19:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:02 smithi142 ceph-mon[106729]: pgmap v563: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 4694/362571 objects degraded (1.295%); 1265/362571 objects misplaced (0.349%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:19:02.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:02.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:03.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:03.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:03.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:03.181 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 126983 126967 93.3414 75.6875 0.0109184 0.0107087 2024-08-17T17:19:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.188 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:03.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:03.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:03.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:03.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:03.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:03.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:19:03.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:03.640 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:03.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:03.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:03.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:03.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:03.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:03.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:04.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:04.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:04.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:04.181 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 128124 128108 93.0852 71.3125 0.00864314 0.0107384 2024-08-17T17:19:04.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:04.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:04.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:04.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:04.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:04.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:04.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:04.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:04.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:19:04.641 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:19:04.642 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:19:04.642 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:19:04.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:04.761 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:04 smithi152 ceph-mon[93456]: pgmap v564: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 4694/373908 objects degraded (1.255%); 1265/373908 objects misplaced (0.338%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:19:04.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:04 smithi142 ceph-mon[103614]: pgmap v564: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 4694/373908 objects degraded (1.255%); 1265/373908 objects misplaced (0.338%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:19:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:04 smithi142 ceph-mon[106729]: pgmap v564: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 4694/373908 objects degraded (1.255%); 1265/373908 objects misplaced (0.338%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:19:04.901 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:05.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:05.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:05.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:05.181 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 129304 129288 92.8629 73.75 0.0145301 0.0107639 2024-08-17T17:19:05.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:05.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:05.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:05.331 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:05.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:05.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:05.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:05.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:05.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:05.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:05.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:05.756 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/373908 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:05.756 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/373908 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/373908 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:05.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:05.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:05.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:06.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:06.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:06.162 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:06.181 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 130484 130468 92.6457 73.75 0.00482505 0.0107891 2024-08-17T17:19:06.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:19:06.289 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:19:06.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-17T17:19:06.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:06.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:06.336 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:06.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:06.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:06.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:06.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:06.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:06.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:06 smithi152 ceph-mon[93456]: pgmap v565: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 4694/377151 objects degraded (1.245%); 1265/377151 objects misplaced (0.335%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:19:06.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:06.860 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:06 smithi142 ceph-mon[106729]: pgmap v565: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 4694/377151 objects degraded (1.245%); 1265/377151 objects misplaced (0.335%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:19:06.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:06 smithi142 ceph-mon[103614]: pgmap v565: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 4694/377151 objects degraded (1.245%); 1265/377151 objects misplaced (0.335%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:19:06.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:06.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:07.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:07.126 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:07.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:07.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 131435 131419 92.2725 59.4375 0.00322627 0.0107991 2024-08-17T17:19:07.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:07.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:07.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:07.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:07.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:07.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:07.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:07.606 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:19:07.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:07.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:07.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:07.794 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:07 smithi142 ceph-mon[106729]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T17:19:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/311548980' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:07.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:07 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:07.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:07 smithi142 ceph-mon[103614]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T17:19:07.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/311548980' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:07.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:07 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:07.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:07.939 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:08.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:07 smithi152 ceph-mon[93456]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T17:19:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/311548980' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:07 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:08.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:08.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0117 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 132118 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 91.7367 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 40.2527 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 226.938 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1467 2024-08-17T17:19:08.182 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 644.053 2024-08-17T17:19:08.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 3631 2024-08-17T17:19:08.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 0 2024-08-17T17:19:08.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0108962 2024-08-17T17:19:08.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0362043 2024-08-17T17:19:08.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 2.38813 2024-08-17T17:19:08.183 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00105076 2024-08-17T17:19:08.186 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:19:08.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:08.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:08.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:08.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:08.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:08.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:08.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:08.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:08.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:08.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:08.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[103614]: pgmap v567: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 4694/387558 objects degraded (1.211%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:19:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:19:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[103614]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T17:19:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[106729]: pgmap v567: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 4694/387558 objects degraded (1.211%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:19:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:19:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:08 smithi142 ceph-mon[106729]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T17:19:08.901 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:09.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:09.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:08 smithi152 ceph-mon[93456]: pgmap v567: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 7.9 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 4694/387558 objects degraded (1.211%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:19:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:08 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:08 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:19:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:08 smithi152 ceph-mon[93456]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T17:19:09.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.173 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:09.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:09.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:09.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:09.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:09.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:09.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:09.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:09.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:09.688 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:09.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:09.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:09.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:09.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:09 smithi142 ceph-mon[106729]: 9.13 scrub starts 2024-08-17T17:19:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:09 smithi142 ceph-mon[103614]: 9.13 scrub starts 2024-08-17T17:19:09.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:09.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:09 smithi152 ceph-mon[93456]: 9.13 scrub starts 2024-08-17T17:19:10.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:10.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:10.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:10.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:10.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.411 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:10.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:10.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:10.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:19:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:19:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:19:10.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:10.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:10.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:10.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:10.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:10 smithi142 ceph-mon[103614]: 9.13 scrub ok 2024-08-17T17:19:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:10 smithi142 ceph-mon[103614]: pgmap v569: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 8.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 81 MiB/s wr, 1.31k op/s; 4694/393462 objects degraded (1.193%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:19:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4694/393462 objects degraded (1.193%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:10 smithi142 ceph-mon[106729]: 9.13 scrub ok 2024-08-17T17:19:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:10 smithi142 ceph-mon[106729]: pgmap v569: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 8.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 81 MiB/s wr, 1.31k op/s; 4694/393462 objects degraded (1.193%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:19:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4694/393462 objects degraded (1.193%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:10.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:10.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:11.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:10 smithi152 ceph-mon[93456]: 9.13 scrub ok 2024-08-17T17:19:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:10 smithi152 ceph-mon[93456]: pgmap v569: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 8.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 81 MiB/s wr, 1.31k op/s; 4694/393462 objects degraded (1.193%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:19:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4694/393462 objects degraded (1.193%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:11.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:11.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:11.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:11.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:11.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:11.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:11.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:11.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:11.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.658 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:11.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:11.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.796 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:11.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:11.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:11.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:12.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:12.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:19:12.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:12.086 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:12.086 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:19:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:19:11] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-08-17T17:19:12.086 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:12.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:12.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:12.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:12.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:12.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:12.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:12.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:12.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:12.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:12.578 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:12.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:12.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:12.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:12.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:12.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:12.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:12.957 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:12 smithi142 ceph-mon[103614]: pgmap v570: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 97 active+clean; 8.0 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 54 MiB/s wr, 917 op/s; 4312/393183 objects misplaced (1.097%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:19:12.957 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:12 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4694/393462 objects degraded (1.193%), 1 pg degraded) 2024-08-17T17:19:12.959 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:12 smithi142 ceph-mon[106729]: pgmap v570: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 97 active+clean; 8.0 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 54 MiB/s wr, 917 op/s; 4312/393183 objects misplaced (1.097%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:19:12.959 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:12 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4694/393462 objects degraded (1.193%), 1 pg degraded) 2024-08-17T17:19:12.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:13.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:13.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:12 smithi152 ceph-mon[93456]: pgmap v570: 99 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 97 active+clean; 8.0 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 54 MiB/s wr, 917 op/s; 4312/393183 objects misplaced (1.097%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:19:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:12 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4694/393462 objects degraded (1.193%), 1 pg degraded) 2024-08-17T17:19:13.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:13.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:13.217 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:19:13.217 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:19:13.217 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:19:13.217 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:19:13.217 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:19:13.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:13.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:13.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:13.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:13.535 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:13.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:13.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:13.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:13.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:13.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:13.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:19:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:19:13.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:13.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-08-17T17:19:14.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:14.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:14.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:14.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:14.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:14.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:14.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:14.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:14.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:14.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:14.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:14.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:14.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:14.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:14.719 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:14.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:14.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:14 smithi142 ceph-mon[106729]: pgmap v571: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 716 op/s; 4312/385809 objects misplaced (1.118%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:19:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:19:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:14 smithi142 ceph-mon[106729]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T17:19:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:14 smithi142 ceph-mon[103614]: pgmap v571: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 716 op/s; 4312/385809 objects misplaced (1.118%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:19:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:19:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:14 smithi142 ceph-mon[103614]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T17:19:14.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:14.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:15.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:15.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:14 smithi152 ceph-mon[93456]: pgmap v571: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 716 op/s; 4312/385809 objects misplaced (1.118%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:19:15.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "20"}]': finished 2024-08-17T17:19:15.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:14 smithi152 ceph-mon[93456]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T17:19:15.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:15.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:15.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:15.214 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:15.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:15.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:15.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:15.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:15.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:15.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:15.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:15.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:15.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:15.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:15.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:15.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:15.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:15.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:15.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:16.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[106729]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T17:19:16.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:19:16.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:19:16.026 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[106729]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T17:19:16.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[103614]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T17:19:16.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:19:16.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:19:16.027 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:15 smithi142 ceph-mon[103614]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T17:19:16.039 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:16.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:16.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:19:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:15 smithi152 ceph-mon[93456]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T17:19:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:19:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:19:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:15 smithi152 ceph-mon[93456]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T17:19:16.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:16.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:16.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:16.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:16.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:16.368 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:16.470 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:16.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:16.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:16.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:16.636 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:16.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:16.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:16.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:16.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:16.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:16.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:17.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:17.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:16 smithi152 ceph-mon[93456]: pgmap v574: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 131 B/s rd, 0 B/s wr, 281 op/s; 4312/382416 objects misplaced (1.128%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:19:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:16 smithi142 ceph-mon[106729]: pgmap v574: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 131 B/s rd, 0 B/s wr, 281 op/s; 4312/382416 objects misplaced (1.128%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:19:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:16 smithi142 ceph-mon[103614]: pgmap v574: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 131 B/s rd, 0 B/s wr, 281 op/s; 4312/382416 objects misplaced (1.128%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:19:17.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:17.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:17.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:17.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:17.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:17.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:17.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:17.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:17.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:17.575 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:17.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:17.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:17.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:17.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:17.931 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:17.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:17.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:17.985 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:19:17.985 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:17 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:19:18.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:17 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-08-17T17:19:18.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:18.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:18.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:18.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:18.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:18.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:18.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:18.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:18.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:18.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:18.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:18.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:18.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:18.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:18.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:18.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:18.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:18.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:19:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:19:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:19:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:19:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:19:19.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:19.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:19.030 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:19:19.030 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:19:19.030 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:19.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[106729]: pgmap v576: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.6 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 464 op/s; 3658/375858 objects misplaced (0.973%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:19:19.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:19:19.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[106729]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T17:19:19.031 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2877308517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:19.031 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[103614]: pgmap v576: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.6 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 464 op/s; 3658/375858 objects misplaced (0.973%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:19:19.031 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:19:19.031 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[103614]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T17:19:19.031 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2877308517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:19.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:19.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:18 smithi152 ceph-mon[93456]: pgmap v576: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 7.6 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 464 op/s; 3658/375858 objects misplaced (0.973%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:19:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:18 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "22"}]': finished 2024-08-17T17:19:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:18 smithi152 ceph-mon[93456]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T17:19:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2877308517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:19.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:19.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:19.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:19.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:19.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:19.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:19.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:19.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:19.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:19.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:19.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:19.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:19.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:19.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:19.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:19.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:20.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:19 smithi152 ceph-mon[93456]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T17:19:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:19 smithi152 ceph-mon[93456]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T17:19:20.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:20.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:20.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:19 smithi142 ceph-mon[106729]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T17:19:20.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:19 smithi142 ceph-mon[106729]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T17:19:20.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:19 smithi142 ceph-mon[103614]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T17:19:20.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:19 smithi142 ceph-mon[103614]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T17:19:20.147 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:20.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:20.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:20.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:20.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:20.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:20.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:20.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:19:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:19:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:19:20.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:20.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:20.760 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:20.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:20.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[103614]: pgmap v579: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.6 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 8340/372231 objects degraded (2.241%); 3658/372231 objects misplaced (0.983%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:19:20.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 8340/372231 objects degraded (2.241%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:20.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:20.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:20.803 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[106729]: pgmap v579: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.6 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 8340/372231 objects degraded (2.241%); 3658/372231 objects misplaced (0.983%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:19:20.804 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 8340/372231 objects degraded (2.241%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:20.804 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:20.804 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:20.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:20.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:20.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:21.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:20 smithi152 ceph-mon[93456]: pgmap v579: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 7.6 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 8340/372231 objects degraded (2.241%); 3658/372231 objects misplaced (0.983%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:19:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:20 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 8340/372231 objects degraded (2.241%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:21.081 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:21.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:21.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:21.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:21.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:21.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:21.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:21.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:21.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:21.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:21.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:21.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:21.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:19:21.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:21.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:19:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:19:21] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-08-17T17:19:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.913 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:21.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:21.941 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:22.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:22.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:22.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:22.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:22.282 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:22.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:22.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:22.461 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:22.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:22.597 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:22.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:22.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:22.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:22.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[106729]: pgmap v581: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 7.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 8340/371103 objects degraded (2.247%); 2869/371103 objects misplaced (0.773%); 8.5 MiB/s, 135 objects/s recovering 2024-08-17T17:19:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:19:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:19:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[103614]: pgmap v581: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 7.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 8340/371103 objects degraded (2.247%); 2869/371103 objects misplaced (0.773%); 8.5 MiB/s, 135 objects/s recovering 2024-08-17T17:19:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:19:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:19:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:22.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:22.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:23.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:22 smithi152 ceph-mon[93456]: pgmap v581: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 7.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 8340/371103 objects degraded (2.247%); 2869/371103 objects misplaced (0.773%); 8.5 MiB/s, 135 objects/s recovering 2024-08-17T17:19:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:19:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:19:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:23.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:23.164 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:23.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:23.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:23.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:23.395 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:19:23.395 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:19:23.395 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:19:23.395 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:19:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:19:23.397 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:19:23.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:23.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:23.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:23.577 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:23.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:23.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:23.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:23.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:23.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:23.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:24.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:24.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:24.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:24.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:24.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:24.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:24.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:24.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:24.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:24.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:24.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:24.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:24.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:24.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:24.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:24.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:24.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:24 smithi142 ceph-mon[103614]: pgmap v582: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 16222/364059 objects degraded (4.456%); 2869/364059 objects misplaced (0.788%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:19:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:24 smithi142 ceph-mon[106729]: pgmap v582: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 16222/364059 objects degraded (4.456%); 2869/364059 objects misplaced (0.788%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:19:24.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:24.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:24.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:25.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:24 smithi152 ceph-mon[93456]: pgmap v582: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 16222/364059 objects degraded (4.456%); 2869/364059 objects misplaced (0.788%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:19:25.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:25.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:25.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:25.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:25.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:25.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:25.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:25.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:25.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:25.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:25.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:25.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:25.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:25.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:25.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:25.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:25.881 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:25.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:25.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:26.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:26.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16222/364059 objects degraded (4.456%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16222/364059 objects degraded (4.456%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:26.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16222/364059 objects degraded (4.456%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:26.146 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:26.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:26.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:26.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:26.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:26.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:26.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:26.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:26.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:26.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:26.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:26 smithi142 ceph-mon[106729]: pgmap v583: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 16222/361926 objects degraded (4.482%); 2869/361926 objects misplaced (0.793%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:19:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:26 smithi142 ceph-mon[103614]: pgmap v583: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 16222/361926 objects degraded (4.482%); 2869/361926 objects misplaced (0.793%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:19:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:26.917 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:26.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:26.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:19:26.979 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:19:26.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-08-17T17:19:27.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:26 smithi152 ceph-mon[93456]: pgmap v583: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 16222/361926 objects degraded (4.482%); 2869/361926 objects misplaced (0.793%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:19:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:27.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:27.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:27.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:27.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:27.368 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:27.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:27.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:27.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:27.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:27.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:27.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:27.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:27.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:28.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:28.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:28.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:28.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:28.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:28.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:28.317 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:28.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:28.461 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:28.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:28.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:28.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:28.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:28.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:28.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:28.792 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:19:28.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:28.883 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:28.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:28.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:29.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:29.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:28 smithi152 ceph-mon[93456]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 16222/353028 objects degraded (4.595%); 2372/353028 objects misplaced (0.672%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:19:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/350639961' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:29.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:28 smithi142 ceph-mon[106729]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 16222/353028 objects degraded (4.595%); 2372/353028 objects misplaced (0.672%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:19:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/350639961' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:28 smithi142 ceph-mon[103614]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 16222/353028 objects degraded (4.595%); 2372/353028 objects misplaced (0.672%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:19:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/350639961' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:29.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:29.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:29.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:29.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:29.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:29.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:29.554 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:29.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:29.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:29.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:29.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:29.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:29.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:29.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:29.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:29.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:29 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:19:29.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/350639961' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:29.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:29 smithi142 ceph-mon[103614]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T17:19:29.963 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:29 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:19:29.963 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/350639961' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:29.963 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:29 smithi142 ceph-mon[106729]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T17:19:30.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:30.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:29 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:19:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/350639961' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:29 smithi152 ceph-mon[93456]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T17:19:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:30.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:30.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:30.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:30.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:30.288 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:19:30.288 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:19:30.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:30.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:30.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:30.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:30.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:30.526 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:19:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:19:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:19:30.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:30.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:30.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:30.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:30.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:30.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:30.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:30.926 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:19:30.927 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:30 smithi142 ceph-mon[106729]: pgmap v586: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 15582/346029 objects degraded (4.503%); 2372/346029 objects misplaced (0.685%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:19:30.927 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15582/346029 objects degraded (4.503%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:30.927 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:30.927 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:30 smithi142 ceph-mon[103614]: pgmap v586: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 15582/346029 objects degraded (4.503%); 2372/346029 objects misplaced (0.685%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:19:30.927 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15582/346029 objects degraded (4.503%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:30.951 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:19:30.951 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:30.975 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:19:30.975 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:30.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:31.000 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:19:31.000 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:31.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:31.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:19:31.028 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:31.060 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:19:31.061 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:31.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:30 smithi152 ceph-mon[93456]: pgmap v586: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 15582/346029 objects degraded (4.503%); 2372/346029 objects misplaced (0.685%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:19:31.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15582/346029 objects degraded (4.503%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:31.090 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:19:31.091 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:31.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:31.130 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:19:31.130 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:31.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:31.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:31.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:19:31.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:31.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:31.393 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.430 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:31.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:31.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:31.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:31.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:31.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:31.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:31.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:31.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:31.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:31.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3709122321' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:32.079 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3709122321' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:32.079 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:19:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:19:31] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-08-17T17:19:32.080 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3709122321' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:32.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:32.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:32.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.276 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:32.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:32.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.414 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:32.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:32.507 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:32.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:32.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:32.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:32.584 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:32.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:32.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:32.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:32.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:32.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:32.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:33.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:32 smithi152 ceph-mon[93456]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 15582/344316 objects degraded (4.525%); 1754/344316 objects misplaced (0.509%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:19:33.095 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:32 smithi142 ceph-mon[106729]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 15582/344316 objects degraded (4.525%); 1754/344316 objects misplaced (0.509%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:19:33.096 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:32 smithi142 ceph-mon[103614]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 15582/344316 objects degraded (4.525%); 1754/344316 objects misplaced (0.509%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:19:33.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:33.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:33.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:33.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:33.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:33.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:33.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:33.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:33.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:33.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:33.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:33.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:33.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:33.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:33.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:33.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:34.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.059 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:34.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:34.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:19:34.160 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:34.193 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:34.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:34.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:34.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:34.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:34.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:19:34.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:34.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:34.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:34.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:34.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:34.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:34.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:34.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:34 smithi142 ceph-mon[106729]: pgmap v588: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 15034/335304 objects degraded (4.484%); 1754/335304 objects misplaced (0.523%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:19:34.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:34 smithi142 ceph-mon[103614]: pgmap v588: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 15034/335304 objects degraded (4.484%); 1754/335304 objects misplaced (0.523%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:19:34.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:34.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:34 smithi152 ceph-mon[93456]: pgmap v588: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 15034/335304 objects degraded (4.484%); 1754/335304 objects misplaced (0.523%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:19:35.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:35.125 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:35.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:35.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:35.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:35.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:35.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:35.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:35.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:35.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:35.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:35.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:35.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:35.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:35.939 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:36.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15034/335304 objects degraded (4.484%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:36.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15034/335304 objects degraded (4.484%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15034/335304 objects degraded (4.484%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:36.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:36.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:36.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:36.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:36.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:36.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:36.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:36.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:36.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:36.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:36.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:36.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:36.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:36.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:36.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:36.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:36.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:37.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:37.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:36 smithi152 ceph-mon[93456]: pgmap v589: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 15034/332799 objects degraded (4.517%); 1754/332799 objects misplaced (0.527%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:19:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:36 smithi142 ceph-mon[106729]: pgmap v589: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 15034/332799 objects degraded (4.517%); 1754/332799 objects misplaced (0.527%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:19:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:36 smithi142 ceph-mon[103614]: pgmap v589: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 15034/332799 objects degraded (4.517%); 1754/332799 objects misplaced (0.527%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:19:37.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:37.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:37.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:37.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:37.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:37.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:37.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:37.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:37.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:37.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:37.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:37.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:37.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:37.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:37.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:37.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:37.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:38.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:38.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:38.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:38.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:38.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:38.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:38.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:38.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:38.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:38.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:38.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:38.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:38.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.801 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:38.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:38.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:38.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:39.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:39.047 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:38 smithi142 ceph-mon[106729]: pgmap v590: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 15034/324420 objects degraded (4.634%); 1234/324420 objects misplaced (0.380%); 7.3 MiB/s, 116 objects/s recovering 2024-08-17T17:19:39.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:38 smithi142 ceph-mon[103614]: pgmap v590: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 15034/324420 objects degraded (4.634%); 1234/324420 objects misplaced (0.380%); 7.3 MiB/s, 116 objects/s recovering 2024-08-17T17:19:39.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:39.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:39.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:39.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:39.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:38 smithi152 ceph-mon[93456]: pgmap v590: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 15034/324420 objects degraded (4.634%); 1234/324420 objects misplaced (0.380%); 7.3 MiB/s, 116 objects/s recovering 2024-08-17T17:19:39.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:39.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:39.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:39.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:39.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:39.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:39.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:39.769 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:39.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:39.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:40.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:40.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:40.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:40.133 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:160999 9.14 e179/160 9:28510209:::benchmark_data_smithi142_157452_object28878:head v 179'7032, mlcod=179'7029)", 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.131188+0000", 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.00055746099999999998, 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058330999999999997, 2024-08-17T17:19:40.134 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131188+0000", 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.135 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131188+0000", 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131188+0000", 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5300000000000004e-07 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.136 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131194+0000", 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5010000000000004e-06 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131205+0000", 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0706e-05 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131211+0000", 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2689999999999996e-06 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.137 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131258+0000", 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.672e-05 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131275+0000", 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7793000000000001e-05 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.138 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:160991 9.c 9:3a2bf82c:::benchmark_data_smithi142_157452_object28870:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.113238+0000", 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.018507599, 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018589985999999999, 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:40.139 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 160991 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113238+0000", 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113238+0000", 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113239+0000", 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.477e-06 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113244+0000", 2024-08-17T17:19:40.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8339999999999998e-06 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113257+0000", 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2836e-05 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113265+0000", 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9410000000000007e-06 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.117070+0000", 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038048209999999999 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.117200+0000", 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013034400000000001 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.143 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.117327+0000", 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000127071 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.129822+0000", 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012494658000000001 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:40.144 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.130875+0000", 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010535690000000001 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:160990 9.f e179/154 9:f1951231:::benchmark_data_smithi142_157452_object28869:head v 179'9991, mlcod=179'9989)", 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.113585+0000", 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.018159903000000002, 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018332284000000001, 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.145 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113585+0000", 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113585+0000", 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.146 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113586+0000", 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9500000000000003e-07 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113591+0000", 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8029999999999996e-06 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113599+0000", 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1110000000000008e-06 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113604+0000", 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2120000000000002e-06 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113661+0000", 2024-08-17T17:19:40.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6765000000000003e-05 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.113680+0000", 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8967999999999999e-05 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:160997 9.1 9:8614f601:::benchmark_data_smithi142_157452_object28876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-17T17:19:40.149 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.118370+0000", 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.013375323, 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013596702, 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 160997 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.150 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.151 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.151 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118370+0000", 2024-08-17T17:19:40.151 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.151 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.151 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.151 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118370+0000", 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118371+0000", 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1119999999999999e-06 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.152 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118375+0000", 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4360000000000004e-06 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118388+0000", 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2894000000000001e-05 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118396+0000", 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5580000000000003e-06 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118447+0000", 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0973000000000001e-05 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118556+0000", 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010900699999999999 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.118981+0000", 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00042489399999999999 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.130868+0000", 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011887415 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.155 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:161000 9.18 9:1fd6ee38:::benchmark_data_smithi142_157452_object28879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.123098+0000", 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0086468489999999999, 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0089349259999999993, 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:19:40.156 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 161000 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123098+0000", 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123098+0000", 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123100+0000", 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8279999999999999e-06 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123106+0000", 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9390000000000002e-06 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.158 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123126+0000", 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9726000000000001e-05 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123135+0000", 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2150000000000007e-06 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123195+0000", 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9771000000000001e-05 2024-08-17T17:19:40.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123300+0000", 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000104867 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.123731+0000", 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000430965 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.129287+0000", 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0055560779999999999 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.130960+0000", 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0016728419999999999 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.161 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:160995 9.14 e179/160 9:299ba63a:::benchmark_data_smithi142_157452_object28874:head v 179'7031, mlcod=179'7029)", 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.126007+0000", 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0057385470000000001, 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060934800000000001, 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126007+0000", 2024-08-17T17:19:40.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126007+0000", 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126008+0000", 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5489999999999999e-06 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126016+0000", 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.123e-06 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126032+0000", 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5840000000000001e-05 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126042+0000", 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5859999999999994e-06 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127229+0000", 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011873299999999999 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127247+0000", 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8244e-05 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.165 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:160998 9.3 e179/160 9:c3bd4a20:::benchmark_data_smithi142_157452_object28877:head v 179'7019, mlcod=179'7017)", 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.126935+0000", 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0048100499999999997, 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052110009999999998, 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.166 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126935+0000", 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126935+0000", 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126936+0000", 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5899999999999996e-07 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126942+0000", 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0530000000000003e-06 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126956+0000", 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3967e-05 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.126963+0000", 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8369999999999999e-06 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127967+0000", 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001003868 2024-08-17T17:19:40.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127985+0000", 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8757999999999999e-05 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:161002 9.3 e179/160 9:c0f0c2ba:::benchmark_data_smithi142_157452_object28881:head v 179'7020, mlcod=179'7017)", 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.127594+0000", 2024-08-17T17:19:40.170 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0041510549999999998, 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046007499999999998, 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127594+0000", 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.171 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127594+0000", 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127595+0000", 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6100000000000001e-07 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127600+0000", 2024-08-17T17:19:40.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5480000000000001e-06 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127612+0000", 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1596999999999999e-05 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127619+0000", 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6159999999999998e-06 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.173 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.130535+0000", 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029169259999999998 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.130553+0000", 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7447000000000001e-05 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.174 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:161003 9.2 9:46b8f4f4:::benchmark_data_smithi142_157452_object28882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:19:40.127629+0000", 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0041159600000000001, 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046116719999999998, 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 161003 2024-08-17T17:19:40.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127629+0000", 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127629+0000", 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127631+0000", 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5680000000000001e-06 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127636+0000", 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3499999999999996e-06 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127649+0000", 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2427e-05 2024-08-17T17:19:40.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127656+0000", 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3239999999999997e-06 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127716+0000", 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9857e-05 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:40.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.127845+0000", 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012893599999999999 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.128250+0000", 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000405542 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:19:40.131727+0000", 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034767560000000001 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.179 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:40.180 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.180 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:40.180 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:19:40.180 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 9 2024-08-17T17:19:40.180 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:19:40.180 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:40.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:40.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:40.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:40.453 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:40.490 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:19:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:19:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:19:40.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:40.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:40.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:40.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:40.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:40.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:40.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:41.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:41.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:40 smithi142 ceph-mon[106729]: pgmap v591: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 14424/318183 objects degraded (4.533%); 1234/318183 objects misplaced (0.388%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:19:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14424/318183 objects degraded (4.533%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:41.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:40 smithi142 ceph-mon[103614]: pgmap v591: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 14424/318183 objects degraded (4.533%); 1234/318183 objects misplaced (0.388%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:19:41.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14424/318183 objects degraded (4.533%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:41.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:41.243 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:19:41.267 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:19:41.268 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:19:41.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:41.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:40 smithi152 ceph-mon[93456]: pgmap v591: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 14424/318183 objects degraded (4.533%); 1234/318183 objects misplaced (0.388%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:19:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14424/318183 objects degraded (4.533%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:41.378 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:41.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:41.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:41.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:41.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:41.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:41.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:41.814 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:41.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:41.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:41.967 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:19:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:19:41] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-08-17T17:19:42.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:42.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:42.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:42.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:19:42.328 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:19:42.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-08-17T17:19:42.333 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:42.402 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:42.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:42.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:42.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:42 smithi142 ceph-mon[106729]: pgmap v592: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 14424/316278 objects degraded (4.561%); 632/316278 objects misplaced (0.200%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T17:19:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:42 smithi142 ceph-mon[103614]: pgmap v592: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 14424/316278 objects degraded (4.561%); 632/316278 objects misplaced (0.200%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T17:19:42.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:42.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:42.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:19:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:42 smithi152 ceph-mon[93456]: pgmap v592: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 14424/316278 objects degraded (4.561%); 632/316278 objects misplaced (0.200%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T17:19:42.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:42.863 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:42.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:43.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:43.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:43.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:43.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:43.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:43.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:43.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:43.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:43.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:43.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:43.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:43.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:43.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:43.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:43.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:43.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:44.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:44.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:44.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:44.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:44.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:44.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:44.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:44.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:44.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:44.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:44.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:44.721 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:44 smithi142 ceph-mon[103614]: pgmap v593: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 13764/306684 objects degraded (4.488%); 632/306684 objects misplaced (0.206%); 7.7 MiB/s, 122 objects/s recovering 2024-08-17T17:19:44.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:44 smithi142 ceph-mon[106729]: pgmap v593: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 13764/306684 objects degraded (4.488%); 632/306684 objects misplaced (0.206%); 7.7 MiB/s, 122 objects/s recovering 2024-08-17T17:19:44.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:44 smithi152 ceph-mon[93456]: pgmap v593: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 13764/306684 objects degraded (4.488%); 632/306684 objects misplaced (0.206%); 7.7 MiB/s, 122 objects/s recovering 2024-08-17T17:19:44.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:44.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:44.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:44.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.033 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:45.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:45.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:45.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:45.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:45.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:45.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:45.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:45.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:45.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:45.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:45.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13764/306684 objects degraded (4.488%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13764/306684 objects degraded (4.488%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:45.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:45.708 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:45.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:45.790 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:45.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13764/306684 objects degraded (4.488%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:19:45.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:45.953 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:45.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:46.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:46.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:46.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:46.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:46.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:46.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:46.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:46.552 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:46 smithi142 ceph-mon[106729]: pgmap v594: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 13764/303969 objects degraded (4.528%); 632/303969 objects misplaced (0.208%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:19:46.553 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:46 smithi142 ceph-mon[103614]: pgmap v594: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 13764/303969 objects degraded (4.528%); 632/303969 objects misplaced (0.208%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:19:46.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:46.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:46.690 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:46.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:46 smithi152 ceph-mon[93456]: pgmap v594: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 13764/303969 objects degraded (4.528%); 632/303969 objects misplaced (0.208%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:19:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:46.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:46.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:46.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:47.052 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:19:47.052 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:47.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:47.070 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:19:47.070 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:47.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:47.088 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:19:47.088 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:47.105 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:19:47.105 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:47.122 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:19:47.122 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:47.140 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:19:47.140 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:47.158 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:19:47.158 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:47.175 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:19:47.176 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:47.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:47.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:47.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:47.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:47.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:47.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:47.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:47.461 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:19:47.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:47.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:47 smithi142 ceph-mon[106729]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T17:19:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/119898597' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:19:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1484167671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:47 smithi142 ceph-mon[103614]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T17:19:47.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/119898597' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:19:47.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1484167671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:47.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:47.732 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:47 smithi152 ceph-mon[93456]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T17:19:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/119898597' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:19:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1484167671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:47.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:47.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:47.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.996 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:47.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:48.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:48.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:48.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:48.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:48.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:48.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:48.402 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:48.504 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:48.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:48.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:48.669 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:48.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:48.771 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:48.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:48 smithi152 ceph-mon[93456]: pgmap v596: 99 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 951 op/s; 13764/295857 objects degraded (4.652%); 7.8 MiB/s, 125 objects/s recovering 2024-08-17T17:19:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:48 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:19:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:48 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/119898597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:19:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:48 smithi152 ceph-mon[93456]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T17:19:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[106729]: pgmap v596: 99 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 951 op/s; 13764/295857 objects degraded (4.652%); 7.8 MiB/s, 125 objects/s recovering 2024-08-17T17:19:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:19:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/119898597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:19:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[106729]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T17:19:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[103614]: pgmap v596: 99 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 951 op/s; 13764/295857 objects degraded (4.652%); 7.8 MiB/s, 125 objects/s recovering 2024-08-17T17:19:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:19:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:48.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/119898597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:19:48.876 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:48 smithi142 ceph-mon[103614]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T17:19:48.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:48.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:49.043 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:49.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:49.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:49.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:49.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:49.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:49.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:49.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:49.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:49.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:49.578 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:49.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:49.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:49.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:49.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:49.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:49.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:49.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:50.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:50.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:50.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:50.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:50.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:50.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:50.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:50.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:50.473 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:19:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:19:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:19:50.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:50.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:50.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:50.675 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:50.769 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:50 smithi142 ceph-mon[106729]: pgmap v598: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 creating+peering, 96 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 6540/289902 objects degraded (2.256%); 6144/289902 objects misplaced (2.119%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T17:19:50.769 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6540/289902 objects degraded (2.256%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:50.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:50 smithi142 ceph-mon[103614]: pgmap v598: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 creating+peering, 96 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 6540/289902 objects degraded (2.256%); 6144/289902 objects misplaced (2.119%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T17:19:50.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6540/289902 objects degraded (2.256%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:50.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:50.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:50 smithi152 ceph-mon[93456]: pgmap v598: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 creating+peering, 96 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 6540/289902 objects degraded (2.256%); 6144/289902 objects misplaced (2.119%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T17:19:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6540/289902 objects degraded (2.256%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:50.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:50.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.954 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:50.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:51.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:51.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:51.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:51.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:51.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:51.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:51.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:51.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:51.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:51.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:51.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:51.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:51.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:51.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:51.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:52.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:52.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:52.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:52.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:19:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:19:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:19:51] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-08-17T17:19:52.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:52.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:52.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:52.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:52.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:52.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:52.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:52.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:52.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:52.709 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:52 smithi142 ceph-mon[106729]: pgmap v599: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 creating+peering, 96 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 6540/288669 objects degraded (2.266%); 6144/288669 objects misplaced (2.128%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:19:52.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:52 smithi142 ceph-mon[103614]: pgmap v599: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 creating+peering, 96 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 6540/288669 objects degraded (2.266%); 6144/288669 objects misplaced (2.128%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:19:52.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:52.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:52 smithi152 ceph-mon[93456]: pgmap v599: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 creating+peering, 96 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 6540/288669 objects degraded (2.266%); 6144/288669 objects misplaced (2.128%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:19:52.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:52.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:53.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:53.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:53.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:53.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:53.270 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:19:53.270 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:19:53.270 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:19:53.270 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:19:53.270 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:19:53.270 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:19:53.271 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:19:53.271 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:19:53.271 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:19:53.271 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:19:53.271 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:19:53.271 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:19:53.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:53.297 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:19:53.297 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:19:53.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:53.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:53.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:53.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:53.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:53.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:53.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:53.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:53.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:53.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:19:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:53 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:19:53.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:53.841 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:53 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:19:53.842 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:53 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:19:53.871 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:53.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:53.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:54.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:54.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:54.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:19:54.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:54.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:54.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:54.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:54.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:54.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:54.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:54.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:54.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:54 smithi142 ceph-mon[106729]: pgmap v600: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 6246/278325 objects degraded (2.244%); 5884/278325 objects misplaced (2.114%); 7.2 MiB/s, 114 objects/s recovering 2024-08-17T17:19:54.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:54 smithi142 ceph-mon[103614]: pgmap v600: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 6246/278325 objects degraded (2.244%); 5884/278325 objects misplaced (2.114%); 7.2 MiB/s, 114 objects/s recovering 2024-08-17T17:19:54.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:54.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:54.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:54.739 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:19:54.739 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:19:54.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-08-17T17:19:54.779 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:54.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:54 smithi152 ceph-mon[93456]: pgmap v600: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 6246/278325 objects degraded (2.244%); 5884/278325 objects misplaced (2.114%); 7.2 MiB/s, 114 objects/s recovering 2024-08-17T17:19:54.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:54.862 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:54.863 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:54.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:54.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:54.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:55.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:55.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:55.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:55.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:55.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:55.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:55.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:55.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:55.508 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:55.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:55.643 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:55.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:55.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.818 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:55.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6246/278325 objects degraded (2.244%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:55.861 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6246/278325 objects degraded (2.244%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:55.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6246/278325 objects degraded (2.244%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:55.920 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:55.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:55.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:56.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:56.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:56.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:56.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:56.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:56.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:56.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:56.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:56.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:56.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:56.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:56.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:56.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:56 smithi152 ceph-mon[93456]: pgmap v601: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 6246/275634 objects degraded (2.266%); 5884/275634 objects misplaced (2.135%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:19:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:56 smithi142 ceph-mon[103614]: pgmap v601: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 6246/275634 objects degraded (2.266%); 5884/275634 objects misplaced (2.135%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:19:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:56 smithi142 ceph-mon[106729]: pgmap v601: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 6246/275634 objects degraded (2.266%); 5884/275634 objects misplaced (2.135%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:19:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:19:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:56.883 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:56.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:57.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:57.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:57.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:57.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:57.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:57.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:57.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:57.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:57.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:57.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:57.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:57.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:57.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:57.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:57.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:57.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:58.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:58.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:58.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:58.234 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:58.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:58.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:58.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:58.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:58.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:58.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:58.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:58.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:58.802 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:58 smithi142 ceph-mon[106729]: pgmap v602: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 6246/267090 objects degraded (2.339%); 5884/267090 objects misplaced (2.203%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:19:58.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:58 smithi142 ceph-mon[103614]: pgmap v602: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 6246/267090 objects degraded (2.339%); 5884/267090 objects misplaced (2.203%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:19:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:58 smithi152 ceph-mon[93456]: pgmap v602: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 6246/267090 objects degraded (2.339%); 5884/267090 objects misplaced (2.203%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:19:58.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:58.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:59.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:59.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:59.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:59.156 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:19:59.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:59.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:59.190 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:19:59.190 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:59.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:59.223 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:19:59.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:59.252 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:19:59.252 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:59.281 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:19:59.281 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:59.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:59.316 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:19:59.316 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:59.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:59.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:59.355 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:19:59.355 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:59.382 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:19:59.382 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76674 8.b 8.f644187b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.409 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.340003+0000", 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.06513778499999, 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98417220800000005, 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76674 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.410 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340003+0000", 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340003+0000", 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340005+0000", 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.838e-06 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340071+0000", 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6319e-05 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340082+0000", 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1541e-05 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340091+0000", 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3909999999999998e-06 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340146+0000", 2024-08-17T17:19:59.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5093000000000003e-05 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340160+0000", 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4194e-05 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.324163+0000", 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98400335699999997 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.324175+0000", 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1474999999999999e-05 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76676 8.0 8.54e62300 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.340110+0000", 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.06503042200001, 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98566959399999998, 2024-08-17T17:19:59.415 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76676 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.416 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340110+0000", 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340110+0000", 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340111+0000", 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9599999999999999e-07 2024-08-17T17:19:59.417 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340174+0000", 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3225000000000005e-05 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340182+0000", 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3939999999999996e-06 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340189+0000", 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355681+0000", 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015492662000000001 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355691+0000", 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8390000000000002e-06 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325769+0000", 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97007827599999996 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325780+0000", 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0197000000000001e-05 2024-08-17T17:19:59.420 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76677 8.0 8.f41a3230 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.340201+0000", 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.06493991799999, 2024-08-17T17:19:59.421 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98562624399999998, 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76677 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.422 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340201+0000", 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340201+0000", 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340201+0000", 2024-08-17T17:19:59.423 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8500000000000001e-07 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340246+0000", 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4861000000000003e-05 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340254+0000", 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7419999999999997e-06 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.424 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340260+0000", 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1550000000000004e-06 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355716+0000", 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015456218000000001 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355724+0000", 2024-08-17T17:19:59.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0579999999999998e-06 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325813+0000", 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97008853399999995 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325827+0000", 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4090999999999999e-05 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.426 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76681 8.0 8.92c0d8e0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.340269+0000", 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.06487149600002, 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98559793299999998, 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.427 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76681 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340269+0000", 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340269+0000", 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340270+0000", 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7599999999999997e-07 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340309+0000", 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9366999999999998e-05 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340314+0000", 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7369999999999997e-06 2024-08-17T17:19:59.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340317+0000", 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.202e-06 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355747+0000", 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015430233 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355754+0000", 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2130000000000002e-06 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325857+0000", 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.970102405 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325867+0000", 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0200000000000001e-05 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.431 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76686 8.f 8.7d3938f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.340323+0000", 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.06481748499999, 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98528524200000001, 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.432 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76686 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340323+0000", 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340323+0000", 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340323+0000", 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8500000000000002e-07 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340360+0000", 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6245000000000003e-05 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340367+0000", 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5499999999999997e-06 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340372+0000", 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9740000000000001e-06 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354656+0000", 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014284236000000001 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354666+0000", 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9899999999999992e-06 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325595+0000", 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97092884700000004 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325608+0000", 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3115000000000001e-05 2024-08-17T17:19:59.436 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76688 8.f 8.27913cdf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.340382+0000", 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.06475827700001, 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98526125799999997, 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.437 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76688 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340382+0000", 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340382+0000", 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340383+0000", 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9999999999999998e-07 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340429+0000", 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6350999999999997e-05 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340443+0000", 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3509000000000001e-05 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.340449+0000", 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6100000000000002e-06 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354693+0000", 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014243473 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354700+0000", 2024-08-17T17:19:59.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4109999999999999e-06 2024-08-17T17:19:59.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325634+0000", 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97093423499999998 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325643+0000", 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1689999999999998e-06 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76690 8.0 8.1061b0c0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.345320+0000", 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.05982018100002, 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98059265299999998, 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.444 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76690 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.345320+0000", 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.345320+0000", 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.345321+0000", 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7600000000000006e-07 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.345360+0000", 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8600999999999999e-05 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.345367+0000", 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4719999999999998e-06 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.345373+0000", 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4960000000000004e-06 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355777+0000", 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010404313 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.355784+0000", 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8689999999999998e-06 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325903+0000", 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97011882800000004 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325913+0000", 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0098000000000001e-05 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76700 8.f 8.9a3d33cf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.354841+0000", 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.05029949300001, 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97083307799999996, 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.449 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76700 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354841+0000", 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354841+0000", 2024-08-17T17:19:59.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354842+0000", 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3880000000000001e-06 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354894+0000", 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1378000000000002e-05 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354904+0000", 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0217000000000001e-05 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354911+0000", 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9920000000000001e-06 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354978+0000", 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6650999999999996e-05 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.354985+0000", 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5159999999999997e-06 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325665+0000", 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97068007000000001 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325674+0000", 2024-08-17T17:19:59.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8659999999999995e-06 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76731 8.f 8.a01537ef (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.385229+0000", 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 440.01991168299998, 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.94047750100000005, 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:19:59.454 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76731 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385229+0000", 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385229+0000", 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385230+0000", 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.61e-06 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385276+0000", 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5151000000000002e-05 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385291+0000", 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5218000000000001e-05 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385298+0000", 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.092e-06 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385352+0000", 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3798999999999998e-05 2024-08-17T17:19:59.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.385360+0000", 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4700000000000002e-06 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325698+0000", 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.94033738200000005 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.325706+0000", 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7790000000000001e-06 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148234 8.b 8:d4d71f30:::benchmark_data_smithi142_139631_object26938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.070259+0000", 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.33488155399999, 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0198849080000001, 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.459 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.461 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148234 2024-08-17T17:19:59.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.461 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.070259+0000", 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.070259+0000", 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.070261+0000", 2024-08-17T17:19:59.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0770000000000002e-06 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.070272+0000", 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0708e-05 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.070290+0000", 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8080999999999998e-05 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.070302+0000", 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.187e-05 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.071762+0000", 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001460055 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.071776+0000", 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3827e-05 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.073942+0000", 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.002166788 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.074069+0000", 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000126785 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.074152+0000", 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2776999999999997e-05 2024-08-17T17:19:59.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.083722+0000", 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0095697119999999993 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.083731+0000", 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8509999999999994e-06 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090040+0000", 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063080769999999996 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090095+0000", 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5067e-05 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090144+0000", 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9233000000000001e-05 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.467 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148249 8.b 8:d47e8c49:::benchmark_data_smithi142_139631_object26953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.093803+0000", 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.31133762799999, 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99647287799999995, 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148249 2024-08-17T17:19:59.468 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.093803+0000", 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.093803+0000", 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.093804+0000", 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5370000000000001e-06 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.093810+0000", 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5260000000000001e-06 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.093827+0000", 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7181999999999999e-05 2024-08-17T17:19:59.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.093835+0000", 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0790000000000001e-06 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.094001+0000", 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016547 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.094010+0000", 2024-08-17T17:19:59.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4860000000000003e-06 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.077774+0000", 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98376361700000003 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.077913+0000", 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013915800000000001 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.077982+0000", 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9362000000000004e-05 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.083765+0000", 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005782667 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.086761+0000", 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029960569999999999 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090187+0000", 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034260409999999999 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090234+0000", 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7206000000000002e-05 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090276+0000", 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1489999999999997e-05 2024-08-17T17:19:59.474 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148256 8.b 8:dddd2e68:::benchmark_data_smithi142_139631_object26960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.101929+0000", 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.30321182199998, 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99894171399999998, 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.475 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148256 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.101929+0000", 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.101929+0000", 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.101930+0000", 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8199999999999999e-06 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.101937+0000", 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1040000000000003e-06 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.101954+0000", 2024-08-17T17:19:59.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7535999999999999e-05 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.101962+0000", 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3229999999999998e-06 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.102182+0000", 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00021950999999999999 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.102192+0000", 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0001e-05 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.080434+0000", 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97824165600000001 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.080547+0000", 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000113806 2024-08-17T17:19:59.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.080653+0000", 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105077 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.083840+0000", 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031877590000000001 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090154+0000", 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063138329999999996 2024-08-17T17:19:59.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100762+0000", 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010607810000000001 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100819+0000", 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6595999999999998e-05 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100870+0000", 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1882999999999998e-05 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148261 8.b 8:db61e7d2:::benchmark_data_smithi142_139631_object26965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.110470+0000", 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.29467075700001, 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99714070700000002, 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.482 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148261 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.110470+0000", 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.110470+0000", 2024-08-17T17:19:59.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.110472+0000", 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8610000000000001e-06 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.110480+0000", 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1470000000000001e-06 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.484 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.110498+0000", 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8671000000000001e-05 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.110507+0000", 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8559999999999996e-06 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.111017+0000", 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000509963 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.111029+0000", 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1610000000000001e-05 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.083877+0000", 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.97284797000000001 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.083965+0000", 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8341e-05 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.084072+0000", 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010669400000000001 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090339+0000", 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062671139999999998 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090353+0000", 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3689000000000001e-05 2024-08-17T17:19:59.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107508+0000", 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017154982999999999 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107563+0000", 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5572999999999998e-05 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107610+0000", 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7234999999999998e-05 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148284 8.b 8:d7b50505:::benchmark_data_smithi142_139631_object26988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.143272+0000", 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.26186829300002, 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.96446558299999996, 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.489 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148284 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143272+0000", 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.490 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143272+0000", 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143274+0000", 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7910000000000001e-06 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143280+0000", 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.677e-06 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143295+0000", 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5532e-05 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143303+0000", 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9140000000000008e-06 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143371+0000", 2024-08-17T17:19:59.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8342000000000004e-05 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.143382+0000", 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0821e-05 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.086790+0000", 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.94340800199999997 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.086917+0000", 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012661000000000001 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.087032+0000", 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114906 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.093946+0000", 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0069139379999999997 2024-08-17T17:19:59.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100667+0000", 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067213159999999997 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107655+0000", 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.006988378 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107695+0000", 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9629999999999998e-05 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107738+0000", 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2725999999999997e-05 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148312 8.b 8:dcfdee2c:::benchmark_data_smithi142_139631_object27016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.496 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.179179+0000", 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.22596157300001, 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.92872116000000005, 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148312 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.497 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179179+0000", 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179179+0000", 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179181+0000", 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.829e-06 2024-08-17T17:19:59.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179187+0000", 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1689999999999997e-06 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179204+0000", 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6881e-05 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179212+0000", 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3869999999999995e-06 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179277+0000", 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4714000000000004e-05 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.179287+0000", 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9049999999999992e-06 2024-08-17T17:19:59.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090384+0000", 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.91109743600000004 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090500+0000", 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115819 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.090624+0000", 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012438400000000001 2024-08-17T17:19:59.502 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.502 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.503 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.503 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100585+0000", 2024-08-17T17:19:59.503 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0099606150000000008 2024-08-17T17:19:59.503 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.503 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.503 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100672+0000", 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6874999999999997e-05 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107809+0000", 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0071372099999999997 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107856+0000", 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6584e-05 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107900+0000", 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4351999999999998e-05 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.505 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148313 8.b 8:d18473d4:::benchmark_data_smithi142_139631_object27017:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.180996+0000", 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.224144368, 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93347365299999996, 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148313 2024-08-17T17:19:59.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.180996+0000", 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.180996+0000", 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.180998+0000", 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.59e-06 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.181005+0000", 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.447e-06 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.508 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.181022+0000", 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7203000000000001e-05 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.181030+0000", 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7249999999999997e-06 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.181097+0000", 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7028999999999998e-05 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.181110+0000", 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.241e-05 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.093978+0000", 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.912868016 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.094094+0000", 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116047 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.094213+0000", 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011971 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100653+0000", 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064395980000000004 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100722+0000", 2024-08-17T17:19:59.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9457999999999998e-05 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114372+0000", 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013649766000000001 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114414+0000", 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1811000000000003e-05 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.512 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114470+0000", 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5843e-05 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148314 8.b 8:da6b0cb9:::benchmark_data_smithi142_139631_object27018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.182210+0000", 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.22293018699997, 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93244201500000001, 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.513 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148314 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182210+0000", 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.514 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182210+0000", 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182211+0000", 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.17e-06 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182216+0000", 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4100000000000001e-06 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182231+0000", 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5379999999999998e-05 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182239+0000", 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3849999999999996e-06 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182293+0000", 2024-08-17T17:19:59.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4514e-05 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.182302+0000", 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0089999999999996e-06 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.097156+0000", 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.91485378100000003 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.097254+0000", 2024-08-17T17:19:59.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7788000000000006e-05 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.097351+0000", 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7194000000000005e-05 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100709+0000", 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033576309999999998 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107455+0000", 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067464409999999997 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114531+0000", 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0070756339999999999 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114616+0000", 2024-08-17T17:19:59.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5498e-05 2024-08-17T17:19:59.520 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.520 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.520 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114652+0000", 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6180000000000003e-05 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148323 8.b 8:d84fdd55:::benchmark_data_smithi142_139631_object27027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.194634+0000", 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.21050659000002, 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93066821300000002, 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.521 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148323 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194634+0000", 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194634+0000", 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194636+0000", 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6279999999999999e-06 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194641+0000", 2024-08-17T17:19:59.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9860000000000002e-06 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194655+0000", 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4785e-05 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194663+0000", 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9680000000000006e-06 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194728+0000", 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4618999999999999e-05 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.194741+0000", 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3176999999999999e-05 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100907+0000", 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.90616548699999999 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.525 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.100999+0000", 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2565000000000006e-05 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.101095+0000", 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6088999999999994e-05 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107442+0000", 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.006346417 2024-08-17T17:19:59.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107912+0000", 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046994499999999999 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.125128+0000", 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017216314999999999 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.125253+0000", 2024-08-17T17:19:59.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000125196 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.125302+0000", 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9036000000000003e-05 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148339 8.b 8:d2194d34:::benchmark_data_smithi142_139631_object27043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.215929+0000", 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.18921181600001, 2024-08-17T17:19:59.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.916341823, 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148339 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.215929+0000", 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.529 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.215929+0000", 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.215931+0000", 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8670000000000001e-06 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.215939+0000", 2024-08-17T17:19:59.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2339999999999994e-06 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.215959+0000", 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0611e-05 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.215968+0000", 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5229999999999995e-06 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.216078+0000", 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109923 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.216088+0000", 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0572e-05 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.104426+0000", 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88833800799999996 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.532 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.104537+0000", 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111056 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.104645+0000", 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010775100000000001 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.533 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107956+0000", 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033107369999999998 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.107965+0000", 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1069999999999994e-06 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.132181+0000", 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024216222999999999 2024-08-17T17:19:59.534 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.132230+0000", 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8545999999999999e-05 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.132271+0000", 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0664999999999998e-05 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.535 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:148345 8.b 8:d329f60c:::benchmark_data_smithi142_139631_object27049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:14:05.225696+0000", 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.17944449100003, 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.90667455900000005, 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 148345 2024-08-17T17:19:59.536 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225696+0000", 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225696+0000", 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:19:59.537 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225698+0000", 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9700000000000002e-06 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225704+0000", 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0440000000000001e-06 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225720+0000", 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5562000000000001e-05 2024-08-17T17:19:59.538 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225728+0000", 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4859999999999997e-06 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225798+0000", 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9636999999999998e-05 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:05.225810+0000", 2024-08-17T17:19:59.539 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1865000000000001e-05 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.108006+0000", 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88219659100000003 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.108134+0000", 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012833300000000001 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.540 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.108245+0000", 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000110272 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.114504+0000", 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062595250000000002 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.125315+0000", 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010810663 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.541 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.132308+0000", 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0069935400000000003 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.132343+0000", 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4802999999999999e-05 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:14:06.132371+0000", 2024-08-17T17:19:59.542 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7268e-05 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:19:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:59.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:59.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:59.677 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:59.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:59.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:19:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:59.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:19:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/890549837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:19:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/890549837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:59.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:19:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/890549837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:59.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:59.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:19:59.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:00.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:00.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:00.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:00.188 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:00.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:00.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:00.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:00.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:00.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:00.560 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:20:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:20:00.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:00.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:00.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:00.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:00.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:00.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-mon[93456]: pgmap v603: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 5918/260523 objects degraded (2.272%); 5554/260523 objects misplaced (2.132%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:20:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-mon[93456]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded 2024-08-17T17:20:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-mon[93456]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:20:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-mon[93456]: [WRN] PG_DEGRADED: Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded 2024-08-17T17:20:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-mon[93456]: pg 9.15 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2024-08-17T17:20:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:00.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[106729]: pgmap v603: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 5918/260523 objects degraded (2.272%); 5554/260523 objects misplaced (2.132%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[106729]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[106729]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[106729]: [WRN] PG_DEGRADED: Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[106729]: pg 9.15 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[103614]: pgmap v603: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 984 op/s; 5918/260523 objects degraded (2.272%); 5554/260523 objects misplaced (2.132%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[103614]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[103614]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[103614]: [WRN] PG_DEGRADED: Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded 2024-08-17T17:20:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[103614]: pg 9.15 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2024-08-17T17:20:00.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5918/260523 objects degraded (2.272%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:00.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:00.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:00.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:01.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:01.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:01.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.155 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:01.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:01.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:01.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:01.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:01.389 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:01.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:01.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:01.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:01.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:01.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:01.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:01.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:01.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:01.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:01.933 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:02.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:02.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:20:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:20:01] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-08-17T17:20:02.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:02.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:02.200 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:02.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:02.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:02.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:02.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:02.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:02.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:02.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:02.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:02 smithi142 ceph-mon[106729]: pgmap v604: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 5918/259143 objects degraded (2.284%); 5554/259143 objects misplaced (2.143%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:20:02.682 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:02 smithi142 ceph-mon[103614]: pgmap v604: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 5918/259143 objects degraded (2.284%); 5554/259143 objects misplaced (2.143%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:20:02.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:02.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:20:02.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:02 smithi152 ceph-mon[93456]: pgmap v604: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 5918/259143 objects degraded (2.284%); 5554/259143 objects misplaced (2.143%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:20:02.846 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:02.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:02.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:03.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:03.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:03.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:03.222 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:03.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:03.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:03.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:03.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:03.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:03.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:03.587 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:20:03.614 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1297420165' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:03.615 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:03 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:03.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:03.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:03.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1297420165' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:03.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:03 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1297420165' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:03 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:03.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:03.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:03.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:04.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:04.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:04.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:04.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:04.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:04.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:04.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.368 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:04.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:04.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:04.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:04.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:04.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:04.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:04.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[106729]: pgmap v605: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 5606/248997 objects degraded (2.251%); 5213/248997 objects misplaced (2.094%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:20:04.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:04.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:04.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[106729]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T17:20:04.767 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[103614]: pgmap v605: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 5606/248997 objects degraded (2.251%); 5213/248997 objects misplaced (2.094%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:20:04.767 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:04.767 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:04.767 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:04 smithi142 ceph-mon[103614]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T17:20:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:04.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:04.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:04.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:05.017 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:04 smithi152 ceph-mon[93456]: pgmap v605: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 5606/248997 objects degraded (2.251%); 5213/248997 objects misplaced (2.094%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:20:05.017 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:04 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:05.017 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:05.017 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:04 smithi152 ceph-mon[93456]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T17:20:05.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:05.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:05.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:05.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:05.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:05.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:05.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.331 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:05.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:05.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:05.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:05.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:05.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:05.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:05.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:05.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:05.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:05.724 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:05.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:05.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:05.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:05.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5606/248997 objects degraded (2.251%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5606/248997 objects degraded (2.251%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:05.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:05.995 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5606/248997 objects degraded (2.251%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:05.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:06.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:06.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:06.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:06.270 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:06.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:06.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:06.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:06.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.557 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:06.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:06.569 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:186062 9.10 9:097e844b:::benchmark_data_smithi142_157452_object53941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:06.552769+0000", 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.015127253, 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015172104, 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:06.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 186062 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.552769+0000", 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.552769+0000", 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:06.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.552768+0000", 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.552775+0000", 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6000000000000001e-06 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:06.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.552791+0000", 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5107e-05 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.552796+0000", 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4929999999999998e-06 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.553814+0000", 2024-08-17T17:20:06.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001018218 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.553952+0000", 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013765000000000001 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.554070+0000", 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011842500000000001 2024-08-17T17:20:06.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.558089+0000", 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004018471 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.558281+0000", 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000192337 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:186072 9.14 e182/160 9:2ca2cbbb:::benchmark_data_smithi142_157452_object53951:head v 182'7829, mlcod=182'7827)", 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:06.567050+0000", 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00084694499999999999, 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00097804599999999999, 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567050+0000", 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567050+0000", 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567048+0000", 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567060+0000", 2024-08-17T17:20:06.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1901e-05 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567081+0000", 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1229999999999998e-05 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567089+0000", 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2400000000000007e-06 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567156+0000", 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6340000000000005e-05 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567183+0000", 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7747999999999999e-05 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:186069 9.e e182/154 9:7a0e2231:::benchmark_data_smithi142_157452_object53948:head v 182'11601, mlcod=182'11599)", 2024-08-17T17:20:06.579 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:06.567896+0000", 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.3799999999999996e-07, 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018564699999999999, 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567896+0000", 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567896+0000", 2024-08-17T17:20:06.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567895+0000", 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567902+0000", 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3270000000000003e-06 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567917+0000", 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4918e-05 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567923+0000", 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0410000000000003e-06 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:06.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567972+0000", 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8816000000000002e-05 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:06.567990+0000", 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8009e-05 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 3 2024-08-17T17:20:06.583 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:06.602 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:20:06.602 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:20:06.603 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:20:06.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:06.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:06.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:20:06.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:06 smithi152 ceph-mon[93456]: pgmap v607: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 5606/246327 objects degraded (2.276%); 5213/246327 objects misplaced (2.116%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:20:06.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:06.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:06.889 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:06.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:06.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:06.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:07.042 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:06 smithi142 ceph-mon[103614]: pgmap v607: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 5606/246327 objects degraded (2.276%); 5213/246327 objects misplaced (2.116%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:20:07.043 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:06 smithi142 ceph-mon[106729]: pgmap v607: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 5606/246327 objects degraded (2.276%); 5213/246327 objects misplaced (2.116%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:20:07.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:07.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:07.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:07.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:07.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:20:07.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:07.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:07.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:07.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:07.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:07.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:07.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:07.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:07.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:07.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:07.809 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:07.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:07.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:07.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:08.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:08.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:08.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:08.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:08.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:08.388 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:08.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:08.528 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:08.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:08.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:08.641 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:20:08.685 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:20:08.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-17T17:20:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:08.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:08.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:08.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:08.926 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:08 smithi142 ceph-mon[106729]: pgmap v608: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 5606/238395 objects degraded (2.352%); 5213/238395 objects misplaced (2.187%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:20:08.927 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:08 smithi142 ceph-mon[103614]: pgmap v608: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 5606/238395 objects degraded (2.352%); 5213/238395 objects misplaced (2.187%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:20:08.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:08.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:09.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:09.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:08 smithi152 ceph-mon[93456]: pgmap v608: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 5606/238395 objects degraded (2.352%); 5213/238395 objects misplaced (2.187%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:20:09.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:09.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:09.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:09.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:09.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:20:09.477 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:20:09.478 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:09.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:20:09.507 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:20:09.538 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:09.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:09.566 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:20:09.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:09.595 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:20:09.596 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:20:09.622 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:09.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:09.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:09.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:09.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:09.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:09.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:10.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:10.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/729434101' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:10.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:10.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/729434101' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:10.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/729434101' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:10.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:10.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:10.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:10.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:10.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.350 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:10.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:10.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:10.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:20:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:20:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:20:10.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.645 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:10.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:10.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:10.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:10 smithi142 ceph-mon[106729]: pgmap v609: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 5332/232308 objects degraded (2.295%); 4930/232308 objects misplaced (2.122%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5332/232308 objects degraded (2.295%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:10 smithi142 ceph-mon[103614]: pgmap v609: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 5332/232308 objects degraded (2.295%); 4930/232308 objects misplaced (2.122%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5332/232308 objects degraded (2.295%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:10.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:10.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:10.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:11.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:11.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:11.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:11.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:10 smithi152 ceph-mon[93456]: pgmap v609: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 5332/232308 objects degraded (2.295%); 4930/232308 objects misplaced (2.122%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:11.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5332/232308 objects degraded (2.295%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:11.086 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:11.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:11.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:11.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:11.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:11.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:11.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:11.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:11.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:11.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:11.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:11.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:11.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:11.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:11.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:11.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:11.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:11.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:11.941 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:11.941 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:11.967 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:11.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:11.968 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:20:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:20:11] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.51.0" 2024-08-17T17:20:11.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.015 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:12.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:12.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:12.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:12.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:12.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:12.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:12.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:12.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:12.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:12.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:12.799 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:20:12.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:12.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:12.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:12.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:12 smithi152 ceph-mon[93456]: pgmap v610: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 5332/231060 objects degraded (2.308%); 4930/231060 objects misplaced (2.134%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2092977782' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:13.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:13.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:12 smithi142 ceph-mon[106729]: pgmap v610: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 5332/231060 objects degraded (2.308%); 4930/231060 objects misplaced (2.134%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2092977782' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:13.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:12 smithi142 ceph-mon[103614]: pgmap v610: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 5332/231060 objects degraded (2.308%); 4930/231060 objects misplaced (2.134%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2092977782' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:13.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:13.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:13.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:13.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:13.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:13.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:13.481 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:13.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:13.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:13.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:13.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:13.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:13.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:20:13.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:13.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:14.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:13 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2092977782' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:13 smithi152 ceph-mon[93456]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T17:20:14.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:14.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:14.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:13 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:14.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2092977782' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:14.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:13 smithi142 ceph-mon[106729]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T17:20:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:13 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:14.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2092977782' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:14.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:13 smithi142 ceph-mon[103614]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T17:20:14.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:14.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:14.303 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:14.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:14.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:14.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:14.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:14.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:14.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:14.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:14.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:14.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:14.796 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:14.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:14.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:no valid command found; 10 closest matches: 2024-08-17T17:20:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:0 2024-08-17T17:20:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:1 2024-08-17T17:20:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:2 2024-08-17T17:20:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:abort 2024-08-17T17:20:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:assert 2024-08-17T17:20:14.896 INFO:teuthology.orchestra.run.smithi142.stderr:bench [] [] [] [] 2024-08-17T17:20:14.896 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:20:14.896 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs files list 2024-08-17T17:20:14.896 INFO:teuthology.orchestra.run.smithi142.stderr:bluefs stats 2024-08-17T17:20:14.896 INFO:teuthology.orchestra.run.smithi142.stderr:bluestore allocator dump block 2024-08-17T17:20:14.896 INFO:teuthology.orchestra.run.smithi142.stderr:Error EINVAL: invalid command 2024-08-17T17:20:14.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:14.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:14 smithi152 ceph-mon[93456]: pgmap v612: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 5040/221319 objects degraded (2.277%); 4634/221319 objects misplaced (2.094%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:20:15.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:15.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:14 smithi142 ceph-mon[106729]: pgmap v612: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 5040/221319 objects degraded (2.277%); 4634/221319 objects misplaced (2.094%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:20:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:14 smithi142 ceph-mon[103614]: pgmap v612: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 5040/221319 objects degraded (2.277%); 4634/221319 objects misplaced (2.094%); 3.6 MiB/s, 58 objects/s recovering 2024-08-17T17:20:15.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:15.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:15.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:15.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:15.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:15.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:15.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:15.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:15.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:15.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:15.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.815 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:15.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 5040/221319 objects degraded (2.277%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 5040/221319 objects degraded (2.277%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:15.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:15.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:15.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:16.018 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 5040/221319 objects degraded (2.277%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:20:16.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:16.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:16.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:16.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:16.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:16.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:20:16.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:16.356 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:16.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:16.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:16.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:16.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:16.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:16.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:16.730 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:16.730 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:16.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:16.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:20:16.872 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T17:20:16.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:16.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:17.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:17.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:17.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:16 smithi142 ceph-mon[106729]: pgmap v613: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 5040/218874 objects degraded (2.303%); 4634/218874 objects misplaced (2.117%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:20:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:16 smithi142 ceph-mon[103614]: pgmap v613: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 5040/218874 objects degraded (2.303%); 4634/218874 objects misplaced (2.117%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:20:17.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:17.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:17.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:17.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:17.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:16 smithi152 ceph-mon[93456]: pgmap v613: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 5040/218874 objects degraded (2.303%); 4634/218874 objects misplaced (2.117%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:20:17.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:17.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:17.438 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:17.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:20:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:17.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:17.481 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:20:17.482 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:17.513 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:20:17.513 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:17.537 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:20:17.537 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:17.564 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:20:17.564 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:17.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:17.594 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:20:17.594 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:17.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:17.629 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:20:17.630 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:17.660 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:20:17.660 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:17.687 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:20:17.687 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:17.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:17.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:17.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:17.891 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:17.905 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:17.925 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:18.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:18.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:18.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2724510309' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:18.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2724510309' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:18.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:18.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:18.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:18.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2724510309' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:18.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:18.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:18.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:18.485 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:18.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:18.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:18.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:18.720 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:18.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:18.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:18.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:18.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:19.020 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:19.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:19.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:19.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:18 smithi142 ceph-mon[106729]: pgmap v614: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 5040/210972 objects degraded (2.389%); 4634/210972 objects misplaced (2.197%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:20:19.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:18 smithi142 ceph-mon[103614]: pgmap v614: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 5040/210972 objects degraded (2.389%); 4634/210972 objects misplaced (2.197%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:20:19.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:19.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:19.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:19.291 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:19.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:18 smithi152 ceph-mon[93456]: pgmap v614: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 5040/210972 objects degraded (2.389%); 4634/210972 objects misplaced (2.197%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:20:19.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:19.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:19.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:19.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:19.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:19.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:19.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:19.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:19.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:20.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:20.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:20.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:20.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:20.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:20.368 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:20.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:20.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:20.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:20:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:20:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:20:20.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:20.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:20.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:20.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.855 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:20:20.871 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:21.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:21.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:21.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:20 smithi142 ceph-mon[106729]: pgmap v615: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 4796/204945 objects degraded (2.340%); 4336/204945 objects misplaced (2.116%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:20:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4796/204945 objects degraded (2.340%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:20:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/26286029' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:20 smithi142 ceph-mon[103614]: pgmap v615: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 4796/204945 objects degraded (2.340%); 4336/204945 objects misplaced (2.116%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:20:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4796/204945 objects degraded (2.340%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:20:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/26286029' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:21.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:21.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:21.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:21.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:21.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:21.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:21.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:21.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:20 smithi152 ceph-mon[93456]: pgmap v615: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 4796/204945 objects degraded (2.340%); 4336/204945 objects misplaced (2.116%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:20:21.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4796/204945 objects degraded (2.340%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:20:21.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/26286029' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:21.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:21.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:21.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:21.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:21.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:21.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:21.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:21.775 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:21.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:21.849 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:20:21] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.51.0" 2024-08-17T17:20:21.873 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:20:21.873 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:20:21.873 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:20:21.873 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2024-08-17T17:20:21.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 0 2024-08-17T17:20:21.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:21.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:22.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:22.029 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:21 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:20:22.029 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/26286029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:22.029 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:21 smithi152 ceph-mon[93456]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T17:20:22.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:22.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:20:22.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/26286029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:22.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-mon[106729]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T17:20:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:20:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/26286029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:21 smithi142 ceph-mon[103614]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T17:20:22.146 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:22.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:22.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:22.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:22.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:22.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:22.477 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:200418 9.13 e184/174 9:ce73755e:::benchmark_data_smithi142_157452_object68297:head v 184'8299, mlcod=162'4095)", 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:22.516375+0000", 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0057426329999999996, 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057764080000000002, 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.516375+0000", 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.516375+0000", 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.516374+0000", 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.516384+0000", 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7820000000000005e-06 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.516399+0000", 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5506e-05 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.516407+0000", 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3200000000000002e-06 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.519250+0000", 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028431730000000001 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.519308+0000", 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8634000000000001e-05 2024-08-17T17:20:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:200426 9.12 9:4c7d34fe:::benchmark_data_smithi142_157452_object68305:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:22.520426+0000", 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0016919179999999999, 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017876459999999999, 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200426 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520426+0000", 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520426+0000", 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520424+0000", 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520432+0000", 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6450000000000005e-06 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520449+0000", 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6628000000000001e-05 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520455+0000", 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1029999999999998e-06 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520529+0000", 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4602000000000002e-05 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520672+0000", 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000142712 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520800+0000", 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012764100000000001 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:200427 9.a 9:539d8dc5:::benchmark_data_smithi142_157452_object68306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:22.520471+0000", 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001646242, 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001809473, 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200427 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520471+0000", 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520471+0000", 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520471+0000", 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520474+0000", 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7599999999999998e-06 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520482+0000", 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5869999999999993e-06 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.520486+0000", 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6820000000000001e-06 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521679+0000", 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001193487 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521785+0000", 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010534 2024-08-17T17:20:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521906+0000", 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012116399999999999 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:200430 9.17 9:e879c9d4:::benchmark_data_smithi142_157452_object68309:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:22.521457+0000", 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00066029900000000004, 2024-08-17T17:20:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000891766, 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200430 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521457+0000", 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521457+0000", 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521456+0000", 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521462+0000", 2024-08-17T17:20:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6470000000000003e-06 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521478+0000", 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5803000000000001e-05 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521484+0000", 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1380000000000004e-06 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.521554+0000", 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0314999999999993e-05 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.522324+0000", 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00076936799999999998 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:200417 9.f 9:f027f037:::benchmark_data_smithi142_157452_object68296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:20:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:22.505014+0000", 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.017103578000000001, 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017402977, 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200417 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.505014+0000", 2024-08-17T17:20:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.505014+0000", 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.505014+0000", 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.505016+0000", 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3070000000000001e-06 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.505021+0000", 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9819999999999999e-06 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.505024+0000", 2024-08-17T17:20:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9440000000000001e-06 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.512053+0000", 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070295189999999997 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.512197+0000", 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014337 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:20:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.512334+0000", 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013741099999999999 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.517970+0000", 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056354509999999997 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:22.519803+0000", 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018334359999999999 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:20:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 5 2024-08-17T17:20:22.549 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:22.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:22.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:22.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:22.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:22.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:22.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:22.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:22.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:23.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:22 smithi142 ceph-mon[106729]: pgmap v617: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4796/203799 objects degraded (2.353%); 4336/203799 objects misplaced (2.128%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:20:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:20:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:22 smithi142 ceph-mon[103614]: pgmap v617: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4796/203799 objects degraded (2.353%); 4336/203799 objects misplaced (2.128%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:20:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:20:23.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:23.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:23.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:23.283 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:22 smithi152 ceph-mon[93456]: pgmap v617: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4796/203799 objects degraded (2.353%); 4336/203799 objects misplaced (2.128%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:20:23.283 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:20:23.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:23.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:20:23.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:23.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:23.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:23.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:20:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:23.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:23.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:23.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:23.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:23.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:23.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:23.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:24.101 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:24.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:24.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:24.275 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:24.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:24.405 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:24.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:24.525 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:24.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:24.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:24.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:24.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:24.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:24.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:24.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:24.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:24 smithi152 ceph-mon[93456]: pgmap v618: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 4542/194295 objects degraded (2.338%); 4078/194295 objects misplaced (2.099%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:24.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:25.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.117 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:25.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:24 smithi142 ceph-mon[106729]: pgmap v618: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 4542/194295 objects degraded (2.338%); 4078/194295 objects misplaced (2.099%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:24 smithi142 ceph-mon[103614]: pgmap v618: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 4542/194295 objects degraded (2.338%); 4078/194295 objects misplaced (2.099%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:25.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:25.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:25.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.391 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:25.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.402 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:25.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:25.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:25.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:25.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:25.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:25.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:25.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:25.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:25.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.948 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:26.050 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:26.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:26.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:26.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4542/194295 objects degraded (2.338%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.272 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:26.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:26.322 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4542/194295 objects degraded (2.338%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:26.322 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4542/194295 objects degraded (2.338%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:26.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:26.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:26.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:26.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:26.564 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:26.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:26.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:26.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:26.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:26.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:26.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:26.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:26.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:26.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:27.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:27.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:27.182 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:26 smithi142 ceph-mon[103614]: pgmap v619: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 898 op/s; 4542/191907 objects degraded (2.367%); 4078/191907 objects misplaced (2.125%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:27.183 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:27.183 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:27.186 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:26 smithi142 ceph-mon[106729]: pgmap v619: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 898 op/s; 4542/191907 objects degraded (2.367%); 4078/191907 objects misplaced (2.125%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:27.186 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:27.186 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:27.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:26 smithi152 ceph-mon[93456]: pgmap v619: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 898 op/s; 4542/191907 objects degraded (2.367%); 4078/191907 objects misplaced (2.125%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:27.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:27.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:27.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:27.238 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:27.245 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:20:27.245 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:27.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:27.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:27.290 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:20:27.290 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:27.320 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:20:27.320 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:27.348 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:20:27.348 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:27.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:27.375 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:20:27.376 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:27.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:27.407 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:20:27.407 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:27.435 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:20:27.435 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:27.446 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:27.447 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:20:27.447 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:20:27.447 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:20:27.447 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:27.463 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:20:27.463 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:27.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:27.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:27.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:27.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:27.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:27.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:27.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:27.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:27.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:28.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2425874981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:28.117 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2425874981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.117 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2425874981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:28.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:28.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:28.383 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:28.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:28.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:28.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:20:28.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:28.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:28.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:28.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:28.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:28.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:28.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:29.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:29.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:28 smithi142 ceph-mon[106729]: pgmap v620: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 4542/184725 objects degraded (2.459%); 4078/184725 objects misplaced (2.208%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:28 smithi142 ceph-mon[103614]: pgmap v620: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 4542/184725 objects degraded (2.459%); 4078/184725 objects misplaced (2.208%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:29.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:29.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:29.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:28 smithi152 ceph-mon[93456]: pgmap v620: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 4542/184725 objects degraded (2.459%); 4078/184725 objects misplaced (2.208%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:20:29.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:29.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:29.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:29.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:29.557 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:29.590 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:29.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:29.692 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:29.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:29.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:29.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:29.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:29.907 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.5 primary-affinity to 0 (802) 2024-08-17T17:20:29.918 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:29.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:30.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:30.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:30.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4099217551' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-08-17T17:20:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-08-17T17:20:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4099217551' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-08-17T17:20:30.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-08-17T17:20:30.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:30.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:30.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:30.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4099217551' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-08-17T17:20:30.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-08-17T17:20:30.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:30.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:30.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:30.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:30.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:30.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:30.547 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:20:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:20:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:20:30.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:30.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:30.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:30.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:30.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:30.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:30.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:31.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:31.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[103614]: pgmap v621: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 4260/178860 objects degraded (2.382%); 3742/178860 objects misplaced (2.092%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:20:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-08-17T17:20:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[103614]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T17:20:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4260/178860 objects degraded (2.382%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[106729]: pgmap v621: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 4260/178860 objects degraded (2.382%); 3742/178860 objects misplaced (2.092%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:20:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-08-17T17:20:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[106729]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T17:20:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4260/178860 objects degraded (2.382%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:31.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:31.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:31.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:30 smithi152 ceph-mon[93456]: pgmap v621: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 4260/178860 objects degraded (2.382%); 3742/178860 objects misplaced (2.092%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:20:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-08-17T17:20:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:30 smithi152 ceph-mon[93456]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T17:20:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4260/178860 objects degraded (2.382%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:31.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:31.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:31.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:31.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:31.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:31.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:31.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:31.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:31.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:31.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:31.695 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:31.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:31.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:31.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:31.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:31.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:31.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:32.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:32.113 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:31 smithi142 ceph-mon[106729]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T17:20:32.113 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:31 smithi142 ceph-mon[103614]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T17:20:32.116 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:20:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:20:31] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.51.0" 2024-08-17T17:20:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.129 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:32.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:32.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:32.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:32.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:31 smithi152 ceph-mon[93456]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T17:20:32.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:32.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:32.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.544 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:32.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:32.572 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:32.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:32.688 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:32.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.836 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:32.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:32.934 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:20:32.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:32.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:33.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:33.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[106729]: pgmap v624: 99 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 93 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 4260/176580 objects degraded (2.413%); 3742/176580 objects misplaced (2.119%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:20:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:20:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2211185371' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[103614]: pgmap v624: 99 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 93 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 4260/176580 objects degraded (2.413%); 3742/176580 objects misplaced (2.119%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:20:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:20:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2211185371' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:33.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:33.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:33.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:32 smithi152 ceph-mon[93456]: pgmap v624: 99 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 93 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 4260/176580 objects degraded (2.413%); 3742/176580 objects misplaced (2.119%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:20:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:32 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:20:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2211185371' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:33.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:33.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:33.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:33.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:33.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:33.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:33.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:33.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:34.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:34.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.055 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:34.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:34.065 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:33 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:34.065 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:33 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:34.065 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:33 smithi142 ceph-mon[106729]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T17:20:34.066 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:33 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:34.066 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:33 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:34.066 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:33 smithi142 ceph-mon[103614]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T17:20:34.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:34.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:34.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:34.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:33 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:33 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:34.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:33 smithi152 ceph-mon[93456]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T17:20:34.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:34.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:34.426 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:34.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:34.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:34.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:34.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:34.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:34.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:34.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:34.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:34.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:35.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:35.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:35.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:34 smithi142 ceph-mon[106729]: pgmap v626: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 3870/171057 objects degraded (2.262%); 3502/171057 objects misplaced (2.047%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:20:35.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:34 smithi142 ceph-mon[103614]: pgmap v626: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 3870/171057 objects degraded (2.262%); 3502/171057 objects misplaced (2.047%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:20:35.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:35.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:35.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:35.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:34 smithi152 ceph-mon[93456]: pgmap v626: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 3870/171057 objects degraded (2.262%); 3502/171057 objects misplaced (2.047%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:20:35.342 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:35.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:35.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:35.493 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:20:35.493 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:35.512 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:20:35.512 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:35.546 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:20:35.546 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:35.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:35.579 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:20:35.579 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:35.608 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:20:35.608 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:35.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:35.640 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:20:35.640 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:35.666 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:20:35.667 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:35.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:20:35.704 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:35.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:35.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:35.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:35.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:35.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:35.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:36.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:36.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:20:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3870/171057 objects degraded (2.262%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2959289982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3870/171057 objects degraded (2.262%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2959289982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:36.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:36.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:36.274 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:36.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:36.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:36.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3870/171057 objects degraded (2.262%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2959289982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:36.398 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:36.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:36.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:36.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.607 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:36.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:36.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:36.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:36.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:36.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:36.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:36.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.010 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:37.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:37.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:37.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:37.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:37.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:37.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:37.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:36 smithi152 ceph-mon[93456]: pgmap v627: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 3870/168021 objects degraded (2.303%); 3502/168021 objects misplaced (2.084%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:20:37.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:36 smithi142 ceph-mon[103614]: pgmap v627: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 3870/168021 objects degraded (2.303%); 3502/168021 objects misplaced (2.084%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:20:37.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:36 smithi142 ceph-mon[106729]: pgmap v627: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 3870/168021 objects degraded (2.303%); 3502/168021 objects misplaced (2.084%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T17:20:37.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:37.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:37.474 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:20:37.474 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:20:37.474 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-17T17:20:37.474 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:20:37.474 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:20:37.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-17T17:20:37.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:37.594 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:37.594 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:20:37.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:20:37.595 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:20:37.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.595 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45389.0:76699 8.8 8:1a0757d5:::benchmark_data_smithi142_139631_object76698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:20:37.595 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:12:39.354311+0000", 2024-08-17T17:20:37.595 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 478.23652536100002, 2024-08-17T17:20:37.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.353368014, 2024-08-17T17:20:37.596 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.596 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:37.596 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.596 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45389", 2024-08-17T17:20:37.596 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:20:37.597 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76699 2024-08-17T17:20:37.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.597 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354311+0000", 2024-08-17T17:20:37.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354311+0000", 2024-08-17T17:20:37.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354310+0000", 2024-08-17T17:20:37.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354377+0000", 2024-08-17T17:20:37.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7178e-05 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354385+0000", 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8830000000000006e-06 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354389+0000", 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6370000000000001e-06 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354489+0000", 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9598000000000004e-05 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.354497+0000", 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6400000000000003e-06 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.328242+0000", 2024-08-17T17:20:37.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97374512000000002 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.328253+0000", 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0363000000000001e-05 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.688861+0000", 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3606085910000001 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.688975+0000", 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113804 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.689082+0000", 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106543 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.695532+0000", 2024-08-17T17:20:37.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064502819999999999 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.697226+0000", 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001693703 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705737+0000", 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085115750000000004 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:20:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705773+0000", 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6091999999999999e-05 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.707679+0000", 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001905976 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45389.0:76726 8.8 8:16520044:::benchmark_data_smithi142_139631_object76725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:20:37.606 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:12:39.380353+0000", 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 478.21048346399999, 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3284583159999999, 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45389", 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76726 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380353+0000", 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380353+0000", 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380352+0000", 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380399+0000", 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7360000000000001e-05 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380410+0000", 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0569e-05 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380416+0000", 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0090000000000004e-06 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380468+0000", 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2253999999999998e-05 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:39.380474+0000", 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9689999999999999e-06 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.328271+0000", 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94779743400000005 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.328278+0000", 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.348e-06 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.691432+0000", 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3631537229999999 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.691531+0000", 2024-08-17T17:20:37.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8975999999999996e-05 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.691631+0000", 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100814 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.697259+0000", 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056281600000000001 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.697264+0000", 2024-08-17T17:20:37.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.391e-06 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.707707+0000", 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010442922 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.707748+0000", 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0847999999999999e-05 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.708811+0000", 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001063755 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45389.0:76676 8.0 8:00c4672a:::benchmark_data_smithi142_139631_object76675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:12:40.324770+0000", 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 477.266067029, 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.362250263, 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.615 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45389", 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76676 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.324770+0000", 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.324770+0000", 2024-08-17T17:20:37.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.324768+0000", 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.324963+0000", 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019545599999999999 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.324979+0000", 2024-08-17T17:20:37.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5798999999999999e-05 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.324988+0000", 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0710000000000001e-06 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325056+0000", 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7603999999999999e-05 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op must wait for map", 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325067+0000", 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0998e-05 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.329820+0000", 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047529210000000002 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.329832+0000", 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2011000000000001e-05 2024-08-17T17:20:37.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.686941+0000", 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3571087509999999 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.687020+0000", 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9215000000000006e-05 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45389.0:76677 8.0 8:0c4c582f:::benchmark_data_smithi142_139631_object76676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:12:40.325064+0000", 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 477.26577247400002, 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3774320709999999, 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45389", 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76677 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325064+0000", 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325064+0000", 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325063+0000", 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325220+0000", 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015700800000000001 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325232+0000", 2024-08-17T17:20:37.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2102e-05 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325238+0000", 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1700000000000002e-06 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325343+0000", 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104448 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:20:37.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325353+0000", 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8949999999999993e-06 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.330301+0000", 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049482959999999996 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.330313+0000", 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2019e-05 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.687053+0000", 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.356739683 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.687803+0000", 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074976299999999997 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.702426+0000", 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014623834 2024-08-17T17:20:37.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.702496+0000", 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9713000000000002e-05 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:37.628 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45389.0:76681 8.0 8:071b0349:::benchmark_data_smithi142_139631_object76680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:12:40.325306+0000", 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 477.26553065799999, 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.380381544, 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45389", 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76681 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325306+0000", 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325306+0000", 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325305+0000", 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325386+0000", 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0733000000000001e-05 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325398+0000", 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1891e-05 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325403+0000", 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7080000000000002e-06 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325443+0000", 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9239999999999997e-05 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325451+0000", 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1159999999999999e-06 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.330360+0000", 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049093030000000003 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.330370+0000", 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0256e-05 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.687884+0000", 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.35751318 2024-08-17T17:20:37.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.688824+0000", 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00094082400000000004 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705578+0000", 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016753502999999999 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705687+0000", 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109552 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45389.0:76690 8.0 8:030d8608:::benchmark_data_smithi142_139631_object76689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:12:40.325417+0000", 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 477.26541943199999, 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3822933989999999, 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.635 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45389", 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76690 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325417+0000", 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325417+0000", 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325416+0000", 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325481+0000", 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4398000000000002e-05 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325490+0000", 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9360000000000005e-06 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325495+0000", 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2680000000000002e-06 2024-08-17T17:20:37.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325534+0000", 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8866999999999999e-05 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.325541+0000", 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5009999999999998e-06 2024-08-17T17:20:37.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.330417+0000", 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048754310000000004 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:40.330576+0000", 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015959000000000001 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.688883+0000", 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3583070990000001 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.689008+0000", 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012448099999999999 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.689192+0000", 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000184284 2024-08-17T17:20:37.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.697220+0000", 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0080274470000000001 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705711+0000", 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084917970000000006 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:37.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705866+0000", 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015403899999999999 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.705927+0000", 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1898999999999998e-05 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:12:41.707711+0000", 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001783071 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.643 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:75908 9.12 9.288ed0d2 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.639876+0000", 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.95096069900001, 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.77264730500000001, 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75908 2024-08-17T17:20:37.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.639876+0000", 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.639876+0000", 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.639875+0000", 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.639966+0000", 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1095999999999997e-05 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.639978+0000", 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1977999999999999e-05 2024-08-17T17:20:37.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.639984+0000", 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4640000000000003e-06 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.640946+0000", 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096200400000000003 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.640952+0000", 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5650000000000001e-06 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412516+0000", 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.77156368200000003 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412523+0000", 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4510000000000003e-06 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.648 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:75941 9.12 9.9ef5bf32 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.659095+0000", 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.931741808, 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75345445700000002, 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.649 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75941 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.659095+0000", 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.659095+0000", 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.659093+0000", 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.659161+0000", 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8350999999999997e-05 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.659175+0000", 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4062e-05 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.659181+0000", 2024-08-17T17:20:37.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9660000000000001e-06 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.661400+0000", 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022188730000000001 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.661407+0000", 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7850000000000003e-06 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412544+0000", 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75113694499999994 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412549+0000", 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.541e-06 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:75948 9.12 9.93c4ecd2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.662955+0000", 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.927881388, 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74961652700000003, 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75948 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.662955+0000", 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.662955+0000", 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.662954+0000", 2024-08-17T17:20:37.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.663103+0000", 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148336 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.663116+0000", 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.362e-05 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.663120+0000", 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1300000000000003e-06 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.663194+0000", 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4003000000000003e-05 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.663202+0000", 2024-08-17T17:20:37.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9270000000000005e-06 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412566+0000", 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74936393700000004 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412572+0000", 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4149999999999998e-06 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:75968 9.12 9.12e08d32 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.677683+0000", 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.913153413, 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.73491056099999996, 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75968 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.660 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.677683+0000", 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.677683+0000", 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.677683+0000", 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:37.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.677712+0000", 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9067000000000001e-05 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.677718+0000", 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4540000000000004e-06 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.677722+0000", 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5769999999999998e-06 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.683309+0000", 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055876820000000001 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.683320+0000", 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.023e-05 2024-08-17T17:20:37.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412589+0000", 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72926898900000003 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412594+0000", 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0690000000000001e-06 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.664 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:75978 9.12 9.ceb75452 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.684589+0000", 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.906247463, 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72802793099999996, 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.665 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75978 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684589+0000", 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684589+0000", 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684588+0000", 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684654+0000", 2024-08-17T17:20:37.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5569999999999997e-05 2024-08-17T17:20:37.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684667+0000", 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3356000000000001e-05 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684673+0000", 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4289999999999998e-06 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684713+0000", 2024-08-17T17:20:37.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9439e-05 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.684719+0000", 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8699999999999997e-06 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412611+0000", 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72789198600000005 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412617+0000", 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3249999999999996e-06 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:75984 9.12 9.c9549eb2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.689478+0000", 2024-08-17T17:20:37.671 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.90135812899999, 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72319347199999995, 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75984 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689478+0000", 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689478+0000", 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689477+0000", 2024-08-17T17:20:37.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689540+0000", 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3356999999999997e-05 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689554+0000", 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3672000000000001e-05 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689560+0000", 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2990000000000002e-06 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689602+0000", 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1567999999999999e-05 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.689609+0000", 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.199e-06 2024-08-17T17:20:37.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412660+0000", 2024-08-17T17:20:37.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72305056700000003 2024-08-17T17:20:37.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412672+0000", 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2084e-05 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76020 9.12 9.bc100ff2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.711121+0000", 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.879715635, 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.70157785299999997, 2024-08-17T17:20:37.677 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76020 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711121+0000", 2024-08-17T17:20:37.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711121+0000", 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711120+0000", 2024-08-17T17:20:37.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711152+0000", 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1866999999999998e-05 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711162+0000", 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.842e-06 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711167+0000", 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.792e-06 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711562+0000", 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039573800000000003 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.711568+0000", 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2839999999999997e-06 2024-08-17T17:20:37.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412693+0000", 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.70112484600000002 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412699+0000", 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.246e-06 2024-08-17T17:20:37.682 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76027 9.12 9.6a723dd2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.714796+0000", 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.876040432, 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.69793014799999997, 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76027 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714796+0000", 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714796+0000", 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714795+0000", 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714877+0000", 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1557000000000005e-05 2024-08-17T17:20:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714898+0000", 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1069000000000001e-05 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714904+0000", 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5320000000000003e-06 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714950+0000", 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5651e-05 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.714956+0000", 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2319999999999998e-06 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412719+0000", 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.69776325900000002 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412726+0000", 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8480000000000003e-06 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76059 9.12 9.57eff012 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.740273+0000", 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.850563771, 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.67246973300000001, 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.688 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76059 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740273+0000", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740273+0000", 2024-08-17T17:20:37.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740271+0000", 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740319+0000", 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8191e-05 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740333+0000", 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3701e-05 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740339+0000", 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740384+0000", 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5321000000000002e-05 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.740392+0000", 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0489999999999998e-06 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412738+0000", 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.67234677899999995 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412743+0000", 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1969999999999998e-06 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76082 9.12 9.f8fc872 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.756532+0000", 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.834304041, 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.65622384300000003, 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.693 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76082 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756532+0000", 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756532+0000", 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756532+0000", 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756611+0000", 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9536999999999994e-05 2024-08-17T17:20:37.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756622+0000", 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0963e-05 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756627+0000", 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8690000000000003e-06 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756672+0000", 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4218000000000003e-05 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.756678+0000", 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4729999999999997e-06 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412753+0000", 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.656075305 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412756+0000", 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0240000000000002e-06 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76112 9.12 9.7ada1dd2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.781777+0000", 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.809059883, 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.63099390899999996, 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.698 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76112 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781777+0000", 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781777+0000", 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781776+0000", 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781808+0000", 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2203000000000003e-05 2024-08-17T17:20:37.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781816+0000", 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7880000000000007e-06 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781821+0000", 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7099999999999998e-06 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781870+0000", 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9339000000000003e-05 2024-08-17T17:20:37.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.781876+0000", 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4489999999999998e-06 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412768+0000", 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.63089195600000003 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412771+0000", 2024-08-17T17:20:37.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.033e-06 2024-08-17T17:20:37.703 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.703 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.703 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76126 9.12 9.af317932 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.796464+0000", 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.79437296200001, 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.61632075600000003, 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76126 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796464+0000", 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796464+0000", 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796463+0000", 2024-08-17T17:20:37.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796523+0000", 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9762000000000001e-05 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796531+0000", 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2500000000000006e-06 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796535+0000", 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7589999999999999e-06 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796575+0000", 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0138000000000001e-05 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.796581+0000", 2024-08-17T17:20:37.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0499999999999997e-06 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412781+0000", 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.61620048900000002 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412784+0000", 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.01e-06 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.708 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76172 9.12 9.599fcf12 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.842733+0000", 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.748103265, 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.57006537899999998, 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76172 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.709 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842733+0000", 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842733+0000", 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842732+0000", 2024-08-17T17:20:37.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842805+0000", 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2627999999999996e-05 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842815+0000", 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0441e-05 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842821+0000", 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6489999999999996e-06 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842866+0000", 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5385e-05 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.842872+0000", 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9270000000000002e-06 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412795+0000", 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.56992256299999999 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412799+0000", 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.907e-06 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:76214 9.12 9.7419e352 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e167)", 2024-08-17T17:20:37.713 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:19.886652+0000", 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.704184959, 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.52616074099999999, 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 76214 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:37.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886652+0000", 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886652+0000", 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886644+0000", 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999924 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886687+0000", 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3135e-05 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886705+0000", 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8383e-05 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886712+0000", 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9879999999999998e-06 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886766+0000", 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3609000000000001e-05 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:19.886776+0000", 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0200999999999999e-05 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412809+0000", 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.52603286599999999 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:37.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:20.412812+0000", 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.298e-06 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:37.718 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:37.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:37.785 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:37.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:37.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:38.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:38.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:38.098 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:38.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:38.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:38.215 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:37 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:20:38.215 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:37 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:20:38.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:38.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:38 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:20:38.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:38.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:38.495 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:38.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:38.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:38.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:38.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:38.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:38.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:38.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:38.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:38.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:39.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:20:39.074 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:39 smithi142 ceph-mon[106729]: pgmap v628: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3870/162117 objects degraded (2.387%); 3502/162117 objects misplaced (2.160%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:20:39.074 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:39 smithi142 ceph-mon[103614]: pgmap v628: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3870/162117 objects degraded (2.387%); 3502/162117 objects misplaced (2.160%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:20:39.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:39.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:39.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:39.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:39.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:39.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:39.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:39 smithi152 ceph-mon[93456]: pgmap v628: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3870/162117 objects degraded (2.387%); 3502/162117 objects misplaced (2.160%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:20:39.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:39.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:39.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:39.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:39.589 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:39.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:39.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:39.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:39.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:39.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:39.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:39.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:40.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:40.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.147 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:40.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:40.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:40.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:40.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:40.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:40.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:20:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:20:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:20:40.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:40.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:40.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:40.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:40.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:41.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:41.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:41.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:41.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:41.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.312 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:41.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:41.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:41 smithi152 ceph-mon[93456]: pgmap v629: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 3724/154572 objects degraded (2.409%); 3194/154572 objects misplaced (2.066%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:41.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3724/154572 objects degraded (2.409%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:41.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:41.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:41.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:41.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:41 smithi142 ceph-mon[106729]: pgmap v629: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 3724/154572 objects degraded (2.409%); 3194/154572 objects misplaced (2.066%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3724/154572 objects degraded (2.409%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:41 smithi142 ceph-mon[103614]: pgmap v629: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 3724/154572 objects degraded (2.409%); 3194/154572 objects misplaced (2.066%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:20:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3724/154572 objects degraded (2.409%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:41.686 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:41.686 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:20:41.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:216651 9.b e187/154 9:d6b9135e:::benchmark_data_smithi142_157452_object84530:head v 187'13687, mlcod=187'13685)", 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:41.680707+0000", 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0048795990000000001, 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049116730000000001, 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680707+0000", 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680707+0000", 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680706+0000", 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680714+0000", 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6109999999999994e-06 2024-08-17T17:20:41.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680734+0000", 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.963e-05 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680742+0000", 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2470000000000008e-06 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680873+0000", 2024-08-17T17:20:41.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013068199999999999 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.680922+0000", 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8738e-05 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.690 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:216616 9.13 e187/174 9:ce83af59:::benchmark_data_smithi142_157452_object84495:head v 187'8817, mlcod=169'4586)", 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:41.684082+0000", 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001504079, 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015964390000000001, 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684082+0000", 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684082+0000", 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684081+0000", 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684089+0000", 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0209999999999999e-06 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684107+0000", 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8145e-05 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:41.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684114+0000", 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7669999999999998e-06 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684246+0000", 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000131996 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:41.684295+0000", 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9199000000000001e-05 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:41.694 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:41.695 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:41.695 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:41.695 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:20:41.695 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 2 2024-08-17T17:20:41.695 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:41.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:41.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:41.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:41.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:41.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:41.981 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:20:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:20:41] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-08-17T17:20:42.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:42.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:42.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:42.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:42.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:42.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:42.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:42.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:42.396 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:42.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:42.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:20:42.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:42.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:42.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[106729]: pgmap v630: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 3724/151377 objects degraded (2.460%); 3194/151377 objects misplaced (2.110%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:20:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[103614]: pgmap v630: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 3724/151377 objects degraded (2.460%); 3194/151377 objects misplaced (2.110%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:20:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.689 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:20:42.689 INFO:teuthology.orchestra.run.smithi142.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:20:42.689 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:20:42.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:42.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:42.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:42 smithi152 ceph-mon[93456]: pgmap v630: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 3724/151377 objects degraded (2.460%); 3194/151377 objects misplaced (2.110%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:20:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:42.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:42.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:42.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:42.955 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:43.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:43.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:43.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:43.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:43.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:43.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:43.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:43.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:43.451 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:20:43.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:43.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:43.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3413850236' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:43.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:43.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:43.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:43.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3413850236' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:43.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:43 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:43.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:43.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:20:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3413850236' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:43 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:43.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:43.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:43.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:44.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:44.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:44.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:44.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:44.262 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:44.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:44.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:44.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:44.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:44.504 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:44.504 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:44.504 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[103614]: pgmap v631: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 3463/144624 objects degraded (2.394%); 2836/144624 objects misplaced (1.961%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:20:44.505 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[103614]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T17:20:44.505 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:44.505 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:44.505 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[106729]: pgmap v631: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 3463/144624 objects degraded (2.394%); 2836/144624 objects misplaced (1.961%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:20:44.505 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:44 smithi142 ceph-mon[106729]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T17:20:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:44 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:44 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:44 smithi152 ceph-mon[93456]: pgmap v631: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 3463/144624 objects degraded (2.394%); 2836/144624 objects misplaced (1.961%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:20:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:44 smithi152 ceph-mon[93456]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T17:20:44.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:44.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:44.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:44.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:44.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:44.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:45.007 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:45.008 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:20:45.008 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:20:45.008 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:20:45.008 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:45.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:45.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:45.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:45.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:45.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:45.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:45.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:45.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:45.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:45.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:45.594 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:45.669 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3463/144624 objects degraded (2.394%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:45.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:45.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:20:45.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:45.836 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3463/144624 objects degraded (2.394%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:45.836 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3463/144624 objects degraded (2.394%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:45.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:45.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:45.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:45.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.046 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:46.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.210 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:46.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:46.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:46.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:46.351 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:20:46.352 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:46.384 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:20:46.384 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:46.415 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:20:46.415 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:46.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.463 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:20:46.463 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:46.495 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:20:46.496 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:46.526 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:20:46.527 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:46.552 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:20:46.552 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:46.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:46.586 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:20:46.587 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.594 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:46.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:46.682 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:46 smithi152 ceph-mon[93456]: pgmap v633: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 3463/141264 objects degraded (2.451%); 2836/141264 objects misplaced (2.008%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:20:46.682 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1265341084' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:46.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:46.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:46.821 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:46.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:46 smithi142 ceph-mon[106729]: pgmap v633: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 3463/141264 objects degraded (2.451%); 2836/141264 objects misplaced (2.008%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:20:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1265341084' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:46 smithi142 ceph-mon[103614]: pgmap v633: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 3463/141264 objects degraded (2.451%); 2836/141264 objects misplaced (2.008%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:20:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1265341084' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:46.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:46.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:47.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:47.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:47.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:20:47.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:47.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:47.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:47.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:47.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:47.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:47.582 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:47.624 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:47.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:47.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:47.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:47.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:47.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:48.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:48.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:48.142 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:48.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:48.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:48.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:48.416 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:48.518 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:48.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:48.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:48.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:48.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:48.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:48 smithi152 ceph-mon[93456]: pgmap v634: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 3463/135552 objects degraded (2.555%); 2836/135552 objects misplaced (2.092%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:20:48.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:48 smithi142 ceph-mon[106729]: pgmap v634: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 3463/135552 objects degraded (2.555%); 2836/135552 objects misplaced (2.092%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:20:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:48 smithi142 ceph-mon[103614]: pgmap v634: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 3463/135552 objects degraded (2.555%); 2836/135552 objects misplaced (2.092%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:20:48.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:48.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:48.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:49.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:49.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:49.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:49.197 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:49.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:49.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:49.335 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:20:49.335 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:20:49.335 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-17T17:20:49.335 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:20:49.335 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:20:49.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-17T17:20:49.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:49.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:49.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:49.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:49.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:49.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:49.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:49.768 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:49.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:49.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:49.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:49.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:50.041 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:50.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:50.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:50.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:50.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:50.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:50.467 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:20:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:20:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:20:50.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:50.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:50.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:50.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:50.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:50 smithi152 ceph-mon[93456]: pgmap v635: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3164/128085 objects degraded (2.470%); 2382/128085 objects misplaced (1.860%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:20:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3164/128085 objects degraded (2.470%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:50.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:50 smithi142 ceph-mon[106729]: pgmap v635: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3164/128085 objects degraded (2.470%); 2382/128085 objects misplaced (1.860%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:20:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3164/128085 objects degraded (2.470%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:50 smithi142 ceph-mon[103614]: pgmap v635: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3164/128085 objects degraded (2.470%); 2382/128085 objects misplaced (1.860%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:20:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3164/128085 objects degraded (2.470%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:50.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:50.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:50.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:51.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:51.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:51.196 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:51.297 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:51.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:51.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:51.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:51.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:51.551 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:51.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:51.588 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:51.588 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:51.618 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:51.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:51.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:51.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:51.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:51.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:51.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:52.018 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:52.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:52.084 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:20:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:20:51] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-08-17T17:20:52.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:52.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:52.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:52.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:52.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:52.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:52.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:52.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:52.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:52.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:52.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:52.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:52 smithi142 ceph-mon[106729]: pgmap v636: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 3164/124818 objects degraded (2.535%); 2382/124818 objects misplaced (1.908%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:20:52.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:52 smithi142 ceph-mon[103614]: pgmap v636: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 3164/124818 objects degraded (2.535%); 2382/124818 objects misplaced (1.908%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:20:52.776 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:52.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:52.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:52.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:52 smithi152 ceph-mon[93456]: pgmap v636: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 3164/124818 objects degraded (2.535%); 2382/124818 objects misplaced (1.908%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:20:52.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:52.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:53.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:53.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:53.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:53.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:53.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:53.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:53.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:53.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:53.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:53.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:53.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:53.530 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:20:53.590 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/466042910' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:53.590 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/466042910' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:53.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:53.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:53.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:53.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/466042910' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:53.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:53.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:53.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:54.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:54.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:54.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:54.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:54.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:54.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:20:54.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:54.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:54.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:54.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:54.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:54.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:54.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:54 smithi152 ceph-mon[93456]: pgmap v637: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 2870/118548 objects degraded (2.421%); 2022/118548 objects misplaced (1.706%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:54 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/466042910' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:54 smithi152 ceph-mon[93456]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T17:20:54.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[103614]: pgmap v637: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 2870/118548 objects degraded (2.421%); 2022/118548 objects misplaced (1.706%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/466042910' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:54.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[103614]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T17:20:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.875 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:54.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:54.877 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[106729]: pgmap v637: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 2870/118548 objects degraded (2.421%); 2022/118548 objects misplaced (1.706%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:54.877 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:54.877 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/466042910' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:54.877 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:54 smithi142 ceph-mon[106729]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T17:20:54.976 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:55.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:55.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.162 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:55.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.225 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:55.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:55.314 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.389 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:20:55.389 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:55.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.421 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:20:55.421 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:55.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:55.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.464 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:20:55.464 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:55.494 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:20:55.494 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:55.522 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:20:55.522 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:55.549 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:20:55.549 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:55.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:55.581 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:20:55.581 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:55.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:55.608 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:20:55.609 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:55.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:55.713 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/593176983' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:55.713 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2870/118548 objects degraded (2.421%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:55.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:55.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:55.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:55.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/593176983' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2870/118548 objects degraded (2.421%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/593176983' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2870/118548 objects degraded (2.421%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:55.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:55.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:55.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:56.049 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:56.049 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:20:56.049 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:56.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:56.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:56.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:56.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:20:56.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:56.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:56.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:56.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:56.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:56 smithi142 ceph-mon[103614]: pgmap v639: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 2870/115425 objects degraded (2.486%); 2022/115425 objects misplaced (1.752%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:56 smithi142 ceph-mon[106729]: pgmap v639: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 2870/115425 objects degraded (2.486%); 2022/115425 objects misplaced (1.752%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:56.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:56.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:56.713 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:56 smithi152 ceph-mon[93456]: pgmap v639: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 2870/115425 objects degraded (2.486%); 2022/115425 objects misplaced (1.752%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:56.713 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:56.722 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:56.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:56.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:56.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:56.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:57.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:57.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:57.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:57.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:57.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:57.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:57.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:57.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:57.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:57.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:57.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:57.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:57.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:57.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:57.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:57.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:57.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:58.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:58.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:58.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:58.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:58.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:58.328 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:231308 9.c e189/154 9:38e118ed:::benchmark_data_smithi142_157452_object99187:head v 189'14430, mlcod=189'14428)", 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.314477+0000", 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.011599483000000001, 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011630412, 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314477+0000", 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314477+0000", 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314476+0000", 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314482+0000", 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7470000000000002e-06 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314493+0000", 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0954e-05 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314499+0000", 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4820000000000002e-06 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321030+0000", 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065314860000000004 2024-08-17T17:20:58.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321048+0000", 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7918e-05 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231305 9.d 9:b998d4a0:::benchmark_data_smithi142_157452_object99184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.314524+0000", 2024-08-17T17:20:58.333 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.011552054000000001, 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011648481, 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231305 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314524+0000", 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314524+0000", 2024-08-17T17:20:58.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314523+0000", 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314530+0000", 2024-08-17T17:20:58.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4189999999999999e-06 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314538+0000", 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6959999999999994e-06 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314543+0000", 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5639999999999998e-06 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314770+0000", 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000226987 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314882+0000", 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112168 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314988+0000", 2024-08-17T17:20:58.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105469 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231306 9.e 9:7ceb8c2a:::benchmark_data_smithi142_157452_object99185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.314554+0000", 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.01152283, 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011694672999999999, 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:58.339 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231306 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314554+0000", 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314554+0000", 2024-08-17T17:20:58.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314553+0000", 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314556+0000", 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3970000000000001e-06 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314561+0000", 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3519999999999997e-06 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314670+0000", 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010866000000000001 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314991+0000", 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032131299999999999 2024-08-17T17:20:58.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.315096+0000", 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104861 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.315195+0000", 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9216999999999996e-05 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:58.345 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322737+0000", 2024-08-17T17:20:58.345 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0075415050000000004 2024-08-17T17:20:58.345 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.345 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.345 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231307 9.e 9:7df76507:::benchmark_data_smithi142_157452_object99186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.314683+0000", 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.01139341, 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011637364000000001, 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231307 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.346 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314683+0000", 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314683+0000", 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314683+0000", 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314687+0000", 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9210000000000002e-06 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314743+0000", 2024-08-17T17:20:58.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6477999999999997e-05 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314749+0000", 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8189999999999997e-06 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.318759+0000", 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004010212 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.318875+0000", 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115503 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.318985+0000", 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110309 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:231309 9.10 e189/171 9:0c951444:::benchmark_data_smithi142_157452_object99188:head v 189'9129, mlcod=189'9127)", 2024-08-17T17:20:58.350 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.314814+0000", 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.011262972, 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011580076999999999, 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314814+0000", 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314814+0000", 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314813+0000", 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314816+0000", 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8059999999999999e-06 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314821+0000", 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8380000000000001e-06 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.314824+0000", 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9979999999999999e-06 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.316980+0000", 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021555020000000001 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.316997+0000", 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7620000000000001e-05 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.45800.0:231300 9.15 e189/177)", 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.319940+0000", 2024-08-17T17:20:58.354 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.006136547, 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065059990000000002, 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319940+0000", 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319940+0000", 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319939+0000", 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319949+0000", 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0485999999999999e-05 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319961+0000", 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1644999999999999e-05 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319967+0000", 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4439999999999996e-06 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.357 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:231312 9.1 e189/160 9:87a65bf0:::benchmark_data_smithi142_157452_object99191:head v 189'9540, mlcod=189'9538)", 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.319974+0000", 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0061028810000000001, 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065167740000000004, 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319974+0000", 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319974+0000", 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319973+0000", 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319979+0000", 2024-08-17T17:20:58.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9540000000000001e-06 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319984+0000", 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1209999999999998e-06 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.319988+0000", 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.929e-06 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320104+0000", 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115739 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320129+0000", 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.442e-05 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.361 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231291 9.13 9:c8d5d97d:::benchmark_data_smithi142_157452_object99170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.362 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.295653+0000", 2024-08-17T17:20:58.362 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:58.363 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.030423474999999998, 2024-08-17T17:20:58.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030892337999999998, 2024-08-17T17:20:58.363 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.363 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:58.363 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231291 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.295653+0000", 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.295653+0000", 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.295652+0000", 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.295661+0000", 2024-08-17T17:20:58.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5439999999999999e-06 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.295673+0000", 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2330999999999999e-05 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.295679+0000", 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9379999999999997e-06 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.297885+0000", 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022062649999999998 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.299179+0000", 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012943200000000001 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310980+0000", 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011800278000000001 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.311098+0000", 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118675 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.311706+0000", 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00060770099999999999 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:58.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.323726+0000", 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012019894999999999 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.324088+0000", 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036234599999999999 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.324101+0000", 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2621e-05 2024-08-17T17:20:58.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.324195+0000", 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4290999999999996e-05 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231314 9.d 9.1ad4365d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.320196+0000", 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0058805589999999996, 2024-08-17T17:20:58.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064491820000000004, 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231314 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320196+0000", 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320196+0000", 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320195+0000", 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:58.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320201+0000", 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.976e-06 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320212+0000", 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1534e-05 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.320218+0000", 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7960000000000001e-06 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.45800.0:231310 9.19 e189/160 9:9b4f9200:::benchmark_data_smithi142_157452_object99189:head v 189'9056, mlcod=189'9054)", 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.321461+0000", 2024-08-17T17:20:58.374 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0046158989999999997, 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052354480000000002, 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321461+0000", 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321461+0000", 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321460+0000", 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321466+0000", 2024-08-17T17:20:58.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5289999999999997e-06 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321479+0000", 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3095000000000001e-05 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321485+0000", 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6620000000000002e-06 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321543+0000", 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8112000000000002e-05 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.321563+0000", 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9947999999999999e-05 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.45800.0:231300 9.15 e189/177)", 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.322924+0000", 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0031525860000000002, 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038223469999999998, 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322924+0000", 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322924+0000", 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322911+0000", 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999866 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322929+0000", 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8915000000000001e-05 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322941+0000", 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1575e-05 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.322946+0000", 2024-08-17T17:20:58.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2299999999999999e-06 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231300 9.15 9:ae026a40:::benchmark_data_smithi142_157452_object99179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.306348+0000", 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.019728392000000001, 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020447584000000001, 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231300 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306348+0000", 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306348+0000", 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306348+0000", 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306350+0000", 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7889999999999999e-06 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306357+0000", 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.173e-06 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306360+0000", 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4189999999999998e-06 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.306902+0000", 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00054159200000000003 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.307022+0000", 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120652 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.308584+0000", 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015614240000000001 2024-08-17T17:20:58.386 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:231301 9.13 9:c9613c0b:::benchmark_data_smithi142_157452_object99180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:20:58.310132+0000", 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.015944916999999999, 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016737947, 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:20:58.387 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 231301 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310132+0000", 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310132+0000", 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310130+0000", 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310137+0000", 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9140000000000002e-06 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310149+0000", 2024-08-17T17:20:58.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2320999999999999e-05 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.310155+0000", 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8470000000000001e-06 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.315594+0000", 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005438427 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:20:58.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:20:58.317182+0000", 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015884440000000001 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 13 2024-08-17T17:20:58.391 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:20:58.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:58.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:58.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:58.670 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:58 smithi142 ceph-mon[103614]: pgmap v640: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 2870/109863 objects degraded (2.612%); 2022/109863 objects misplaced (1.840%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:58.671 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:58 smithi142 ceph-mon[106729]: pgmap v640: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 2870/109863 objects degraded (2.612%); 2022/109863 objects misplaced (1.840%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:58.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:58.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:58.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:58.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:58 smithi152 ceph-mon[93456]: pgmap v640: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 2870/109863 objects degraded (2.612%); 2022/109863 objects misplaced (1.840%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T17:20:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:58.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:58.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:59.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:59.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:59.137 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.147 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:59.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:59.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:20:59.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:59.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:59.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:59.554 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:20:59.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:59.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:59.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.708 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:20:59.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:20:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1896092141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:20:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1896092141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:20:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1896092141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:20:59.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:59.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:00.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:00.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:00.159 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:00.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:00.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:00.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:00.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:00.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:00.472 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:21:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:21:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:21:00.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:00.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:00.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:00.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:00.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:00.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:00 smithi152 ceph-mon[93456]: pgmap v641: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 2592/101592 objects degraded (2.551%); 1608/101592 objects misplaced (1.583%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:21:00.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:00 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:00.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1896092141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:00.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:00 smithi152 ceph-mon[93456]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T17:21:00.767 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2592/101592 objects degraded (2.551%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:21:00.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:00.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:00.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:00.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[103614]: pgmap v641: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 2592/101592 objects degraded (2.551%); 1608/101592 objects misplaced (1.583%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:21:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1896092141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[103614]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T17:21:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2592/101592 objects degraded (2.551%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:21:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[106729]: pgmap v641: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 2592/101592 objects degraded (2.551%); 1608/101592 objects misplaced (1.583%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:21:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:00.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1896092141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:00.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[106729]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T17:21:00.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2592/101592 objects degraded (2.551%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:21:00.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:00.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:00.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:01.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:01.098 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:01.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:01.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:01.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:01.364 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:01.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:01.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:01.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:01.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:01.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:01.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:01.757 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:21:01.757 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:21:01.757 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:21:01.758 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-17T17:21:01.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:21:01.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:01.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:01.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:01.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:01.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:02.037 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:21:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:21:01] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-08-17T17:21:02.052 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:01 smithi152 systemd[1]: Stopping Ceph osd.5 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:21:02.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:02.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:02.192 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:21:02.192 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:02.206 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:21:02.206 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:21:02.206 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:21:02.207 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:21:02.207 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:21:02.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:21:02.225 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:02.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:02.253 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:21:02.253 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:02.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:02.275 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:21:02.276 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:02.299 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:21:02.299 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:02.334 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:21:02.335 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:02.361 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:21:02.361 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:02.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:02.390 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:21:02.390 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:02.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:02.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:02.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:02.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:02.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:02.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:02.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:02 smithi152 ceph-mon[93456]: pgmap v643: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 2592/98238 objects degraded (2.638%); 1608/98238 objects misplaced (1.637%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:21:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/904735824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:02.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[46487]: 2024-08-17T17:21:02.597+0000 7f674238c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:21:02.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[46487]: 2024-08-17T17:21:02.597+0000 7f674238c640 -1 osd.5 190 *** Got signal Terminated *** 2024-08-17T17:21:02.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[46487]: 2024-08-17T17:21:02.597+0000 7f674238c640 -1 osd.5 190 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:21:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:02 smithi142 ceph-mon[103614]: pgmap v643: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 2592/98238 objects degraded (2.638%); 1608/98238 objects misplaced (1.637%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:21:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/904735824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:02.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:02 smithi142 ceph-mon[106729]: pgmap v643: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 2592/98238 objects degraded (2.638%); 1608/98238 objects misplaced (1.637%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:21:02.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/904735824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:02.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:02.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:03.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:03.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:03.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:21:03.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:03.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:03.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:03.357 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:03.459 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:03.489 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:03.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:03.532 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:03.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:21:03.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:03.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:03.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:03.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:03 smithi152 ceph-mon[93456]: osd.5 marked itself down and dead 2024-08-17T17:21:03.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:03.868 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:03.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:03.869 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:03 smithi142 ceph-mon[103614]: osd.5 marked itself down and dead 2024-08-17T17:21:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:03 smithi142 ceph-mon[106729]: osd.5 marked itself down and dead 2024-08-17T17:21:03.970 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:03.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:04.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:04.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:04.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:04.244 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:04.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:04.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:04.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:04.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:04.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:04.558 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:04 smithi152 podman[122928]: 2024-08-17 17:21:04.30637588 +0000 UTC m=+2.037510714 container died cac09b3f3a95bb6bb091f3c31334bebcca6dd0e22c2cd709cd14b65f5b6afb32 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:21:04.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:04.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:04.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:04.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:04 smithi152 ceph-mon[93456]: pgmap v644: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 2270/91476 objects degraded (2.482%); 1168/91476 objects misplaced (1.277%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:21:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:04 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:21:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:04 smithi152 ceph-mon[93456]: osdmap e191: 8 total, 7 up, 8 in 2024-08-17T17:21:04.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:04 smithi152 podman[122928]: 2024-08-17 17:21:04.710074073 +0000 UTC m=+2.441208916 container cleanup cac09b3f3a95bb6bb091f3c31334bebcca6dd0e22c2cd709cd14b65f5b6afb32 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:21:04.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:04 smithi152 bash[122928]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5 2024-08-17T17:21:04.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:04 smithi142 ceph-mon[103614]: pgmap v644: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 2270/91476 objects degraded (2.482%); 1168/91476 objects misplaced (1.277%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:21:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:04 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:21:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:04 smithi142 ceph-mon[103614]: osdmap e191: 8 total, 7 up, 8 in 2024-08-17T17:21:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:04 smithi142 ceph-mon[106729]: pgmap v644: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 2270/91476 objects degraded (2.482%); 1168/91476 objects misplaced (1.277%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:21:04.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:04 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:21:04.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:04 smithi142 ceph-mon[106729]: osdmap e191: 8 total, 7 up, 8 in 2024-08-17T17:21:04.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:04.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:05.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:05.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:05.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:05.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:05.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:05.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:05.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:05.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:05.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:05.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:05.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:05.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:05.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:05.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:05.862 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:05 smithi142 ceph-mon[103614]: osdmap e192: 8 total, 7 up, 8 in 2024-08-17T17:21:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2270/91476 objects degraded (2.482%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:21:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:05 smithi142 ceph-mon[106729]: osdmap e192: 8 total, 7 up, 8 in 2024-08-17T17:21:05.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2270/91476 objects degraded (2.482%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:21:05.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:05.994 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:05 smithi152 ceph-mon[93456]: osdmap e192: 8 total, 7 up, 8 in 2024-08-17T17:21:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2270/91476 objects degraded (2.482%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:21:06.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.166 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:06.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:06.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:06.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:06.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:06.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:06 smithi152 ceph-mon[93456]: pgmap v647: 99 pgs: 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2270/88197 objects degraded (2.574%); 1168/88197 objects misplaced (1.324%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:21:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:06 smithi152 ceph-mon[93456]: osdmap e193: 8 total, 7 up, 8 in 2024-08-17T17:21:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3871878905' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:06 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[103614]: pgmap v647: 99 pgs: 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2270/88197 objects degraded (2.574%); 1168/88197 objects misplaced (1.324%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:21:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[103614]: osdmap e193: 8 total, 7 up, 8 in 2024-08-17T17:21:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3871878905' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[106729]: pgmap v647: 99 pgs: 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2270/88197 objects degraded (2.574%); 1168/88197 objects misplaced (1.324%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:21:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[106729]: osdmap e193: 8 total, 7 up, 8 in 2024-08-17T17:21:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3871878905' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:06 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:06.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:06.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:07.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:07.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:07.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:07.176 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:21:07.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:07.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:07.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:07.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:07.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:07.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:07.609 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:07.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.746 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:07.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:07.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:07.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:07 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:07 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:07 smithi142 ceph-mon[103614]: osdmap e194: 8 total, 7 up, 8 in 2024-08-17T17:21:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:07 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:07 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:07.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:07 smithi142 ceph-mon[106729]: osdmap e194: 8 total, 7 up, 8 in 2024-08-17T17:21:07.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:08.039 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:07 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:08.039 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:07 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:08.039 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:07 smithi152 ceph-mon[93456]: osdmap e194: 8 total, 7 up, 8 in 2024-08-17T17:21:08.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:08.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:08.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:08.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:08.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:08.323 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:08 smithi152 podman[123136]: 2024-08-17 17:21:08.038176973 +0000 UTC m=+3.797183180 container remove cac09b3f3a95bb6bb091f3c31334bebcca6dd0e22c2cd709cd14b65f5b6afb32 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:21:08.323 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:08 smithi152 bash[123314]: Error: no container with ID cac09b3f3a95bb6bb091f3c31334bebcca6dd0e22c2cd709cd14b65f5b6afb32 found in database: no such container 2024-08-17T17:21:08.361 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:08.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:08.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:08.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:08.534 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:08.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:08.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:08.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:08.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:08.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:08.895 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:08 smithi152 ceph-mon[93456]: pgmap v650: 99 pgs: 6 active+undersized, 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 71 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 10922/84585 objects degraded (12.912%); 1168/84585 objects misplaced (1.381%) 2024-08-17T17:21:08.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:08.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:09.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.073 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:09.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:09.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:08 smithi142 ceph-mon[106729]: pgmap v650: 99 pgs: 6 active+undersized, 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 71 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 10922/84585 objects degraded (12.912%); 1168/84585 objects misplaced (1.381%) 2024-08-17T17:21:09.083 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:08 smithi142 ceph-mon[103614]: pgmap v650: 99 pgs: 6 active+undersized, 9 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 71 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 10922/84585 objects degraded (12.912%); 1168/84585 objects misplaced (1.381%) 2024-08-17T17:21:09.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:09.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:09.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:09.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:09.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:09.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:09.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:09.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:09.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:09.617 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:09.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.750 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:09.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:09.833 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:09.834 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:09.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:09.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:09.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:10.018 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:10.085 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:09 smithi152 podman[123610]: 2024-08-17 17:21:09.806575656 +0000 UTC m=+0.110196460 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:21:10.085 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:09 smithi152 podman[123610]: 2024-08-17 17:21:09.942286946 +0000 UTC m=+0.245907739 container create cc57ecb2e80285fa47f1edb3d417e2aa5600cffd72abf5dc3cc20e11ae691c5f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-08-17T17:21:10.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:10.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:10.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:10.299 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:10.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:10.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:10.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:10.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:21:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:21:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:21:10.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:10.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:10.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:10.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:10.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:10.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:10.907 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:10 smithi142 ceph-mon[106729]: pgmap v651: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 16874/79572 objects degraded (21.206%); 7.1 MiB/s, 116 objects/s recovering 2024-08-17T17:21:10.907 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16874/79572 objects degraded (21.206%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:21:10.908 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:10 smithi142 ceph-mon[103614]: pgmap v651: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 16874/79572 objects degraded (21.206%); 7.1 MiB/s, 116 objects/s recovering 2024-08-17T17:21:10.908 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16874/79572 objects degraded (21.206%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:21:10.926 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:10 smithi152 ceph-mon[93456]: pgmap v651: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 16874/79572 objects degraded (21.206%); 7.1 MiB/s, 116 objects/s recovering 2024-08-17T17:21:10.926 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16874/79572 objects degraded (21.206%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:21:10.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:10.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:10.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:10.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:21:11.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:11.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:11.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:11.227 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:11 smithi152 podman[123610]: 2024-08-17 17:21:11.119182675 +0000 UTC m=+1.422803481 container init cc57ecb2e80285fa47f1edb3d417e2aa5600cffd72abf5dc3cc20e11ae691c5f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD) 2024-08-17T17:21:11.227 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:11 smithi152 podman[123610]: 2024-08-17 17:21:11.126344218 +0000 UTC m=+1.429965009 container start cc57ecb2e80285fa47f1edb3d417e2aa5600cffd72abf5dc3cc20e11ae691c5f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:21:11.227 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:11 smithi152 podman[123610]: 2024-08-17 17:21:11.227378981 +0000 UTC m=+1.530999765 container attach cc57ecb2e80285fa47f1edb3d417e2aa5600cffd72abf5dc3cc20e11ae691c5f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-08-17T17:21:11.251 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:21:11.251 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:11.252 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-08-17T17:21:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:11.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:11.295 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:21:11.295 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:11.330 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:21:11.331 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:11.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:11.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:11.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:11.572 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:11 smithi152 podman[123610]: 2024-08-17 17:21:11.393473824 +0000 UTC m=+1.697094615 container died cc57ecb2e80285fa47f1edb3d417e2aa5600cffd72abf5dc3cc20e11ae691c5f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0) 2024-08-17T17:21:11.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:11.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:11.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:11.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:11.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:11.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:11.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:12.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:12.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2883403820' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:12.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5.service' 2024-08-17T17:21:12.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:21:11] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-08-17T17:21:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2883403820' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2883403820' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:12.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:12.161 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:21:12.161 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-17T17:21:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:21:12.183 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:21:12.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:12.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:12.318 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:12.343 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:12.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:12.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:12.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:12.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:12.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:12.730 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:12.749 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:12 smithi142 ceph-mon[106729]: pgmap v652: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 16114/75858 objects degraded (21.242%); 6.1 MiB/s, 99 objects/s recovering 2024-08-17T17:21:12.749 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:12 smithi142 ceph-mon[103614]: pgmap v652: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 16114/75858 objects degraded (21.242%); 6.1 MiB/s, 99 objects/s recovering 2024-08-17T17:21:12.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:12.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:12.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:13.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:13.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:12 smithi152 ceph-mon[93456]: pgmap v652: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 16114/75858 objects degraded (21.242%); 6.1 MiB/s, 99 objects/s recovering 2024-08-17T17:21:13.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:13.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:13.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:13.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:13.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:13.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:13.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:13.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:13.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:13.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:13.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:13.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:13.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:13.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:13.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:13.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:13.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:14.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:14.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:13 smithi152 ceph-mon[93456]: osdmap e195: 8 total, 7 up, 8 in 2024-08-17T17:21:14.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:13 smithi142 ceph-mon[106729]: osdmap e195: 8 total, 7 up, 8 in 2024-08-17T17:21:14.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:13 smithi142 ceph-mon[103614]: osdmap e195: 8 total, 7 up, 8 in 2024-08-17T17:21:14.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:14.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:14.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:14.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:21:14.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:14.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:14.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:14.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:14.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:14.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:14.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:14.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:14.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:14.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:14.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:14 smithi142 ceph-mon[106729]: pgmap v654: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 13424/68166 objects degraded (19.693%); 621/68166 objects misplaced (0.911%); 7.0 MiB/s, 142 objects/s recovering 2024-08-17T17:21:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:14 smithi142 ceph-mon[106729]: osdmap e196: 8 total, 7 up, 8 in 2024-08-17T17:21:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:14 smithi142 ceph-mon[103614]: pgmap v654: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 13424/68166 objects degraded (19.693%); 621/68166 objects misplaced (0.911%); 7.0 MiB/s, 142 objects/s recovering 2024-08-17T17:21:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:14 smithi142 ceph-mon[103614]: osdmap e196: 8 total, 7 up, 8 in 2024-08-17T17:21:14.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:14.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:14.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:15.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:14 smithi152 ceph-mon[93456]: pgmap v654: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 13424/68166 objects degraded (19.693%); 621/68166 objects misplaced (0.911%); 7.0 MiB/s, 142 objects/s recovering 2024-08-17T17:21:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:14 smithi152 ceph-mon[93456]: osdmap e196: 8 total, 7 up, 8 in 2024-08-17T17:21:15.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:15.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:15.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:15.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:15.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:15.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:15.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:15.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:15.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:15.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:15.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:15.791 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:15.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:15.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:15.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:15.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:16.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13424/68166 objects degraded (19.693%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:16.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13424/68166 objects degraded (19.693%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13424/68166 objects degraded (19.693%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:16.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:16.228 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:16.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:16.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:16.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:16.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:16.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:16.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:16.671 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:16.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:16.764 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:21:16.764 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:21:16.764 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:246699 9.8 e196/191 9:10c7478e:::benchmark_data_smithi142_157452_object114578:head v 196'9643, mlcod=196'9641)", 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:21:16.760642+0000", 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0027032269999999999, 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002742272, 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760642+0000", 2024-08-17T17:21:16.765 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760642+0000", 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760643+0000", 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7000000000000003e-07 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.766 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760650+0000", 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.37e-06 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760665+0000", 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5761999999999999e-05 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760672+0000", 2024-08-17T17:21:16.767 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9589999999999998e-06 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760791+0000", 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000118275 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760814+0000", 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3357e-05 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:21:16.768 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:246702 9.10 e196/171 9:0938fe34:::benchmark_data_smithi142_157452_object114581:head v 196'9591, mlcod=196'9588)", 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:21:16.760736+0000", 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0026090919999999999, 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027005789999999998, 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:21:16.769 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760736+0000", 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760736+0000", 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.770 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760737+0000", 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1600000000000001e-07 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760742+0000", 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5839999999999996e-06 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760750+0000", 2024-08-17T17:21:16.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4810000000000006e-06 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760759+0000", 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3880000000000006e-06 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760896+0000", 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000136363 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:21:16.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:21:16.760912+0000", 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6501e-05 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 2 2024-08-17T17:21:16.773 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:21:16.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:16.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:16.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:16.950 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:17.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:17.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:16 smithi152 ceph-mon[93456]: pgmap v656: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 12878/64794 objects degraded (19.875%); 621/64794 objects misplaced (0.958%); 6.9 MiB/s, 140 objects/s recovering 2024-08-17T17:21:17.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:16 smithi142 ceph-mon[106729]: pgmap v656: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 12878/64794 objects degraded (19.875%); 621/64794 objects misplaced (0.958%); 6.9 MiB/s, 140 objects/s recovering 2024-08-17T17:21:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:16 smithi142 ceph-mon[103614]: pgmap v656: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 12878/64794 objects degraded (19.875%); 621/64794 objects misplaced (0.958%); 6.9 MiB/s, 140 objects/s recovering 2024-08-17T17:21:17.162 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-08-17T17:21:17.162 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-08-17T17:21:17.162 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-08-17T17:21:17.162 DEBUG:teuthology.orchestra.run.smithi152:osd.5> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5.service 2024-08-17T17:21:17.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl start ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:21:17.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:17.223 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:17.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:17.364 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:17.432 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:17.466 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:17.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:17.552 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:17 smithi152 systemd[1]: Starting Ceph osd.5 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:21:17.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:17.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:17.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:17.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:17.814 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:17 smithi152 podman[124158]: 2024-08-17 17:21:17.455077012 +0000 UTC m=+0.023862889 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:21:17.814 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:17 smithi152 podman[124158]: 2024-08-17 17:21:17.5771421 +0000 UTC m=+0.145927968 container create a1a366c3206dd255ea4aac35506b546c500956e1537773f8d5e99b9c18f59bc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-08-17T17:21:17.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:17.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.923 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:17.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:18.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:18.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:18.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:17 smithi152 podman[124158]: 2024-08-17 17:21:17.813930496 +0000 UTC m=+0.382716377 container init a1a366c3206dd255ea4aac35506b546c500956e1537773f8d5e99b9c18f59bc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-08-17T17:21:18.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:17 smithi152 podman[124158]: 2024-08-17 17:21:17.82087646 +0000 UTC m=+0.389662335 container start a1a366c3206dd255ea4aac35506b546c500956e1537773f8d5e99b9c18f59bc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:21:18.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:17 smithi152 podman[124158]: 2024-08-17 17:21:17.922521625 +0000 UTC m=+0.491307500 container attach a1a366c3206dd255ea4aac35506b546c500956e1537773f8d5e99b9c18f59bc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-17T17:21:18.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:18.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:18.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:18.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:18.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:18.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:18.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:18.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-08-17T17:21:18.573 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate[124190]: --> ceph-volume raw activate successful for osd ID: 5 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 bash[124158]: --> ceph-volume raw activate successful for osd ID: 5 2024-08-17T17:21:18.574 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 podman[124158]: 2024-08-17 17:21:18.279314399 +0000 UTC m=+0.848100271 container died a1a366c3206dd255ea4aac35506b546c500956e1537773f8d5e99b9c18f59bc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-08-17T17:21:18.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:18.703 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:21:18.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:18.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:18.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:18.889 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:18.928 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-mon[93456]: pgmap v657: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 11481/58947 objects degraded (19.477%); 621/58947 objects misplaced (1.053%); 1.6 MiB/s, 54 objects/s recovering 2024-08-17T17:21:18.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2793668623' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:18.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:19.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:18 smithi142 ceph-mon[103614]: pgmap v657: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 11481/58947 objects degraded (19.477%); 621/58947 objects misplaced (1.053%); 1.6 MiB/s, 54 objects/s recovering 2024-08-17T17:21:19.013 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2793668623' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:19.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:18 smithi142 ceph-mon[106729]: pgmap v657: 99 pgs: 1 peering, 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 11481/58947 objects degraded (19.477%); 621/58947 objects misplaced (1.053%); 1.6 MiB/s, 54 objects/s recovering 2024-08-17T17:21:19.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2793668623' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:19.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:19.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:21:19.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:19.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:19.214 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:18 smithi152 podman[124356]: 2024-08-17 17:21:18.928547049 +0000 UTC m=+0.642575978 container remove a1a366c3206dd255ea4aac35506b546c500956e1537773f8d5e99b9c18f59bc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-08-17T17:21:19.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:19.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:19.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:19.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:19.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:19.474 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 podman[124414]: 2024-08-17 17:21:19.117786138 +0000 UTC m=+0.026062955 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:21:19.475 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 podman[124414]: 2024-08-17 17:21:19.229949272 +0000 UTC m=+0.138226079 container create 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-f2c39d4) 2024-08-17T17:21:19.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:19.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:19.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:21:19.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:19.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:19.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:19 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:19.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2793668623' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:19.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:19 smithi152 ceph-mon[93456]: osdmap e197: 8 total, 7 up, 8 in 2024-08-17T17:21:19.779 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 podman[124414]: 2024-08-17 17:21:19.474586861 +0000 UTC m=+0.382863670 container init 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T17:21:19.779 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 podman[124414]: 2024-08-17 17:21:19.480553073 +0000 UTC m=+0.388829877 container start 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:21:19.779 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 ceph-osd[124468]: -- 172.21.15.152:0/4194089949 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563832d541a0 con 0x563832d27000 2024-08-17T17:21:19.779 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 bash[124414]: 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 2024-08-17T17:21:19.779 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 systemd[1]: Started Ceph osd.5 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:21:19.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:19.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:19 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2793668623' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:19 smithi142 ceph-mon[103614]: osdmap e197: 8 total, 7 up, 8 in 2024-08-17T17:21:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:19 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:19.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2793668623' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:19.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:19 smithi142 ceph-mon[106729]: osdmap e197: 8 total, 7 up, 8 in 2024-08-17T17:21:19.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:20.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:20.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:20.030 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:20.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:19 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:21:19.777+0000 7f42cbfbc440 -1 Falling back to public interface 2024-08-17T17:21:20.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:20.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:20.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:20.274 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:20.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:20.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:20.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:20.472 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:21:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:21:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:21:20.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:20.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:20.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:20.697 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:20.798 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:20 smithi152 ceph-mon[93456]: pgmap v659: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 1.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 10124/50871 objects degraded (19.901%); 226/50871 objects misplaced (0.444%); 5.6 MiB/s, 122 objects/s recovering 2024-08-17T17:21:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10124/50871 objects degraded (19.901%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:20.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:20 smithi142 ceph-mon[103614]: pgmap v659: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 1.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 10124/50871 objects degraded (19.901%); 226/50871 objects misplaced (0.444%); 5.6 MiB/s, 122 objects/s recovering 2024-08-17T17:21:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10124/50871 objects degraded (19.901%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:20.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:20 smithi142 ceph-mon[106729]: pgmap v659: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 1.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 10124/50871 objects degraded (19.901%); 226/50871 objects misplaced (0.444%); 5.6 MiB/s, 122 objects/s recovering 2024-08-17T17:21:20.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10124/50871 objects degraded (19.901%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:20.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:20.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:21.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:21.100 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:21.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:21.236 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:21.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:21.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:21.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:21.485 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:21.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:21.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:21.619 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:21.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:21.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:21.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:21.683 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:21:21.684 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:21.720 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:21:21.720 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:21.750 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:21:21.750 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:21.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:21.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:21:21.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:21.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2263053850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:21.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:21.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:21:21.806 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:21.807 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-08-17T17:21:21.834 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:21:21.834 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:21.867 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:21:21.867 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:21.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2263053850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:21.868 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:21:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:21:21] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-08-17T17:21:21.868 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2263053850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:21.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:21.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:22.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:22.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:22.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:22.213 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:22.227 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2024-08-17T17:21:22.227 INFO:teuthology.orchestra.run.smithi152.stderr:0 2024-08-17T17:21:22.227 INFO:teuthology.orchestra.run.smithi152.stderr:1 2024-08-17T17:21:22.227 INFO:teuthology.orchestra.run.smithi152.stderr:2 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore bluefs device info [] 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:config diff 2024-08-17T17:21:22.228 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2024-08-17T17:21:22.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:22.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:22.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:22.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:22.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:22.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:22.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:22.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:22.871 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:22.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:22.998 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:22 smithi152 ceph-mon[93456]: pgmap v660: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 970 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 9260/46623 objects degraded (19.861%); 226/46623 objects misplaced (0.485%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:21:23.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:21:23.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:23.037 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:22 smithi142 ceph-mon[106729]: pgmap v660: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 970 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 9260/46623 objects degraded (19.861%); 226/46623 objects misplaced (0.485%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:21:23.037 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:22 smithi142 ceph-mon[103614]: pgmap v660: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 970 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 9260/46623 objects degraded (19.861%); 226/46623 objects misplaced (0.485%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:21:23.041 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:21:23.041 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-08-17T17:21:23.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:23.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:23.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:23.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:23.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:23.410 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.451 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:23.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:23.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:23.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:23.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:23.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:23.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:23.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:23.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:23.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:23.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:23.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:23.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:24.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:24.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:24.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:24.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:24.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:24.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:24.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:24.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.524 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:24.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:24.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:24.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:24.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:24.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:24.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:25.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:25.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:25.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:25.063 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:24 smithi142 ceph-mon[103614]: pgmap v661: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 799 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7525/38427 objects degraded (19.583%); 4.0 MiB/s, 77 objects/s recovering 2024-08-17T17:21:25.063 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:24 smithi142 ceph-mon[106729]: pgmap v661: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 799 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7525/38427 objects degraded (19.583%); 4.0 MiB/s, 77 objects/s recovering 2024-08-17T17:21:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:24 smithi152 ceph-mon[93456]: pgmap v661: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 799 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 7525/38427 objects degraded (19.583%); 4.0 MiB/s, 77 objects/s recovering 2024-08-17T17:21:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:21:25.106 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:21:25.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:25.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:25.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:25.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:25.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:25.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:25.440 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:25.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:25.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:25.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:25.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:25.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:25.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:25.802 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:25.802 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:25.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:25.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:25.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:26.002 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7525/38427 objects degraded (19.583%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:26.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7525/38427 objects degraded (19.583%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:26.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7525/38427 objects degraded (19.583%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:26.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:26.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:26.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:26.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:26.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:26.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:26.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:26.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:26.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:26.595 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:26.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:21:26.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:26.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:26.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:26.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:26 smithi142 ceph-mon[106729]: pgmap v662: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 718 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6891/34515 objects degraded (19.965%); 3.9 MiB/s, 76 objects/s recovering 2024-08-17T17:21:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:26 smithi142 ceph-mon[103614]: pgmap v662: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 718 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6891/34515 objects degraded (19.965%); 3.9 MiB/s, 76 objects/s recovering 2024-08-17T17:21:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:26.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:26.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:27.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:26 smithi152 ceph-mon[93456]: pgmap v662: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 718 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 6891/34515 objects degraded (19.965%); 3.9 MiB/s, 76 objects/s recovering 2024-08-17T17:21:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:27.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:27.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:27.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:27.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:27.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:27.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:27.375 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:27.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:27.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:27.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:27.669 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:27.771 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:27.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:27.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:27.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:28.005 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:28.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:21:28.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:28.120 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:28.222 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:28.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:28.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:28.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:28.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:28.534 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:28.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:28.691 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:28.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:28.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:28.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:28.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:28.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:29.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:28 smithi152 ceph-mon[93456]: pgmap v663: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 577 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 5323/27759 objects degraded (19.176%); 3.9 MiB/s, 76 objects/s recovering 2024-08-17T17:21:29.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:28 smithi142 ceph-mon[103614]: pgmap v663: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 577 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 5323/27759 objects degraded (19.176%); 3.9 MiB/s, 76 objects/s recovering 2024-08-17T17:21:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:28 smithi142 ceph-mon[106729]: pgmap v663: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 577 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 5323/27759 objects degraded (19.176%); 3.9 MiB/s, 76 objects/s recovering 2024-08-17T17:21:29.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.124 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:29.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:29.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:29.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:29.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:29.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:29.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:29.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:29.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:29.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:29.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:29.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:29.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:29.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:29.833 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3846249934' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:29.833 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:21:29.665+0000 7f42cbfbc440 -1 osd.5 190 log_to_monitors true 2024-08-17T17:21:29.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:29.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:30.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:30.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3846249934' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[103614]: from='osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:21:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[103614]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:21:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3846249934' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[106729]: from='osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:21:30.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:29 smithi142 ceph-mon[106729]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:21:30.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:30.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:30.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:30.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:29 smithi152 ceph-mon[93456]: from='osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:21:30.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:29 smithi152 ceph-mon[93456]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-17T17:21:30.209 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:21:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:21:29.863+0000 7f42c2565640 -1 osd.5 190 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:21:30.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:30.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:30.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:30.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:30.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:21:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:21:30.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:30.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:30.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:30.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:30.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:30.864 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:21:30.880 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:30.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:31.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: pgmap v664: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 401 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 3948/19299 objects degraded (20.457%); 1.3 MiB/s, 59 objects/s recovering 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: osdmap e198: 8 total, 7 up, 8 in 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: from='osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:21:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 3948/19299 objects degraded (20.457%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:31.081 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:31.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: pgmap v664: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 401 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 3948/19299 objects degraded (20.457%); 1.3 MiB/s, 59 objects/s recovering 2024-08-17T17:21:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:31.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: osdmap e198: 8 total, 7 up, 8 in 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: from='osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 3948/19299 objects degraded (20.457%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: pgmap v664: 99 pgs: 1 active+recovering+degraded+remapped, 14 active+undersized, 21 active+undersized+degraded, 63 active+clean; 401 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 3948/19299 objects degraded (20.457%); 1.3 MiB/s, 59 objects/s recovering 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: osdmap e198: 8 total, 7 up, 8 in 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: from='osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:21:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:21:31.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 3948/19299 objects degraded (20.457%), 22 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:31.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:31.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:31.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:31.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:31.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:31.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:31.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:31.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:31.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:31.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:31.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:31.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:31 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:21:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:31 smithi152 ceph-mon[93456]: osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087] boot 2024-08-17T17:21:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:31 smithi152 ceph-mon[93456]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T17:21:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:21:31.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:21:31] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-08-17T17:21:31.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:21:31.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[106729]: osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087] boot 2024-08-17T17:21:31.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[106729]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T17:21:31.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:21:31.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:21:31.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[103614]: osd.5 [v2:172.21.15.152:6808/395680087,v1:172.21.15.152:6809/395680087] boot 2024-08-17T17:21:31.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[103614]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T17:21:31.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-17T17:21:31.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:31.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:31.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:32.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:32.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:32.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:32.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:32.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:32.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:32.252 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:32.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:32.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:32.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:32.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:32.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:32.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:32.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:32.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:32.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:32.838 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:32.838 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:32.856 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:32.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:33.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:33.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:33.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:32 smithi142 ceph-mon[106729]: pgmap v667: 99 pgs: 7 peering, 1 active+recovering+degraded+remapped, 11 active+undersized, 17 active+undersized+degraded, 63 active+clean; 331 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 2692/15957 objects degraded (16.870%); 0 B/s, 33 objects/s recovering 2024-08-17T17:21:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:32 smithi142 ceph-mon[106729]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T17:21:33.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:32 smithi142 ceph-mon[103614]: pgmap v667: 99 pgs: 7 peering, 1 active+recovering+degraded+remapped, 11 active+undersized, 17 active+undersized+degraded, 63 active+clean; 331 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 2692/15957 objects degraded (16.870%); 0 B/s, 33 objects/s recovering 2024-08-17T17:21:33.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:32 smithi142 ceph-mon[103614]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T17:21:33.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:33.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:33.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:33.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:32 smithi152 ceph-mon[93456]: pgmap v667: 99 pgs: 7 peering, 1 active+recovering+degraded+remapped, 11 active+undersized, 17 active+undersized+degraded, 63 active+clean; 331 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 2692/15957 objects degraded (16.870%); 0 B/s, 33 objects/s recovering 2024-08-17T17:21:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:32 smithi152 ceph-mon[93456]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T17:21:33.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:33.424 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:33.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:33.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:33.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:21:33.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:33.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:33.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:33.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:33.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:33.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:34.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:34.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:33 smithi152 ceph-mon[93456]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T17:21:34.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:34.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:34.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:34.297 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:33 smithi142 ceph-mon[106729]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T17:21:34.298 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:33 smithi142 ceph-mon[103614]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T17:21:34.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:34.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:34.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:21:34.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:34.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:34.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:34.513 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:21:34.513 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:21:34.513 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:21:34.513 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:21:34.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:34.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:34.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:34.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:34.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:34.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:34.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:34.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:35.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:35.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:35.160 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:34 smithi152 ceph-mon[93456]: pgmap v670: 99 pgs: 4 activating+undersized+degraded+remapped, 14 peering, 4 active+undersized, 5 active+undersized+degraded, 72 active+clean; 237 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4825/11430 objects degraded (42.213%); 0 B/s, 35 objects/s recovering 2024-08-17T17:21:35.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:35.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:21:35.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:35.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:35.239 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:34 smithi142 ceph-mon[106729]: pgmap v670: 99 pgs: 4 activating+undersized+degraded+remapped, 14 peering, 4 active+undersized, 5 active+undersized+degraded, 72 active+clean; 237 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4825/11430 objects degraded (42.213%); 0 B/s, 35 objects/s recovering 2024-08-17T17:21:35.239 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:34 smithi142 ceph-mon[103614]: pgmap v670: 99 pgs: 4 activating+undersized+degraded+remapped, 14 peering, 4 active+undersized, 5 active+undersized+degraded, 72 active+clean; 237 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4825/11430 objects degraded (42.213%); 0 B/s, 35 objects/s recovering 2024-08-17T17:21:35.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:35.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:35.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:35.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:35.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:35.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:35.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:35.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:35.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:35.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:35.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:35.996 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4825/11430 objects degraded (42.213%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:35.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4825/11430 objects degraded (42.213%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:36.067 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.102 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:36.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:36.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:36.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:36.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4825/11430 objects degraded (42.213%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:36.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:36.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:36.418 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:36.424 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:36.477 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:21:36.478 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:36.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:36.515 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:21:36.515 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:36.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:36.554 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:21:36.555 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:36.586 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:21:36.586 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:36.616 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:21:36.616 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:36.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:36.646 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:21:36.646 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:36.678 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:21:36.678 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:36.708 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:21:36.708 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:36.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:36.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:36.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:36.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:36.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:37.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:37.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:37.191 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:36 smithi142 ceph-mon[103614]: pgmap v671: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 activating+undersized+degraded+remapped, 14 peering, 77 active+clean; 202 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 7281/9741 objects degraded (74.746%); 0 B/s, 25 objects/s recovering 2024-08-17T17:21:37.191 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/436542934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:37.191 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:36 smithi142 ceph-mon[106729]: pgmap v671: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 activating+undersized+degraded+remapped, 14 peering, 77 active+clean; 202 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 7281/9741 objects degraded (74.746%); 0 B/s, 25 objects/s recovering 2024-08-17T17:21:37.191 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/436542934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:37.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:37.252 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 132118 objects 2024-08-17T17:21:37.252 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :149.065 2024-08-17T17:21:37.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:37.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:36 smithi152 ceph-mon[93456]: pgmap v671: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 4 activating+undersized+degraded+remapped, 14 peering, 77 active+clean; 202 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 7281/9741 objects degraded (74.746%); 0 B/s, 25 objects/s recovering 2024-08-17T17:21:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/436542934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:37.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:37.377 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:37.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:37.500 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:37.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:37.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:37.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:37.748 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:37.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:37.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:37.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:37.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:37.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:38.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:38.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:38.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:38.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:38.243 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:38.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:38.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:38.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:38.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:38.373 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:38.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:38.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:38.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:38.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:38.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:38.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:38.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:38.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:38.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:38.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:39.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:39.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:39.264 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:38 smithi142 ceph-mon[106729]: pgmap v672: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 activating+undersized+degraded+remapped, 84 active+clean; 137 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 14274/6609 objects degraded (215.978%); 0 B/s, 72 objects/s recovering 2024-08-17T17:21:39.264 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:38 smithi142 ceph-mon[103614]: pgmap v672: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 activating+undersized+degraded+remapped, 84 active+clean; 137 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 14274/6609 objects degraded (215.978%); 0 B/s, 72 objects/s recovering 2024-08-17T17:21:39.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:38 smithi152 ceph-mon[93456]: pgmap v672: 99 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 4 activating+undersized+degraded+remapped, 84 active+clean; 137 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 14274/6609 objects degraded (215.978%); 0 B/s, 72 objects/s recovering 2024-08-17T17:21:39.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:39.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:39.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:39.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:39.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:39.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:39.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:39.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:39.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:39.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.736 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:39.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:39.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:39.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:39.940 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:21:39.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:39.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:21:40.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:40.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:40.195 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:39 smithi142 ceph-mon[106729]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T17:21:40.195 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/541875780' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:40.195 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:39 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:40.196 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:39 smithi142 ceph-mon[103614]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T17:21:40.196 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/541875780' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:40.196 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:39 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:40.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:39 smithi152 ceph-mon[93456]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T17:21:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/541875780' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:39 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:40.213 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:21:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=cleanup t=2024-08-17T17:21:39.870216601Z level=info msg="Completed cleanup jobs" duration=12.415927ms 2024-08-17T17:21:40.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:40.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:40.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:40.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:40.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:40.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:40.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:40.530 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:21:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:21:40.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:40.603 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:40.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:40.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:40.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:40.973 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:41.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:41.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:41.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:41.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[106729]: pgmap v674: 99 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 27 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 14271/1338 objects degraded (1066.592%); 0 B/s, 63 objects/s recovering 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[106729]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14271/1338 objects degraded (1066.592%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[106729]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[103614]: pgmap v674: 99 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 27 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 14271/1338 objects degraded (1066.592%); 0 B/s, 63 objects/s recovering 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:41.303 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:41.304 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[103614]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T17:21:41.304 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14271/1338 objects degraded (1066.592%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:41.304 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:40 smithi142 ceph-mon[103614]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T17:21:41.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-mon[93456]: pgmap v674: 99 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 27 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 14271/1338 objects degraded (1066.592%); 0 B/s, 63 objects/s recovering 2024-08-17T17:21:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-mon[93456]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T17:21:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14271/1338 objects degraded (1066.592%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:40 smithi152 ceph-mon[93456]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T17:21:41.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:41.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:41.470 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:41.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:41.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:41.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:41.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:41.820 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:41.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:41.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:41.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:41.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:41.973 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:21:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:21:41] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-17T17:21:42.045 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-08-17T17:21:42.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-08-17T17:21:42.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:42.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:42.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:42.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:42.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:42.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:42.405 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:21:42.405 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:21:42.405 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:21:42.405 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:21:42.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:42.471 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:42.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:42.594 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:42.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[106729]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T17:21:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[106729]: pgmap v678: 99 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.3 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 14271/351 objects degraded (4065.812%); 0 B/s, 0 objects/s recovering 2024-08-17T17:21:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[103614]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T17:21:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:42 smithi142 ceph-mon[103614]: pgmap v678: 99 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.3 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 14271/351 objects degraded (4065.812%); 0 B/s, 0 objects/s recovering 2024-08-17T17:21:42.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:42.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:42.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:42 smithi152 ceph-mon[93456]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T17:21:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:42 smithi152 ceph-mon[93456]: pgmap v678: 99 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.3 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 14271/351 objects degraded (4065.812%); 0 B/s, 0 objects/s recovering 2024-08-17T17:21:42.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:42.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:42.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:42.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:42.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:43.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:43.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:43.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:43.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:21:43.340 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:43.393 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:43.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:43.455 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:43 smithi142 ceph-mon[103614]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T17:21:43.455 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:43.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:43.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:43.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:43.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:43.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:43.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:43 smithi152 ceph-mon[93456]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T17:21:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:43.836 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:43.838 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:43 smithi142 ceph-mon[106729]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T17:21:43.838 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:43.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:43.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:44.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:44.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:44.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:44.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:44.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:44.332 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:21:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:44.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:44.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:44.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:44.473 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:21:44.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:44.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:44.592 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:44 smithi142 ceph-mon[106729]: pgmap v680: 99 pgs: 13 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 13149/78 objects degraded (16857.692%); 0 B/s, 386 objects/s recovering 2024-08-17T17:21:44.592 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:44 smithi142 ceph-mon[106729]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T17:21:44.592 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1149134367' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:21:44.593 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:44 smithi142 ceph-mon[103614]: pgmap v680: 99 pgs: 13 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 13149/78 objects degraded (16857.692%); 0 B/s, 386 objects/s recovering 2024-08-17T17:21:44.593 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:44 smithi142 ceph-mon[103614]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T17:21:44.593 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1149134367' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:21:44.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:44.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:44.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:44 smithi152 ceph-mon[93456]: pgmap v680: 99 pgs: 13 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 13149/78 objects degraded (16857.692%); 0 B/s, 386 objects/s recovering 2024-08-17T17:21:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:44 smithi152 ceph-mon[93456]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T17:21:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1149134367' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:21:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:44.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:44.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:44.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:45.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:45.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:45.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:45.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:21:45.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:45.236 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:45.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:45.362 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:45.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:45.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1149134367' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:21:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:45 smithi152 ceph-mon[93456]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T17:21:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13149/78 objects degraded (16857.692%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:45.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:45.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:45.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:45.630 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:45.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:45.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:45.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1149134367' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:21:45.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:45 smithi142 ceph-mon[106729]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T17:21:45.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13149/78 objects degraded (16857.692%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:45.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1149134367' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:21:45.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:45 smithi142 ceph-mon[103614]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T17:21:45.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13149/78 objects degraded (16857.692%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:45.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:45.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:45.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:45.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:46.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:46.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:46.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:46.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:46.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:46.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:46.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:46.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:46.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:46.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:46.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:46.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:46.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:46.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:46 smithi152 ceph-mon[93456]: pgmap v683: 99 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 12388/78 objects degraded (15882.051%); 0 B/s, 574 objects/s recovering 2024-08-17T17:21:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:46 smithi152 ceph-mon[93456]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T17:21:46.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:46.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:46 smithi142 ceph-mon[106729]: pgmap v683: 99 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 12388/78 objects degraded (15882.051%); 0 B/s, 574 objects/s recovering 2024-08-17T17:21:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:46 smithi142 ceph-mon[106729]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T17:21:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:46 smithi142 ceph-mon[103614]: pgmap v683: 99 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 12388/78 objects degraded (15882.051%); 0 B/s, 574 objects/s recovering 2024-08-17T17:21:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:46 smithi142 ceph-mon[103614]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T17:21:46.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:46.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:46.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:47.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:47.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:47.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:47.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:47.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:47.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:47.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:47.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:47.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:47.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:47.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:47.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:47.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:47.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:47.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:47 smithi152 ceph-mon[93456]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T17:21:47.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:47.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:47.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:47 smithi142 ceph-mon[106729]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T17:21:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:47 smithi142 ceph-mon[103614]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T17:21:47.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:47.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:48.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:48.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:48.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:48.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:48.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:48.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:48.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:48.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:48.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:48.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:48.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:48.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:48.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:48.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:48 smithi152 ceph-mon[93456]: pgmap v686: 99 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10836/78 objects degraded (13892.308%); 0 B/s, 578 objects/s recovering 2024-08-17T17:21:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:48 smithi152 ceph-mon[93456]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T17:21:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:48.833 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[103614]: pgmap v686: 99 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10836/78 objects degraded (13892.308%); 0 B/s, 578 objects/s recovering 2024-08-17T17:21:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[103614]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T17:21:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[106729]: pgmap v686: 99 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10836/78 objects degraded (13892.308%); 0 B/s, 578 objects/s recovering 2024-08-17T17:21:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[106729]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T17:21:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:21:48.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:48.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:49.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:49.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:49.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:49.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:49.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:49.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:49.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:49.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:49.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:49.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:49.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:49.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:49.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:49.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:49 smithi152 ceph-mon[93456]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T17:21:49.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:49 smithi142 ceph-mon[106729]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T17:21:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:49 smithi142 ceph-mon[103614]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T17:21:49.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:49.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:49.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:50.050 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:50.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.194 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:50.194 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:50.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:50.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:50.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:50.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:50.504 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:21:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:21:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:21:50.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:50.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:50.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:50.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:50.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.816 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:50.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:50 smithi152 ceph-mon[93456]: pgmap v689: 99 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 9288/78 objects degraded (11907.692%); 0 B/s, 774 objects/s recovering 2024-08-17T17:21:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 9288/78 objects degraded (11907.692%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:50 smithi142 ceph-mon[103614]: pgmap v689: 99 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 9288/78 objects degraded (11907.692%); 0 B/s, 774 objects/s recovering 2024-08-17T17:21:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 9288/78 objects degraded (11907.692%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:50 smithi142 ceph-mon[106729]: pgmap v689: 99 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 9288/78 objects degraded (11907.692%); 0 B/s, 774 objects/s recovering 2024-08-17T17:21:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 9288/78 objects degraded (11907.692%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:50.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:51.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:51.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:51.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:51.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:51.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:51.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:51.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:51.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:51.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:51.426 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:51.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:51.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:51.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:51.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:51.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:51.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:51.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:51.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:51.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:51.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:52.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:52.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:21:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:21:51] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-17T17:21:52.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:52.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:52.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:52.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:52.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:52.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:52.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:52.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:52.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:52 smithi142 ceph-mon[106729]: pgmap v690: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 7746/78 objects degraded (9930.769%); 0 B/s, 723 objects/s recovering 2024-08-17T17:21:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:52 smithi142 ceph-mon[103614]: pgmap v690: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 7746/78 objects degraded (9930.769%); 0 B/s, 723 objects/s recovering 2024-08-17T17:21:52.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:52.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:52.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:52.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:52 smithi152 ceph-mon[93456]: pgmap v690: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 7746/78 objects degraded (9930.769%); 0 B/s, 723 objects/s recovering 2024-08-17T17:21:52.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:52.922 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:53.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:53.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:53.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:53.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:53.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:53.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:53.394 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:53.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:53.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:53.518 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:53.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:53.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:53.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:53.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:53.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:53 smithi152 ceph-mon[93456]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T17:21:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:53 smithi142 ceph-mon[103614]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T17:21:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:53 smithi142 ceph-mon[106729]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T17:21:53.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:53.912 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:53.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:54.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:54.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:54.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:54.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.162 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:54.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:54.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:54.285 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:54.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:54.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:54.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:54.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:54.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:54.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:54 smithi142 ceph-mon[106729]: pgmap v692: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 7746/78 objects degraded (9930.769%); 0 B/s, 463 objects/s recovering 2024-08-17T17:21:54.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:54 smithi142 ceph-mon[106729]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T17:21:54.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:54 smithi142 ceph-mon[103614]: pgmap v692: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 7746/78 objects degraded (9930.769%); 0 B/s, 463 objects/s recovering 2024-08-17T17:21:54.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:54 smithi142 ceph-mon[103614]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T17:21:54.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:54.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:54.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:54.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:54.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:54 smithi152 ceph-mon[93456]: pgmap v692: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 7746/78 objects degraded (9930.769%); 0 B/s, 463 objects/s recovering 2024-08-17T17:21:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:54 smithi152 ceph-mon[93456]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T17:21:54.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:54.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:55.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:55.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:55.125 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:21:55.126 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:55.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:55.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:21:55.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:55.152 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:55.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:55.159 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:21:55.159 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:55.176 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:21:55.177 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:55.192 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:21:55.192 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:55.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:21:55.207 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:55.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:21:55.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:55.238 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:21:55.238 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:55.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:55.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:55.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:55.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:55.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:55.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:55.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:55.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:55.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:55.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:55.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:55 smithi152 ceph-mon[93456]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T17:21:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3566435895' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7746/78 objects degraded (9930.769%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:55 smithi152 ceph-mon[93456]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T17:21:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[103614]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T17:21:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3566435895' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7746/78 objects degraded (9930.769%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[103614]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T17:21:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[106729]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T17:21:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3566435895' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7746/78 objects degraded (9930.769%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:55 smithi142 ceph-mon[106729]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T17:21:55.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:55.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:56.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:56.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:56.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:56.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:56.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:56.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:56.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:56.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:56.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:56.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:56.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:56.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:56.656 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:21:56.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:56.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:56 smithi152 ceph-mon[93456]: pgmap v696: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6995/78 objects degraded (8967.949%); 0 B/s, 187 objects/s recovering 2024-08-17T17:21:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/393846408' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:21:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:21:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:56.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:21:56.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:56 smithi152 ceph-mon[93456]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T17:21:56.869 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[103614]: pgmap v696: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6995/78 objects degraded (8967.949%); 0 B/s, 187 objects/s recovering 2024-08-17T17:21:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/393846408' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:21:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:21:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:21:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[103614]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T17:21:56.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[106729]: pgmap v696: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6995/78 objects degraded (8967.949%); 0 B/s, 187 objects/s recovering 2024-08-17T17:21:56.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/393846408' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:21:56.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:21:56.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:56.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:21:56.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:56 smithi142 ceph-mon[106729]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T17:21:56.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:56.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:56.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:57.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:57.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:57.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:57.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:57.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:57.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:57.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:57.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:57.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:21:57.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:57.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:57.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:57.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:57.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:57.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:57.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:57.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:57.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:58.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:58.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:58.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:58.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:58.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:58.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:58.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:58.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:58.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:58.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:58.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:58.609 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:58.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:58.700 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:58 smithi142 ceph-mon[106729]: pgmap v698: 99 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6227/78 objects degraded (7983.333%); 0 B/s, 456 objects/s recovering 2024-08-17T17:21:58.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:58 smithi142 ceph-mon[103614]: pgmap v698: 99 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6227/78 objects degraded (7983.333%); 0 B/s, 456 objects/s recovering 2024-08-17T17:21:58.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:58.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:21:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:58 smithi152 ceph-mon[93456]: pgmap v698: 99 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6227/78 objects degraded (7983.333%); 0 B/s, 456 objects/s recovering 2024-08-17T17:21:58.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:58.883 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:58.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:59.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:59.109 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:59.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:59.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:59.254 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:59.356 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:59.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:59.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:59.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:59.558 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_4' removed 2024-08-17T17:21:59.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:59.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:59.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:59.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:59 smithi152 ceph-mon[93456]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T17:21:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/26099140' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:21:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/26099140' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:21:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:21:59 smithi152 ceph-mon[93456]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T17:21:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[106729]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T17:21:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/26099140' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:21:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/26099140' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:21:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[106729]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T17:21:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[103614]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T17:21:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/26099140' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:21:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/26099140' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:21:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:21:59 smithi142 ceph-mon[103614]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T17:21:59.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:21:59.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:59.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:00.020 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:00.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:00.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:00.186 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:22:00.187 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:22:00.188 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-08-17T17:22:00.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-08-17T17:22:00.246 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:00.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:22:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:22:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:22:00.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:00.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:00.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:00.521 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:00.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:00.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:00.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:00.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:00.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:00 smithi152 ceph-mon[93456]: pgmap v700: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 4653/78 objects degraded (5965.385%); 0 B/s, 506 objects/s recovering 2024-08-17T17:22:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4653/78 objects degraded (5965.385%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:00.834 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:00 smithi142 ceph-mon[103614]: pgmap v700: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 4653/78 objects degraded (5965.385%); 0 B/s, 506 objects/s recovering 2024-08-17T17:22:00.835 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4653/78 objects degraded (5965.385%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:00.835 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:00 smithi142 ceph-mon[106729]: pgmap v700: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 4653/78 objects degraded (5965.385%); 0 B/s, 506 objects/s recovering 2024-08-17T17:22:00.835 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4653/78 objects degraded (5965.385%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:22:00.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:00.900 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:00.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:00.984 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:00.985 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:01.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:01.028 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:01.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:01.130 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:01.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:01.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:01.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:01.357 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:01.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:01.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:01.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:01.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:01.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:01.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:01.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:01.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:01 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4653/78 objects degraded (5965.385%), 4 pgs degraded) 2024-08-17T17:22:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:01 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:22:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:01 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4653/78 objects degraded (5965.385%), 4 pgs degraded) 2024-08-17T17:22:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:01 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:22:01.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:01.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:01.909 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:02.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:02.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:01 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4653/78 objects degraded (5965.385%), 4 pgs degraded) 2024-08-17T17:22:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:01 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:22:02.109 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:22:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:22:01] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-17T17:22:02.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:02.161 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:02.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:02.285 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:02.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:02.320 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:02.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:02.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:02.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:02.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:02.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:02.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:02.607 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:22:02.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:02.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:02.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:02.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:02.791 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:02 smithi142 ceph-mon[106729]: pgmap v702: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4222099605' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:02 smithi142 ceph-mon[103614]: pgmap v702: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4222099605' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:02.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:02.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:02.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:22:03.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:03.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:02 smithi152 ceph-mon[93456]: pgmap v702: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4222099605' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:03.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:03.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:03.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:03.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:03.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:03.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:03.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:03.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:03.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:03 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4222099605' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:03 smithi142 ceph-mon[103614]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T17:22:03.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:03 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:03.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4222099605' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:03.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:03 smithi142 ceph-mon[106729]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T17:22:03.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:03.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:04.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:04.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:04.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:03 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4222099605' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:03 smithi152 ceph-mon[93456]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T17:22:04.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:04.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:04.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:04.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:04.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:04.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:04.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:04.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:04.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:04.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:04.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:04.669 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:04.771 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:04.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:04 smithi142 ceph-mon[103614]: pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:04.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:04 smithi142 ceph-mon[106729]: pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:04.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:04.923 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:05.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:05.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:04 smithi152 ceph-mon[93456]: pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:05.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:05.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:05.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:05.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:05.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:05.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:05.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:05.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:05.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:05.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:05.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:22:05.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:05.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:05.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:05.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:05.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:06.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:06.058 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:06.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:06.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:06.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:06.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:06.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:06.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:06.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:06.692 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:06.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:06.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:06 smithi142 ceph-mon[103614]: pgmap v705: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:06.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:06 smithi142 ceph-mon[106729]: pgmap v705: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:06.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:06.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:07.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:07.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:07.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:06 smithi152 ceph-mon[93456]: pgmap v705: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:07.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:07.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:07.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:07.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:07.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:07.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:07.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:07.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:07.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:07.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:07.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:07.692 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:07.739 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:07.739 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:07.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:07.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:07.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:07.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:07.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:08.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:08.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:08.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:08.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.202 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:08.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:08.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:08.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:08.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:08.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:08.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:08.577 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:08.675 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:22:08.678 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:08.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:08.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:08.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:08.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:08.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:08.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:22:09.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:09.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:09.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:08 smithi142 ceph-mon[106729]: pgmap v706: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:09.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2930781389' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:22:09.030 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1463523024' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:09.030 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:08 smithi142 ceph-mon[103614]: pgmap v706: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:09.030 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2930781389' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:22:09.031 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1463523024' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:09.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:08 smithi152 ceph-mon[93456]: pgmap v706: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:09.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2930781389' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:22:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1463523024' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:09.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:09.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:09.200 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:09.201 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:09.213 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:09.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:09.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:09.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:09.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:09.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:09.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:09.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:09.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:09.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:09.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:10.001 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2930781389' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:22:10.001 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:09 smithi142 ceph-mon[106729]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T17:22:10.001 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2930781389' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:22:10.001 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:09 smithi142 ceph-mon[103614]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T17:22:10.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2930781389' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:22:10.053 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:09 smithi152 ceph-mon[93456]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T17:22:10.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:10.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:10.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:10.209 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:10.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:22:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:22:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:22:10.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:10.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:10.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:10.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:10.582 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:10.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:10.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:10.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:10.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:10 smithi142 ceph-mon[106729]: pgmap v708: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:10 smithi142 ceph-mon[103614]: pgmap v708: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:10.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:10.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:11.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:10 smithi152 ceph-mon[93456]: pgmap v708: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:11.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:11.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:11.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:11.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:11.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:11.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:11.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:11.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:11.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:11.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:11.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:11.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:11.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:11.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:11.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:11.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:22:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:22:11] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.51.0" 2024-08-17T17:22:11.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:11.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:12.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:22:12.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:12.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:12.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:12.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:12.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:12.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:12.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:12.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:12.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:12.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:12.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:12.583 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:12.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:12.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:12.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:12.726 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:12.726 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:12.727 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:12 smithi142 ceph-mon[106729]: pgmap v709: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:12.727 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:12.727 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:12.727 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:12 smithi142 ceph-mon[103614]: pgmap v709: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:12.732 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:12 smithi152 ceph-mon[93456]: pgmap v709: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:12.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:12.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:12.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:13.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:13.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:13.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:13.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:13.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:13.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:13.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:13.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:13.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:13.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:13.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:13.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:13.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:13.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:13.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:13.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:13.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:13.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:13.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:14.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:14.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:14.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:14.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76678 8.7 8:e98d2fe7:::benchmark_data_smithi142_139631_object76677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.330434+0000", 2024-08-17T17:22:14.314 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 574.97282837600005, 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3664621810000002, 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76678 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330434+0000", 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330434+0000", 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330436+0000", 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3209999999999999e-06 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330459+0000", 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3008000000000001e-05 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330469+0000", 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0740000000000001e-05 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.330477+0000", 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8690000000000004e-06 2024-08-17T17:22:14.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.342648+0000", 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012170729 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.342658+0000", 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8109999999999997e-06 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.332589+0000", 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98993133600000005 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.332599+0000", 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0441e-05 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686590+0000", 2024-08-17T17:22:14.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3539904709999999 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686696+0000", 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105598 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.686804+0000", 2024-08-17T17:22:14.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010843700000000001 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695010+0000", 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0082063510000000006 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695058+0000", 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7336999999999998e-05 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695141+0000", 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2928000000000003e-05 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695204+0000", 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3542999999999998e-05 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.696896+0000", 2024-08-17T17:22:14.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001692261 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76729 8.7 8:e5dda4d8:::benchmark_data_smithi142_139631_object76728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 574.91968779399997, 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3218559409999999, 2024-08-17T17:22:14.323 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76729 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383575+0000", 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0399999999999996e-07 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383612+0000", 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6918999999999997e-05 2024-08-17T17:22:14.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383621+0000", 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1730000000000001e-06 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383627+0000", 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7529999999999998e-06 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383690+0000", 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2704000000000001e-05 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:39.383699+0000", 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0589999999999992e-06 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334127+0000", 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95042768600000005 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334134+0000", 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0670000000000003e-06 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.689412+0000", 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.355278092 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.689538+0000", 2024-08-17T17:22:14.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012647900000000001 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.689690+0000", 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015227200000000001 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.695072+0000", 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005381533 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.696946+0000", 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0018738050000000001 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.703090+0000", 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0061438350000000003 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.703285+0000", 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019538699999999999 2024-08-17T17:22:14.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.705431+0000", 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0021456729999999999 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76731 8.f 8:f7eca805:::benchmark_data_smithi142_139631_object76730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:40.327572+0000", 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 573.97569027999998, 2024-08-17T17:22:14.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.404185164, 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76731 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327572+0000", 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327572+0000", 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327573+0000", 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9400000000000004e-07 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327633+0000", 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9722000000000002e-05 2024-08-17T17:22:14.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327641+0000", 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1869999999999997e-06 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327648+0000", 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7299999999999999e-06 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327716+0000", 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7831999999999997e-05 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327721+0000", 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7610000000000004e-06 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334094+0000", 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063721790000000004 2024-08-17T17:22:14.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334101+0000", 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5290000000000003e-06 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.706562+0000", 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.372461156 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.706682+0000", 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000119981 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.706840+0000", 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000157236 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.714756+0000", 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079168780000000005 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.717828+0000", 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030717959999999999 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.729937+0000", 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012108934 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.730148+0000", 2024-08-17T17:22:14.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000210924 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.731757+0000", 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001609425 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45389.0:76744 8.f 8:f39abda7:::benchmark_data_smithi142_139631_object76743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e147)", 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:12:40.327674+0000", 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 573.97558818899995, 2024-08-17T17:22:14.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4061216009999999, 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45389", 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3131907480", 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 76744 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327674+0000", 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327674+0000", 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327675+0000", 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0999999999999998e-07 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327798+0000", 2024-08-17T17:22:14.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012282499999999999 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327807+0000", 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9979999999999992e-06 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327814+0000", 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7190000000000004e-06 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327924+0000", 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000110155 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.327929+0000", 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7729999999999996e-06 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334270+0000", 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063409240000000004 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:40.334280+0000", 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2369999999999998e-06 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.711398+0000", 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3771184869999999 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.711547+0000", 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014860000000000001 2024-08-17T17:22:14.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.711697+0000", 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014978999999999999 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.717702+0000", 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060053709999999998 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.731817+0000", 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014114658 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.732016+0000", 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019955799999999999 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.732183+0000", 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016642999999999999 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:12:41.733796+0000", 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001613466 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73423 9.10 9:0e0af26c:::benchmark_data_smithi142_157452_object73422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69595156400001, 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4164945229999999, 2024-08-17T17:22:14.347 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73423 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607312+0000", 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4619999999999999e-06 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607340+0000", 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7764999999999999e-05 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607353+0000", 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2471e-05 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607364+0000", 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1392999999999999e-05 2024-08-17T17:22:14.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607428+0000", 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3658000000000006e-05 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607439+0000", 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0777e-05 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013664+0000", 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4062256989999999 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013801+0000", 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000136594 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013929+0000", 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012783400000000001 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018572+0000", 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046436580000000002 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018584+0000", 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1574e-05 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018612+0000", 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7915e-05 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022687+0000", 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.004075543 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022742+0000", 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4341e-05 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023806+0000", 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001063839 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73452 9.10 9:08329d80:::benchmark_data_smithi142_157452_object73451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.355 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69588280299999, 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4217852020000001, 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73452 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:22:14.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.15e-07 2024-08-17T17:22:14.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607410+0000", 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9464000000000001e-05 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607416+0000", 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2319999999999998e-06 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607420+0000", 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2649999999999998e-06 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607541+0000", 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012065699999999999 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607548+0000", 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1559999999999998e-06 2024-08-17T17:22:14.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016317+0000", 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.408768746 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016455+0000", 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013853699999999999 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016639+0000", 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018399900000000001 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018680+0000", 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020403840000000001 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023861+0000", 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051812050000000004 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027763+0000", 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039021400000000001 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.028010+0000", 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024739199999999998 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.028049+0000", 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9082000000000003e-05 2024-08-17T17:22:14.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029165+0000", 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001115528 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73460 9.10 9:08db73f6:::benchmark_data_smithi142_157452_object73459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.695822077, 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4227749750000001, 2024-08-17T17:22:14.365 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73460 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6199999999999999e-07 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607459+0000", 2024-08-17T17:22:14.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7660999999999999e-05 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607463+0000", 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7570000000000004e-06 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607468+0000", 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.34e-06 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607628+0000", 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016045799999999999 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607638+0000", 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8360000000000004e-06 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017660+0000", 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.410022291 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.369 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017755+0000", 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4372999999999999e-05 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017851+0000", 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6256999999999997e-05 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020381+0000", 2024-08-17T17:22:14.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025300259999999999 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027826+0000", 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0074454389999999999 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027858+0000", 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1591000000000003e-05 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029231+0000", 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001372877 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029290+0000", 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9687e-05 2024-08-17T17:22:14.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030215+0000", 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00092502000000000003 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73489 9.10 9:0a567102:::benchmark_data_smithi142_157452_object73488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.695789464, 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4276421909999999, 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.373 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73489 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8599999999999999e-07 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607502+0000", 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8651000000000001e-05 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607511+0000", 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7800000000000006e-06 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607517+0000", 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8259999999999998e-06 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607995+0000", 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00047819600000000001 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608003+0000", 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4479999999999994e-06 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.377 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018702+0000", 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4106989999999999 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018795+0000", 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2928000000000002e-05 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018901+0000", 2024-08-17T17:22:14.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105565 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022627+0000", 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037258590000000002 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027930+0000", 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053030780000000001 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033448+0000", 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005518406 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033579+0000", 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013044700000000001 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033635+0000", 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5965999999999999e-05 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035115+0000", 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001480755 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73517 9.10 9:09edc6df:::benchmark_data_smithi142_157452_object73516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69571899900001, 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.428877572, 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.381 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73517 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.382 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5499999999999998e-07 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607565+0000", 2024-08-17T17:22:14.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0472e-05 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607569+0000", 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4059999999999998e-06 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607572+0000", 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3560000000000001e-06 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608033+0000", 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046030099999999998 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608037+0000", 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9820000000000002e-06 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019497+0000", 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.411460553 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.385 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019597+0000", 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9899e-05 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019700+0000", 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000102571 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.386 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023871+0000", 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041716840000000002 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027970+0000", 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040992240000000003 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033519+0000", 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0055483060000000002 2024-08-17T17:22:14.387 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035184+0000", 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0016652030000000001 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035240+0000", 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5546e-05 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.388 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036421+0000", 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011816140000000001 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.389 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73531 9.10 9:0aa603eb:::benchmark_data_smithi142_157452_object73530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.695684349, 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4341237490000001, 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73531 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.390 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.391 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607579+0000", 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.03e-07 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607597+0000", 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8376000000000002e-05 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607601+0000", 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.963e-06 2024-08-17T17:22:14.392 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607604+0000", 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8870000000000001e-06 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608070+0000", 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046594300000000002 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.393 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608077+0000", 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2099999999999996e-06 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020422+0000", 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4123452620000001 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.394 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020558+0000", 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135914 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020719+0000", 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016086799999999999 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023882+0000", 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003162621 2024-08-17T17:22:14.395 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036562+0000", 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012679953000000001 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040088+0000", 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035266020000000002 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040135+0000", 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6859e-05 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040193+0000", 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8128000000000003e-05 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041702+0000", 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0015088600000000001 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.397 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73543 9.10 9:0a994c9f:::benchmark_data_smithi142_157452_object73542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69564238199999, 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4357055299999999, 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.398 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73543 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.399 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607621+0000", 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3599999999999999e-07 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607638+0000", 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.702e-05 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607644+0000", 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.037e-06 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607647+0000", 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0410000000000002e-06 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608107+0000", 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045997900000000001 2024-08-17T17:22:14.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608110+0000", 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3220000000000001e-06 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021537+0000", 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.413427129 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021655+0000", 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117779 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021811+0000", 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015619200000000001 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026398+0000", 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045870349999999997 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.403 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036616+0000", 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010218213 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041785+0000", 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051685489999999997 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.404 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041907+0000", 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012203500000000001 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041969+0000", 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2238999999999998e-05 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043326+0000", 2024-08-17T17:22:14.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013566240000000001 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73564 9.10 9:0c98c0a8:::benchmark_data_smithi142_157452_object73563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69561125999999, 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.441867732, 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.406 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73564 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607652+0000", 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1199999999999999e-07 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607715+0000", 2024-08-17T17:22:14.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3386000000000006e-05 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607725+0000", 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8679999999999994e-06 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607730+0000", 2024-08-17T17:22:14.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.04e-06 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608144+0000", 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00041373799999999997 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608152+0000", 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4060000000000005e-06 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023936+0000", 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.415784218 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.024030+0000", 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3970999999999999e-05 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.024123+0000", 2024-08-17T17:22:14.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2508000000000003e-05 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027775+0000", 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003651774 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036678+0000", 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0089036219999999999 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043422+0000", 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067443060000000003 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.048000+0000", 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045772770000000003 2024-08-17T17:22:14.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.048076+0000", 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6469000000000003e-05 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049519+0000", 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001442837 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.414 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73583 9.10 9:0b96fbe3:::benchmark_data_smithi142_157452_object73582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69552798300001, 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.442907784, 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.415 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73583 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.416 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8999999999999998e-07 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607756+0000", 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1569e-05 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607761+0000", 2024-08-17T17:22:14.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5689999999999998e-06 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607764+0000", 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9529999999999999e-06 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608180+0000", 2024-08-17T17:22:14.418 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000415847 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608183+0000", 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1930000000000002e-06 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025148+0000", 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4169648399999999 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025286+0000", 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138417 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025447+0000", 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000160946 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027868+0000", 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00242055 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041848+0000", 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013980726000000001 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.421 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047049+0000", 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052002410000000004 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049578+0000", 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025293820000000002 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049642+0000", 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3837000000000006e-05 2024-08-17T17:22:14.422 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050642+0000", 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001000424 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.423 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73678 9.10 9:08769dcd:::benchmark_data_smithi142_157452_object73677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69549451500001, 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.44418486, 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73678 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.424 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.65e-07 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607782+0000", 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3886999999999999e-05 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607787+0000", 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6739999999999996e-06 2024-08-17T17:22:14.426 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.427 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.427 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.427 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607790+0000", 2024-08-17T17:22:14.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7970000000000001e-06 2024-08-17T17:22:14.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608215+0000", 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00042531699999999998 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608224+0000", 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9339999999999995e-06 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026442+0000", 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.418218049 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026597+0000", 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015451500000000001 2024-08-17T17:22:14.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026767+0000", 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170335 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027979+0000", 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012120810000000001 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.431 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050711+0000", 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.022731950000000001 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050763+0000", 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2281999999999999e-05 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050966+0000", 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020286300000000001 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051021+0000", 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5359000000000003e-05 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051953+0000", 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00093155199999999999 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73695 9.10 9:0bc067c9:::benchmark_data_smithi142_157452_object73694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69546926999999, 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4639758309999999, 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73695 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.434 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607794+0000", 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.96e-07 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607808+0000", 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4695000000000001e-05 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607813+0000", 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5129999999999998e-06 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607816+0000", 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7240000000000001e-06 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608255+0000", 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00043993199999999997 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608260+0000", 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9010000000000002e-06 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030261+0000", 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4220003640000001 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030373+0000", 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011251000000000001 2024-08-17T17:22:14.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030485+0000", 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111542 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033537+0000", 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030526939999999999 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.439 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050876+0000", 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017338675000000001 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051992+0000", 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011160079999999999 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.070213+0000", 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018221029 2024-08-17T17:22:14.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.070274+0000", 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0436999999999998e-05 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071769+0000", 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014955109999999999 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73736 9.10 9:0f48bdcd:::benchmark_data_smithi142_157452_object73735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69544339999999, 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4653223989999999, 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73736 2024-08-17T17:22:14.442 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4999999999999999e-07 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607834+0000", 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4226000000000001e-05 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607841+0000", 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4939999999999997e-06 2024-08-17T17:22:14.444 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607847+0000", 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3850000000000001e-06 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608291+0000", 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000444864 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608300+0000", 2024-08-17T17:22:14.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6119999999999999e-06 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032028+0000", 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4237279540000001 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032170+0000", 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014252899999999999 2024-08-17T17:22:14.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032268+0000", 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7950999999999998e-05 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036506+0000", 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042373360000000004 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054725+0000", 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018219227000000001 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055000+0000", 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00027527500000000001 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071822+0000", 2024-08-17T17:22:14.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016821922999999999 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071885+0000", 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2649000000000003e-05 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073142+0000", 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012567240000000001 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73777 9.10 9:08e4faeb:::benchmark_data_smithi142_157452_object73776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69541137499999, 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4667512089999999, 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.450 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73777 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4900000000000002e-07 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607866+0000", 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4863e-05 2024-08-17T17:22:14.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607870+0000", 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0010000000000003e-06 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607876+0000", 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8320000000000002e-06 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608327+0000", 2024-08-17T17:22:14.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045121099999999997 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608331+0000", 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9169999999999999e-06 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036717+0000", 2024-08-17T17:22:14.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4283860289999999 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036865+0000", 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014740099999999999 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.037067+0000", 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020239499999999999 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043346+0000", 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062784049999999999 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054765+0000", 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011419155 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073198+0000", 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018433105000000002 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073365+0000", 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016761400000000001 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073424+0000", 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.876e-05 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074602+0000", 2024-08-17T17:22:14.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001178272 2024-08-17T17:22:14.458 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.458 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.458 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.458 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73834 9.10 9:0efa507b:::benchmark_data_smithi142_157452_object73833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.459 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:22:14.459 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.69536871299999, 2024-08-17T17:22:14.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.467875166, 2024-08-17T17:22:14.459 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.459 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73834 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.460 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0999999999999999e-07 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607911+0000", 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7147999999999999e-05 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607916+0000", 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9400000000000001e-06 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607921+0000", 2024-08-17T17:22:14.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1590000000000003e-06 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608364+0000", 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00044320200000000002 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608372+0000", 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8730000000000007e-06 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038589+0000", 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.430217689 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038737+0000", 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014733899999999999 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038912+0000", 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017523699999999999 2024-08-17T17:22:14.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043352+0000", 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044401780000000004 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054813+0000", 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011461135000000001 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073240+0000", 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018426973999999999 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074657+0000", 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014169829999999999 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074711+0000", 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4185999999999999e-05 2024-08-17T17:22:14.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075769+0000", 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010577130000000001 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73885 9.10 9:08993fba:::benchmark_data_smithi142_157452_object73884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.695337886, 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.468984944, 2024-08-17T17:22:14.467 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73885 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:22:14.468 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5499999999999999e-07 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607940+0000", 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4739999999999999e-05 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607944+0000", 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5630000000000002e-06 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607947+0000", 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7439999999999999e-06 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608397+0000", 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045033600000000002 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608401+0000", 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5870000000000001e-06 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043476+0000", 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4350746729999999 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043632+0000", 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000156176 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043817+0000", 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000184754 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047067+0000", 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032502579999999998 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055046+0000", 2024-08-17T17:22:14.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079788489999999997 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073280+0000", 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018234017000000002 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075815+0000", 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025350659999999999 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075873+0000", 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8557000000000002e-05 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.076910+0000", 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010363690000000001 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.475 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73890 9.10 9:0c2319f1:::benchmark_data_smithi142_157452_object73889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 237.695312024, 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5039437520000001, 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:22:14.476 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73890 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4900000000000002e-07 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607965+0000", 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3899e-05 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:22:14.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607969+0000", 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5879999999999999e-06 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607972+0000", 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7520000000000001e-06 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608429+0000", 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045679500000000002 2024-08-17T17:22:14.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608437+0000", 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3999999999999992e-06 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045264+0000", 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.436826288 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045402+0000", 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138221 2024-08-17T17:22:14.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045554+0000", 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000152585 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047073+0000", 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001518637 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055084+0000", 2024-08-17T17:22:14.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080111230000000002 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073321+0000", 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018236588000000001 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.077006+0000", 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0036852550000000001 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.482 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.077063+0000", 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6827999999999999e-05 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.111894+0000", 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034831543999999999 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:22:14.483 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:22:14.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:14.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:14 smithi142 ceph-mon[106729]: pgmap v710: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:14.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:14 smithi142 ceph-mon[103614]: pgmap v710: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:14.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:14.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:14.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:14.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:14 smithi152 ceph-mon[93456]: pgmap v710: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:22:14.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:14.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:14.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:14.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:15.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:15.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:15.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:15.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:15.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:15.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:15.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:15.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:15.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:15.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:15.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:15.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:15.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:15.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:15.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:15.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:15.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:15.856 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:15.877 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:15.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:15.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:16.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:16.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:16.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:16.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:16.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:16.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:16.382 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:16.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:16.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:16.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:16.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:16 smithi142 ceph-mon[106729]: pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:16 smithi142 ceph-mon[103614]: pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:16.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:16.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:16.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:16 smithi152 ceph-mon[93456]: pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:16.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:16.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:16.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:17.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:17.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:17.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:17.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:17.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:22:17.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:17.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:17.362 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:17.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:17.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:17.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:17.536 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_5' created 2024-08-17T17:22:17.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:17.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:17.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:17.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/424431679' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:22:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:17 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:22:17.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/424431679' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:22:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:17 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:22:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/424431679' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:22:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:17 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:22:17.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:17.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:18.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:18.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:18.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:18.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:18.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-08-17T17:22:18.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:18.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:18.367 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:18.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:18.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:18.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:18.533 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:22:18.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:18.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:18.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:18 smithi152 ceph-mon[93456]: pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:18 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:22:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:18 smithi152 ceph-mon[93456]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T17:22:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1467707474' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:18.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[106729]: pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:22:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[106729]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T17:22:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1467707474' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[103614]: pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:22:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[103614]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T17:22:18.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1467707474' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:18.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:18.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:19.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:19.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:19.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:19.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:19.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:19.263 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:19.279 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:19.285 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:19.294 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:19.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:19.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:19.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:19.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:19.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:19.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:19.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:19.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:19.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1467707474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1467707474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:19.778 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1467707474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:19.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:19.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:19.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:19.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:20.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:20.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:20.129 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:20.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:20.257 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:20.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:20.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:20.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:20.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:20.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:20.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:20.556 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:22:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:22:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:22:20.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:20.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:20.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:20.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:20 smithi152 ceph-mon[93456]: pgmap v715: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:20 smithi152 ceph-mon[93456]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T17:22:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:20 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:22:20.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:20 smithi142 ceph-mon[106729]: pgmap v715: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:20 smithi142 ceph-mon[106729]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T17:22:20.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:20 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:22:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:20 smithi142 ceph-mon[103614]: pgmap v715: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:20 smithi142 ceph-mon[103614]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T17:22:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:20 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:22:20.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:20.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:21.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:21.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:21.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:21.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:21.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:21.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:22:21.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:21.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:21.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:21.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:21.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:21.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:21.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:21.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:21.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:21.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:21.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:21.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:21.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:21.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:22.115 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:22:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:22:21] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.51.0" 2024-08-17T17:22:22.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:22.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:22.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:22.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:22.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:22.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:22.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:22.528 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:22.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:22.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:22.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:22.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:22.820 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:22 smithi142 ceph-mon[106729]: pgmap v717: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:22.820 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:22 smithi142 ceph-mon[103614]: pgmap v717: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:22 smithi152 ceph-mon[93456]: pgmap v717: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:22.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:22.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:23.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:23.037 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:23.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:23.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:23.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:23.267 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:23.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:23.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:23.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:23.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:23.481 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:22:23.481 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:23.497 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:22:23.497 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:23.513 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:22:23.513 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:23.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:23.530 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:22:23.530 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:23.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:23.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:23.545 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:22:23.546 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:23.562 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:22:23.562 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:23.579 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:22:23.580 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:23.595 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:22:23.596 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:23.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:23.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:23.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:23.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:23.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:23.809 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3318213567' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:23.809 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3318213567' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3318213567' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:23.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:23.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:24.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:24.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:24.146 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:24.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.169 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:24.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:24.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:24.288 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:24.289 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:24.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:24.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:24.414 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:24.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:24.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:24.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:24.659 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:24.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:24.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:24.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:24 smithi152 ceph-mon[93456]: pgmap v718: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:24.835 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:24 smithi142 ceph-mon[103614]: pgmap v718: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:24.836 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:24 smithi142 ceph-mon[106729]: pgmap v718: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:24.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:24.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:24.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:25.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:25.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:25.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:25.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:25.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:25.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:25.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:25.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:25.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:25.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:25.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:25.574 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:22:25.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:25.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:25.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:25.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:25.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:25 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1508425128' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:22:25.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:25 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:22:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:25 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1508425128' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:22:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:25 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:22:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:25 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1508425128' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:22:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:25 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:22:25.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:25.927 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:25.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:26.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:26.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:26.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:26.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:26.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:26.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:26.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:22:26.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:26.429 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:26.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:26.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:26.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:26.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:26.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:26.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:26.790 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:26.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:26.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:26 smithi152 ceph-mon[93456]: pgmap v719: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:26 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:22:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:26 smithi152 ceph-mon[93456]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T17:22:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[106729]: pgmap v719: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:22:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[106729]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[103614]: pgmap v719: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[103614]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:26.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:26.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:26.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:27.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:27.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:27.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:27.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:27.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:27.317 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:27.419 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:27.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:27.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:27.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:27.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:27.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:27.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:27.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:27.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:27.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:28.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.076 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:28.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:28.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:28.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:28.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:28.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:28.381 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:28.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:28.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:28.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:28.576 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:28.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.580 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-08-17T17:22:28.679 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:28.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:28.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:28.780 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:28 smithi142 ceph-mon[106729]: pgmap v721: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:28.780 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/55072842' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:22:28.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:28 smithi142 ceph-mon[103614]: pgmap v721: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:28.781 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/55072842' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:22:28.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:28.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:22:28.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:28.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:28.934 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:28.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:29.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:28 smithi152 ceph-mon[93456]: pgmap v721: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/55072842' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:22:29.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:29.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:29.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:29.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:29.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:29.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:29.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:29.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:29.528 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:29.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:29.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:29.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:29.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:29.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/55072842' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:22:29.766 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:29 smithi142 ceph-mon[106729]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T17:22:29.766 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/55072842' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:22:29.767 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:29 smithi142 ceph-mon[103614]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T17:22:29.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:29.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:29.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:29.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:29.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:30.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:30.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/55072842' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:22:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:29 smithi152 ceph-mon[93456]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T17:22:30.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:30.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:30.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:30.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:30.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:30.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:30.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:30.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:30.503 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-08-17T17:22:30.505 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:22:30.506 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:22:30.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:30.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:22:30.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:22:30.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_192453 2024-08-17T17:22:30.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:22:30.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:22:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:22:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:22:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:22:30.587 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:30 smithi142 ceph-mon[106729]: pgmap v723: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:30.587 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:30 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:22:30.609 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:30.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:30.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:30.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:30 smithi142 ceph-mon[103614]: pgmap v723: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:30 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:22:30.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:30.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:31.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:30 smithi152 ceph-mon[93456]: pgmap v723: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:22:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:30 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:22:31.129 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:31.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:31.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:31.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:31.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:31.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:31.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 4086 4070 254.357 254.375 0.00517711 0.003911 2024-08-17T17:22:31.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:31.603 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:22:31.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:31.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:31.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.864 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:22:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:22:31] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-08-17T17:22:31.864 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3051140018' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:31.864 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:31.864 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3051140018' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:31.864 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:31.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:32.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3051140018' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:32.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:32.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:32.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:32.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:32.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:32.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:32.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:32.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:32.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:32.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 7880 7864 245.731 237.125 0.00394512 0.00406191 2024-08-17T17:22:32.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:32.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:32.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:32.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:32.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[106729]: pgmap v724: 89 pgs: 89 active+clean; 5.9 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 688 KiB/s wr, 10 op/s 2024-08-17T17:22:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[106729]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[103614]: pgmap v724: 89 pgs: 89 active+clean; 5.9 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 688 KiB/s wr, 10 op/s 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[103614]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T17:22:32.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:32.917 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:32.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:33.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:32 smithi152 ceph-mon[93456]: pgmap v724: 89 pgs: 89 active+clean; 5.9 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 688 KiB/s wr, 10 op/s 2024-08-17T17:22:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:32 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:32 smithi152 ceph-mon[93456]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T17:22:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:33.098 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:33.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:33.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:33.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:33.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:33.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:33.481 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:33.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:33.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 11423 11407 237.625 221.438 0.00685469 0.00420088 2024-08-17T17:22:33.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:33.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:33.754 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:33.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:33.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:33.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:33.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:34.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:34.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:34.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.208 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:34.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:34.285 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:34.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:34.349 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:22:34.349 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:34.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:34.385 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:22:34.385 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:34.421 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:22:34.421 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:34.452 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:22:34.452 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:34.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:34.482 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:22:34.482 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:34.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:34.512 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:22:34.512 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:34.544 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:22:34.544 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:34.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 14600 14584 227.855 198.562 0.00249412 0.00438067 2024-08-17T17:22:34.580 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:22:34.581 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:34.589 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:34.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:34.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:34.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:34.818 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:34 smithi142 ceph-mon[106729]: pgmap v726: 89 pgs: 89 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s 2024-08-17T17:22:34.818 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/418334319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:34.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:34 smithi142 ceph-mon[103614]: pgmap v726: 89 pgs: 89 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s 2024-08-17T17:22:34.819 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/418334319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:34.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:34.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:34.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:35.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:34 smithi152 ceph-mon[93456]: pgmap v726: 89 pgs: 89 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s 2024-08-17T17:22:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/418334319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:35.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:35.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:35.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:35.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:35.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:35.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:35.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:35.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 17488 17472 218.381 180.5 0.00418496 0.00457227 2024-08-17T17:22:35.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:35.617 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:22:35.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:35.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:35.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1009998812' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:22:35.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1009998812' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:22:35.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:35.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:35.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:36.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:36.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:22:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1009998812' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:22:36.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:36.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:36.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:36.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:36.393 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:36.430 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:36.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:36.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:36.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 20337 20321 211.659 178.062 0.00469914 0.00471848 2024-08-17T17:22:36.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:36.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:22:36.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:36.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:36.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:36 smithi142 ceph-mon[106729]: pgmap v727: 89 pgs: 89 active+clean; 474 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2024-08-17T17:22:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1009998812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:22:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:36 smithi142 ceph-mon[106729]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T17:22:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:36 smithi142 ceph-mon[103614]: pgmap v727: 89 pgs: 89 active+clean; 474 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2024-08-17T17:22:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1009998812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:22:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:36 smithi142 ceph-mon[103614]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T17:22:36.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:36.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:36.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:37.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:36 smithi152 ceph-mon[93456]: pgmap v727: 89 pgs: 89 active+clean; 474 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2024-08-17T17:22:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1009998812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:22:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:36 smithi152 ceph-mon[93456]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T17:22:37.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:37.169 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:37.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:37.307 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:37.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:37.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:37.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:37.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:37.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:37.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:37.567 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 22945 22929 204.705 163 0.0064955 0.00488021 2024-08-17T17:22:37.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:37.700 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:37.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:37.837 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:37.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:37.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:38.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:38.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:38.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:38.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:38.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:38.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:38.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:38.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:38.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 25261 25245 197.209 144.75 0.00472985 0.00506568 2024-08-17T17:22:38.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:38.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:38.741 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:38.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:38.917 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:39.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:39.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:38 smithi152 ceph-mon[93456]: pgmap v729: 89 pgs: 89 active+clean; 849 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:22:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:38 smithi142 ceph-mon[106729]: pgmap v729: 89 pgs: 89 active+clean; 849 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:22:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:38 smithi142 ceph-mon[103614]: pgmap v729: 89 pgs: 89 active+clean; 849 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-17T17:22:39.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.188 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:39.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:39.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 15 27294 27279 189.42 127.125 0.00715225 0.00527287 2024-08-17T17:22:39.582 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:39.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:39.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:39.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:39.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:39.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:39.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:39.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:40.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:40.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:40.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:40.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:40.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:40.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:40.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:40.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:40.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:40.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 29811 29795 186.202 157.25 0.00554494 0.00536466 2024-08-17T17:22:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:22:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:22:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:22:40.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:40.637 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:40 smithi142 ceph-mon[103614]: pgmap v730: 89 pgs: 89 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-08-17T17:22:40.658 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:40.677 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:40.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:40.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:40.896 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:40 smithi142 ceph-mon[106729]: pgmap v730: 89 pgs: 89 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-08-17T17:22:40.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:40.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:40.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:41.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:40 smithi152 ceph-mon[93456]: pgmap v730: 89 pgs: 89 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 177 MiB/s wr, 2.83k op/s 2024-08-17T17:22:41.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:41.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:41.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:41.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:41.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:41.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:41.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:41.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:41.511 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:22:41.511 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:22:41.511 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:22:41.511 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:22:41.511 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:22:41.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 31910 31894 181.2 131.188 0.00362397 0.0055139 2024-08-17T17:22:41.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:41.709 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:41.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:41.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:41.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:41.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:42.015 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:42.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:22:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:22:41] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-08-17T17:22:42.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:42.191 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:42.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:42.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:42.432 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:42.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:42.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:42.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 34214 34198 178.099 144 0.00271677 0.00560887 2024-08-17T17:22:42.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:42.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:42.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:42.734 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:42.735 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:42.735 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:42 smithi142 ceph-mon[103614]: pgmap v731: 89 pgs: 89 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:22:42.735 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:42.735 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:42.735 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:42 smithi142 ceph-mon[106729]: pgmap v731: 89 pgs: 89 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:22:42.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:42.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:42 smithi152 ceph-mon[93456]: pgmap v731: 89 pgs: 89 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:22:42.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:42.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:42.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:43.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.037 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:43.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:43.174 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:43.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:43.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:43.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:43.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 36071 36055 173.324 116.062 0.00514764 0.00576399 2024-08-17T17:22:43.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:43.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:43.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:43.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:43.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:44.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:44.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:44.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:44.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:44.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:44.484 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:44.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 37341 37325 166.613 79.375 0.00416777 0.00599612 2024-08-17T17:22:44.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:44.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:44.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:44.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:44 smithi152 ceph-mon[93456]: pgmap v732: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-08-17T17:22:44.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:44 smithi142 ceph-mon[106729]: pgmap v732: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-08-17T17:22:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:44 smithi142 ceph-mon[103614]: pgmap v732: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-08-17T17:22:44.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:44.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:44.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:45.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:45.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:22:45.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:45.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:45.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:45.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:45.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:45.411 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:45.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:45.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:45.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:45.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 38844 38828 161.767 93.9375 0.00652377 0.00617594 2024-08-17T17:22:45.643 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:45.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:45.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:45.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:45.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:45.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:45.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:46.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:46.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:46.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:46.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:46.229 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:46.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:46.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:46.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:46.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:46.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 40497 40481 158.114 103.312 0.00603634 0.00631942 2024-08-17T17:22:46.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:46.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:46.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:46.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:46 smithi152 ceph-mon[93456]: pgmap v733: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2024-08-17T17:22:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:46 smithi142 ceph-mon[106729]: pgmap v733: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2024-08-17T17:22:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:46 smithi142 ceph-mon[103614]: pgmap v733: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2024-08-17T17:22:46.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:46.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:46.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:47.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:47.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:47.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:47.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:47.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:47.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:47.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:47.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:47.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:47.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:47.568 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 42323 42307 155.526 114.125 0.0125303 0.00642498 2024-08-17T17:22:47.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:47.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:47.736 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:47.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:47.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:47.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:47.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:48.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:48.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:48.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:48.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:48.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:48.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:48.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:48.298 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:48.394 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:48.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:48.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:48.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:48.569 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 44192 44176 153.368 116.812 0.0093831 0.00651378 2024-08-17T17:22:48.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:48.672 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:48 smithi142 ceph-mon[103614]: pgmap v734: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-08-17T17:22:48.672 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:48 smithi142 ceph-mon[106729]: pgmap v734: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-08-17T17:22:48.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:48.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:48 smithi152 ceph-mon[93456]: pgmap v734: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-08-17T17:22:48.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:48.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:48.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:48.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:49.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:49.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:49.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:49.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:49.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:49.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:49.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:49.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:49.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:49.569 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 46034 46018 151.355 115.125 0.0180907 0.00659866 2024-08-17T17:22:49.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:49.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:49.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:49.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:49.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:49.894 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:49.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:49.979 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:50.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:50.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:50.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:50.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:50.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:50.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:50.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:50.477 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:50.561 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:22:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:22:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:22:50.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:22:50.569979+0000 min lat: 0.00124672 max lat: 0.4126 avg lat: 0.00671639 2024-08-17T17:22:50.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:22:50.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 47629 47613 148.771 99.6875 0.00866934 0.00671639 2024-08-17T17:22:50.573 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:22:50.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:50.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:50.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:50.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:50 smithi152 ceph-mon[93456]: pgmap v735: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-08-17T17:22:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2559352901' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:50.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:50 smithi142 ceph-mon[103614]: pgmap v735: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-08-17T17:22:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2559352901' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:50 smithi142 ceph-mon[106729]: pgmap v735: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-08-17T17:22:50.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2559352901' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:50.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:50.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:51.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:51.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:51.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:51.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:51.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:51.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:51.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:51.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:51.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 49483 49467 147.204 115.875 0.00551387 0.00678739 2024-08-17T17:22:51.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:51.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:51.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:51.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:51.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:51.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:51.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2559352901' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:22:51.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:51 smithi142 ceph-mon[106729]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T17:22:51.874 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:22:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:22:51] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-08-17T17:22:51.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2559352901' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:22:51.875 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:51 smithi142 ceph-mon[103614]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T17:22:51.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:51.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:51.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:52.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:22:52.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2559352901' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:22:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:51 smithi152 ceph-mon[93456]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T17:22:52.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:52.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:52.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:52.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:52.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:52.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:52.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:52.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:52.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:52.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 51159 51143 145.274 104.75 0.00997297 0.00687888 2024-08-17T17:22:52.601 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:22:52.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:52.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:52.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:52.791 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:52.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:52.880 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:52 smithi142 ceph-mon[103614]: pgmap v737: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-08-17T17:22:52.880 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1472096543' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:52.880 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:52 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:52.880 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:52 smithi142 ceph-mon[106729]: pgmap v737: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-08-17T17:22:52.880 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1472096543' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:52.881 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:52 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:52.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:52.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:52 smithi152 ceph-mon[93456]: pgmap v737: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-08-17T17:22:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1472096543' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:52 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:53.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:53.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:53.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:53.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:53.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:53.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:53.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:53.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:53.499 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:53.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:53.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 52868 52852 143.6 106.812 0.00584731 0.00695892 2024-08-17T17:22:53.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:53.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[103614]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:22:53.756 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:53.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:53 smithi142 ceph-mon[106729]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T17:22:53.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:53.793 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:53.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:53.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:54.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:53 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:53 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:22:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:53 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:54.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:53 smithi152 ceph-mon[93456]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T17:22:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:54.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:54.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:54.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:54.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:54.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:54.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:54.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:54.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:54.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 54591 54575 142.103 107.688 0.00389466 0.00703196 2024-08-17T17:22:54.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:54.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:54 smithi142 ceph-mon[106729]: pgmap v739: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:22:54.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:54.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:54.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:54.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:54.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:54.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:54.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:55.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:55.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:55.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:55.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:54 smithi152 ceph-mon[93456]: pgmap v739: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:22:55.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:54 smithi142 ceph-mon[103614]: pgmap v739: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:22:55.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:55.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:55.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:55.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:55.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:55.470 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:22:55.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:55.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:55.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 56168 56152 140.361 98.5625 0.00973072 0.00711926 2024-08-17T17:22:55.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:55.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:55.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:55.750 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:55.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:55.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:55.885 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2419302527' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:55.886 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2419302527' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:55.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:56.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2419302527' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:56.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:56.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:56.221 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:22:56.221 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:22:56.221 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:56.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:56.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:56.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:56.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:56.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:56.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:56.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:56.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:56.570 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 57800 57784 138.885 102 0.00787645 0.00719555 2024-08-17T17:22:56.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:56.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:56.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:56.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:56.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:56 smithi142 ceph-mon[103614]: pgmap v740: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:22:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:56 smithi142 ceph-mon[106729]: pgmap v740: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:22:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:56.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:56.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:56.973 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:56.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:56 smithi152 ceph-mon[93456]: pgmap v740: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:22:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:22:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:57.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:57.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:57.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:57.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:57.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:57.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:57.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:57.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.527 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:57.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:57.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 59483 59467 137.628 105.188 0.0105818 0.00726144 2024-08-17T17:22:57.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:57.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:57.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:57.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:57.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:57.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:57.933 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:57.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:58.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:58.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:58.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:58.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:58.223 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:58.223 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:58.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:58.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:58.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:58.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:58.572 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 15 61053 61038 136.22 98.1875 0.027113 0.00733521 2024-08-17T17:22:58.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:58.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:58.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:58.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:58.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:58.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:58.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:58.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:58.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:58.974 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:58 smithi142 ceph-mon[106729]: pgmap v741: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-17T17:22:58.974 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:58 smithi142 ceph-mon[103614]: pgmap v741: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-17T17:22:58.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:59.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:59.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:58 smithi152 ceph-mon[93456]: pgmap v741: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-17T17:22:59.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:22:59.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:59.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:59.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:59.281 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:59.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:59.402 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:59.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:59.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:59.572 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 62468 62452 134.57 88.375 0.00823657 0.00742605 2024-08-17T17:22:59.590 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:59.649 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:22:59.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:59.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:59.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:22:59.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:22:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2549274863' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:22:59.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:22:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2549274863' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:22:59.953 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:22:59.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:00.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:22:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2549274863' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:23:00.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:00.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:00.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:00.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:00.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:00.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:00.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:00.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:00.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 63909 63893 133.086 90.0625 0.0102939 0.00750836 2024-08-17T17:23:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:23:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:23:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:23:00.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:00.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:00.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:00.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:00.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:00.923 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:00 smithi142 ceph-mon[106729]: pgmap v742: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-08-17T17:23:00.923 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2549274863' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:23:00.923 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:00 smithi142 ceph-mon[106729]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T17:23:00.923 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:00 smithi142 ceph-mon[103614]: pgmap v742: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-08-17T17:23:00.924 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2549274863' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:23:00.924 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:00 smithi142 ceph-mon[103614]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T17:23:00.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:01.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:01.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:00 smithi152 ceph-mon[93456]: pgmap v742: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-08-17T17:23:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2549274863' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:23:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:00 smithi152 ceph-mon[93456]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T17:23:01.076 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:01.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:01.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:01.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:01.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:01.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:01.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:23:01.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:01.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:01.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 65472 65456 131.944 97.6875 0.00874509 0.00757409 2024-08-17T17:23:01.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:01.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:01.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:01.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:01.821 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:01.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:01.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:02.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:02.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:23:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:23:01] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-08-17T17:23:02.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:02.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:02.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:02.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:02.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:02.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:02.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:02.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:02.572 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 66964 66948 130.735 93.25 0.0143568 0.00764416 2024-08-17T17:23:02.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:02.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:02.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:02.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:02.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:02.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:02.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:03.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:02 smithi152 ceph-mon[93456]: pgmap v744: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-17T17:23:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:02 smithi142 ceph-mon[106729]: pgmap v744: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-17T17:23:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:02 smithi142 ceph-mon[103614]: pgmap v744: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-17T17:23:03.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:03.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:03.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:03.282 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:03.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:03.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:03.522 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:03.560 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:03.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 68316 68300 129.333 84.5 0.00449369 0.00772695 2024-08-17T17:23:03.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:03.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:03.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:03.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:03.934 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:03.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:03.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:04.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:04.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:04.219 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:04.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:04.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:04.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:04.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:04.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:04.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:04.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 15 69787 69772 128.235 92 0.00636383 0.00779336 2024-08-17T17:23:04.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:04.658 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:04.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:04.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:04.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:04.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:04.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:04.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:05.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:05.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:04 smithi152 ceph-mon[93456]: pgmap v745: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-17T17:23:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:04 smithi142 ceph-mon[106729]: pgmap v745: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-17T17:23:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:04 smithi142 ceph-mon[103614]: pgmap v745: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-17T17:23:05.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:05.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:05.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:05.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:05.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:05.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:05.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:05.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 15 71180 71165 127.059 87.0625 0.00849467 0.00786591 2024-08-17T17:23:05.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:05.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:05.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:05.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:05.825 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:05.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:05.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:05.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:06.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:06.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:06.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:06.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:06.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:06.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:06.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:06.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:06.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:06.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:06.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 72435 72419 125.706 78.375 0.00641652 0.00795032 2024-08-17T17:23:06.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:06.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:06.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:06.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:06.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:06.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:06.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:06.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:06.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.002 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:06 smithi142 ceph-mon[106729]: pgmap v746: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:23:07.002 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:06 smithi142 ceph-mon[103614]: pgmap v746: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:23:07.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:07.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:07.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:07.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:06 smithi152 ceph-mon[93456]: pgmap v746: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:23:07.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:07.192 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:07.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:07.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:07.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:07.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:07.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 73740 73724 124.513 81.5625 0.0114117 0.00802575 2024-08-17T17:23:07.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:07.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:07.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:07.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:07.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:07.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:08.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:08.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:08.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:08.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:08.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:08.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:08.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:08.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:08.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 74961 74945 123.245 76.3125 0.0073416 0.00810823 2024-08-17T17:23:08.578 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:08.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:08.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:08.688 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:23:08.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:08.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:08.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:08.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:08.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:08 smithi152 ceph-mon[93456]: pgmap v747: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:23:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3434346685' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:08 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:09.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:09.120 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:08 smithi142 ceph-mon[106729]: pgmap v747: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:23:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3434346685' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:08 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:09.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:08 smithi142 ceph-mon[103614]: pgmap v747: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:23:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3434346685' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:08 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:09.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:09.254 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:09.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:09.377 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:09.479 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:09.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:09.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:09.573 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 76044 76028 121.82 67.6875 0.0116104 0.00820357 2024-08-17T17:23:09.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:09.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:09.754 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:09.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:09.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:09.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:09.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:09 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:09.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:09.960 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:09 smithi142 ceph-mon[103614]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T17:23:09.960 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:09 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:09.960 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:09.960 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:09 smithi142 ceph-mon[106729]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T17:23:10.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:10.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:10.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:10.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:09 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:10.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:09 smithi152 ceph-mon[93456]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T17:23:10.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:10.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:10.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:10.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:10.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:10.468 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:10.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:23:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:23:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:23:10.574 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:23:10.574581+0000 min lat: 0.00124672 max lat: 0.4126 avg lat: 0.00826157 2024-08-17T17:23:10.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:23:10.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 77444 77428 120.959 87.5 0.0106754 0.00826157 2024-08-17T17:23:10.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:10.698 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:23:10.699 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:10.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:10.737 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:23:10.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:10.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:10.770 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:23:10.770 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:10.801 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:23:10.802 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:10.836 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:23:10.837 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:10.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:10.849 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:10.874 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:23:10.874 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:10.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:10.901 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:23:10.901 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:10.902 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:10 smithi142 ceph-mon[106729]: pgmap v749: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:23:10.902 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2400224917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:10.902 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:10 smithi142 ceph-mon[103614]: pgmap v749: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:23:10.902 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2400224917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:10.931 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:23:10.931 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:10.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.036 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:11.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:10 smithi152 ceph-mon[93456]: pgmap v749: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:23:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2400224917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:11.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:11.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:11.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:11.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:11.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:11.541 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:11.574 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 78739 78723 119.983 80.9375 0.0110183 0.00832966 2024-08-17T17:23:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:11.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:11.652 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:23:11.652 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:23:11.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:78827 10.4 e232/222 10:2b571299:::benchmark_data_smithi142_192453_object78826:head v 232'4915, mlcod=232'4913)", 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:11.650548+0000", 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.00039788200000000001, 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000422922, 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650548+0000", 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:23:11.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650548+0000", 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650549+0000", 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1570000000000001e-06 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650572+0000", 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3632e-05 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650593+0000", 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0811000000000001e-05 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:23:11.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650602+0000", 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9169999999999995e-06 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650782+0000", 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000179984 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:11.650826+0000", 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4254e-05 2024-08-17T17:23:11.656 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:23:11.657 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:23:11.657 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:23:11.657 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:23:11.657 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:23:11.657 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 1 2024-08-17T17:23:11.657 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:23:11.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:11.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:11.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:11.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:11.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:11.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:12.032 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:23:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:23:11] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.51.0" 2024-08-17T17:23:12.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:12.122 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:12.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:12.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:12.361 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:12.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:12.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:12.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:12.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:12.574 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 80102 80086 119.155 85.1875 0.00910368 0.00838728 2024-08-17T17:23:12.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:12.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:12.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:12.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:12.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:12.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:12 smithi142 ceph-mon[103614]: pgmap v750: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:23:12.709 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:12.709 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:12.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:12 smithi142 ceph-mon[106729]: pgmap v750: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:23:12.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:12.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:12 smithi152 ceph-mon[93456]: pgmap v750: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:23:12.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:12.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:13.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:13.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:13.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:13.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:23:13.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:13.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:13.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:13.391 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:13.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:13.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:13.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 81349 81333 118.196 77.9375 0.0084035 0.00845589 2024-08-17T17:23:13.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:13.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:13.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:13.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:13.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:13.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:13.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:13.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:14.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:14.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:14.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:14.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:14.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:14.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:14.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:14.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:14.523 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:14.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 82558 82542 117.227 75.5625 0.010255 0.00852436 2024-08-17T17:23:14.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:14.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:14.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:14.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:14.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:14 smithi152 ceph-mon[93456]: pgmap v751: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T17:23:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:14 smithi142 ceph-mon[106729]: pgmap v751: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T17:23:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:14 smithi142 ceph-mon[103614]: pgmap v751: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T17:23:14.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:14.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:14.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:15.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:15.103 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:15.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:15.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:15.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:15.254 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:15.356 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:15.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:15.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:15.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:15.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 83726 83710 116.244 73 0.0232735 0.00859771 2024-08-17T17:23:15.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:15.670 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:15.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:15.772 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:15.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:23:15.809 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:15.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:15.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:15.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:15.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:16.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:16.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:16.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:16.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:16.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:16.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:16.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:16.349 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:16.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:16.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:16.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:16.569 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:23:16.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 84913 84897 115.33 74.1875 0.0121403 0.00866563 2024-08-17T17:23:16.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:16.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:16.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:16.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:16.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:16.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:16 smithi142 ceph-mon[103614]: pgmap v752: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:23:16.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/385946468' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:16.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:16 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:16 smithi142 ceph-mon[106729]: pgmap v752: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:23:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/385946468' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:16.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:16 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:16.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:16.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:17.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:16 smithi152 ceph-mon[93456]: pgmap v752: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:23:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/385946468' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:16 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:23:17.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:17.193 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:17.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:17.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:17.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:17.470 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:17.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:17.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 85945 85929 114.248 64.5 0.00286601 0.00873041 2024-08-17T17:23:17.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:17.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:17.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:17.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:17.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:17.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:17 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:23:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:17 smithi142 ceph-mon[106729]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T17:23:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:17 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:23:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:17 smithi142 ceph-mon[103614]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T17:23:17.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:17.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:17 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:23:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:17 smithi152 ceph-mon[93456]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T17:23:18.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:18.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:18.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:18.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:18.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:18.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:18.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:18.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:18.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 87085 87069 113.352 71.25 0.00857114 0.00881737 2024-08-17T17:23:18.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:18.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:18.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:18.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:18 smithi142 ceph-mon[106729]: pgmap v754: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:23:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:18 smithi142 ceph-mon[103614]: pgmap v754: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:23:18.935 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:18.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:18.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:19.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:18 smithi152 ceph-mon[93456]: pgmap v754: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:23:19.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:19.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:19.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:19.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:19.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:19.335 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:19.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:19.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:19.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:19.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:19.575 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 88164 88148 112.415 67.4375 0.0123556 0.00889049 2024-08-17T17:23:19.597 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:19 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:19.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:19.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:19.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:19.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:19.832 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:23:19.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:23:19.875 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:19 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:19.899 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:19.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:20.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:19 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:20.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:20.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:20.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:20.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:20.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:20.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:20.401 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:20.401 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:20.427 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:23:20.427 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:23:20.427 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:23:20.428 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:23:20.428 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:23:20.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:20.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:23:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:23:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:23:20.576 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 89140 89124 111.384 61 0.0131099 0.00897264 2024-08-17T17:23:20.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:20.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:20.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:20.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:20.816 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:20.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:20.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:21.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:21.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:21.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[103614]: pgmap v755: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2024-08-17T17:23:21.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[106729]: pgmap v755: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.166 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:21 smithi152 ceph-mon[93456]: pgmap v755: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2024-08-17T17:23:21.166 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.166 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:21.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:21.205 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:21.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:21.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:21.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:21.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:21.490 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:21.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:21.576 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 90177 90161 110.471 64.8125 0.0168733 0.00904727 2024-08-17T17:23:21.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:21.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:21.679 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:21.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:23:21.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:21.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:21.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:21.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:22.014 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:23:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:23:21] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.51.0" 2024-08-17T17:23:22.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:22.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:22.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:22.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:22.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:22.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:22.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:22.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:22.576 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 91294 91278 109.689 69.8125 0.0269926 0.00911157 2024-08-17T17:23:22.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:22.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:22.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:22.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:22.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:22.914 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:22.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:23.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:23.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.055 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:23.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:23.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:23.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:23.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:23 smithi152 ceph-mon[93456]: pgmap v756: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:23:23.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:23.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:23.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:23 smithi142 ceph-mon[103614]: pgmap v756: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:23:23.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:23 smithi142 ceph-mon[106729]: pgmap v756: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:23:23.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:23.484 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:23.576 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 92459 92443 108.993 72.8125 0.0151739 0.00916978 2024-08-17T17:23:23.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:23.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:23.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:23.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:23.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:23.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:23.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:23.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:24.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.036 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:24.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:24.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:24.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:24.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:24.395 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:24.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:24.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:24.548 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:24.577 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 93621 93605 108.319 72.625 0.0180198 0.00922725 2024-08-17T17:23:24.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:24.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:24.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:24.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:24.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:24.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:24.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.010 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:25.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:25.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:25.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:25.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:25.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:25 smithi152 ceph-mon[93456]: pgmap v757: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-08-17T17:23:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:25 smithi142 ceph-mon[106729]: pgmap v757: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-08-17T17:23:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:25 smithi142 ceph-mon[103614]: pgmap v757: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-08-17T17:23:25.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:25.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:25.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:25.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:25.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:25.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:25.577 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 94757 94741 107.64 71 0.0156709 0.00928518 2024-08-17T17:23:25.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:25.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:25.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:25.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:25.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:26.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:26.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:26.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:26.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:26.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:26.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:26.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:26.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:26.577 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 95887 95871 106.979 70.625 0.0130978 0.00934245 2024-08-17T17:23:26.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:26.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:26.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:26.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:27.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:27.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:27.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:27.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:27.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:27.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:27.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:27.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:23:27.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:23:27.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:23:27.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:23:27.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:23:27.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:23:27.363 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:23:27.363 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:23:27.363 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:27.363 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:27.363 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:27.364 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:27.364 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:27.364 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:27.364 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:27.364 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:27.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:27.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:27 smithi152 ceph-mon[93456]: pgmap v758: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:23:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/289815493' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:27.577 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 97032 97016 106.358 71.5625 0.0241342 0.00939744 2024-08-17T17:23:27.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[103614]: pgmap v758: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:23:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/289815493' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[106729]: pgmap v758: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:27.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/289815493' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:27.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:27.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:27.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:27.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:27.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:27.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:27.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:28.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:28.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:28.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:28.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:28.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.314 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:28.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:28.482 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:23:28.487 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:23:28.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:28.577 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 98129 98113 105.706 68.5625 0.00616409 0.00945502 2024-08-17T17:23:28.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:28.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:28.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:28.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:28.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:28.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:28.974 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:29.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:29.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:29.202 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:23:29.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:29.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:29.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:29.386 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":759,"stamp":"2024-08-17T17:23:27.497241+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6123558861,"num_objects":93455,"num_object_clones":0,"num_object_copies":280365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93455,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":93707,"num_write_kb":5981903,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93636,"ondisk_log_size":93636,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23213860,"kb_used_data":18014692,"kb_used_omap":35,"kb_used_meta":5199068,"kb_avail":726747356,"statfs":{"total":767960285184,"available":744189292544,"internally_reserved":0,"allocated":18447044608,"data_stored":18437967127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36357,"internal_metadata":5323846139},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,4,5,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":850001920,"num_objects":12970,"num_object_clones":0,"num_object_copies":38910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12970,"num_write_kb":830080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.935275"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":276,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.573869+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:23:16.573869+0000","last_peered":"2024-08-17T17:23:16.573869+0000","last_clean":"2024-08-17T17:23:16.573869+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:23:16.573869+0000","last_undegraded":"2024-08-17T17:23:16.573869+0000","last_fullsized":"2024-08-17T17:23:16.573869+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":190,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.686624+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:23:08.686624+0000","last_peered":"2024-08-17T17:23:08.686624+0000","last_clean":"2024-08-17T17:23:08.686624+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:23:08.686624+0000","last_undegraded":"2024-08-17T17:23:08.686624+0000","last_fullsized":"2024-08-17T17:23:08.686624+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.686943+0000","last_change":"2024-08-17T17:21:31.903762+0000","last_active":"2024-08-17T17:23:08.686943+0000","last_peered":"2024-08-17T17:23:08.686943+0000","last_clean":"2024-08-17T17:23:08.686943+0000","last_became_active":"2024-08-17T17:21:31.902518+0000","last_became_peered":"2024-08-17T17:21:31.902518+0000","last_unstale":"2024-08-17T17:23:08.686943+0000","last_undegraded":"2024-08-17T17:23:08.686943+0000","last_fullsized":"2024-08-17T17:23:08.686943+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:13.790840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":327,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.589214+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:23:16.589214+0000","last_peered":"2024-08-17T17:23:16.589214+0000","last_clean":"2024-08-17T17:23:16.589214+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:23:16.589214+0000","last_undegraded":"2024-08-17T17:23:16.589214+0000","last_fullsized":"2024-08-17T17:23:16.589214+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":326,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.590457+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:23:16.590457+0000","last_peered":"2024-08-17T17:23:16.590457+0000","last_clean":"2024-08-17T17:23:16.590457+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:23:16.590457+0000","last_undegraded":"2024-08-17T17:23:16.590457+0000","last_fullsized":"2024-08-17T17:23:16.590457+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.688725+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:23:08.688725+0000","last_peered":"2024-08-17T17:23:08.688725+0000","last_clean":"2024-08-17T17:23:08.688725+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:23:08.688725+0000","last_undegraded":"2024-08-17T17:23:08.688725+0000","last_fullsized":"2024-08-17T17:23:08.688725+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":252,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572804+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:23:16.572804+0000","last_peered":"2024-08-17T17:23:16.572804+0000","last_clean":"2024-08-17T17:23:16.572804+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:23:16.572804+0000","last_undegraded":"2024-08-17T17:23:16.572804+0000","last_fullsized":"2024-08-17T17:23:16.572804+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":213,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690382+0000","last_change":"2024-08-17T17:21:31.868761+0000","last_active":"2024-08-17T17:23:08.690382+0000","last_peered":"2024-08-17T17:23:08.690382+0000","last_clean":"2024-08-17T17:23:08.690382+0000","last_became_active":"2024-08-17T17:21:31.868430+0000","last_became_peered":"2024-08-17T17:21:31.868430+0000","last_unstale":"2024-08-17T17:23:08.690382+0000","last_undegraded":"2024-08-17T17:23:08.690382+0000","last_fullsized":"2024-08-17T17:23:08.690382+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:35:33.534743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"233'6000","reported_seq":6033,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:26.861157+0000","last_change":"2024-08-17T17:22:18.545386+0000","last_active":"2024-08-17T17:23:26.861157+0000","last_peered":"2024-08-17T17:23:26.861157+0000","last_clean":"2024-08-17T17:23:26.861157+0000","last_became_active":"2024-08-17T17:22:18.545116+0000","last_became_peered":"2024-08-17T17:22:18.545116+0000","last_unstale":"2024-08-17T17:23:26.861157+0000","last_undegraded":"2024-08-17T17:23:26.861157+0000","last_fullsized":"2024-08-17T17:23:26.861157+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6000,"log_dups_size":0,"ondisk_log_size":6000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:13:37.764971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216000,"num_objects":6000,"num_object_clones":0,"num_object_copies":18000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6000,"num_write_kb":384000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":189,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687627+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:23:08.687627+0000","last_peered":"2024-08-17T17:23:08.687627+0000","last_clean":"2024-08-17T17:23:08.687627+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:23:08.687627+0000","last_undegraded":"2024-08-17T17:23:08.687627+0000","last_fullsized":"2024-08-17T17:23:08.687627+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.686432+0000","last_change":"2024-08-17T17:21:31.876798+0000","last_active":"2024-08-17T17:23:08.686432+0000","last_peered":"2024-08-17T17:23:08.686432+0000","last_clean":"2024-08-17T17:23:08.686432+0000","last_became_active":"2024-08-17T17:21:31.876370+0000","last_became_peered":"2024-08-17T17:21:31.876370+0000","last_unstale":"2024-08-17T17:23:08.686432+0000","last_undegraded":"2024-08-17T17:23:08.686432+0000","last_fullsized":"2024-08-17T17:23:08.686432+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:33:16.352719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"233'5657","reported_seq":5676,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.606201+0000","last_change":"2024-08-17T17:22:18.544826+0000","last_active":"2024-08-17T17:23:22.606201+0000","last_peered":"2024-08-17T17:23:22.606201+0000","last_clean":"2024-08-17T17:23:22.606201+0000","last_became_active":"2024-08-17T17:22:18.544566+0000","last_became_peered":"2024-08-17T17:22:18.544566+0000","last_unstale":"2024-08-17T17:23:22.606201+0000","last_undegraded":"2024-08-17T17:23:22.606201+0000","last_fullsized":"2024-08-17T17:23:22.606201+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5657,"log_dups_size":0,"ondisk_log_size":5657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:08.783581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370737152,"num_objects":5657,"num_object_clones":0,"num_object_copies":16971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5657,"num_write_kb":362048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":190,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687461+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:23:08.687461+0000","last_peered":"2024-08-17T17:23:08.687461+0000","last_clean":"2024-08-17T17:23:08.687461+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:23:08.687461+0000","last_undegraded":"2024-08-17T17:23:08.687461+0000","last_fullsized":"2024-08-17T17:23:08.687461+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":204,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689654+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:23:08.689654+0000","last_peered":"2024-08-17T17:23:08.689654+0000","last_clean":"2024-08-17T17:23:08.689654+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:23:08.689654+0000","last_undegraded":"2024-08-17T17:23:08.689654+0000","last_fullsized":"2024-08-17T17:23:08.689654+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"233'6013","reported_seq":6045,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:26.861534+0000","last_change":"2024-08-17T17:22:18.545432+0000","last_active":"2024-08-17T17:23:26.861534+0000","last_peered":"2024-08-17T17:23:26.861534+0000","last_clean":"2024-08-17T17:23:26.861534+0000","last_became_active":"2024-08-17T17:22:18.544999+0000","last_became_peered":"2024-08-17T17:22:18.544999+0000","last_unstale":"2024-08-17T17:23:26.861534+0000","last_undegraded":"2024-08-17T17:23:26.861534+0000","last_fullsized":"2024-08-17T17:23:26.861534+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6013,"log_dups_size":0,"ondisk_log_size":6013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:54.714976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394067968,"num_objects":6013,"num_object_clones":0,"num_object_copies":18039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6013,"num_write_kb":384832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":189,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687141+0000","last_change":"2024-08-17T17:21:31.904014+0000","last_active":"2024-08-17T17:23:08.687141+0000","last_peered":"2024-08-17T17:23:08.687141+0000","last_clean":"2024-08-17T17:23:08.687141+0000","last_became_active":"2024-08-17T17:21:31.903128+0000","last_became_peered":"2024-08-17T17:21:31.903128+0000","last_unstale":"2024-08-17T17:23:08.687141+0000","last_undegraded":"2024-08-17T17:23:08.687141+0000","last_fullsized":"2024-08-17T17:23:08.687141+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:56:59.504309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690347+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:23:08.690347+0000","last_peered":"2024-08-17T17:23:08.690347+0000","last_clean":"2024-08-17T17:23:08.690347+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:23:08.690347+0000","last_undegraded":"2024-08-17T17:23:08.690347+0000","last_fullsized":"2024-08-17T17:23:08.690347+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"233'5937","reported_seq":5970,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:23.894922+0000","last_change":"2024-08-17T17:22:18.542919+0000","last_active":"2024-08-17T17:23:23.894922+0000","last_peered":"2024-08-17T17:23:23.894922+0000","last_clean":"2024-08-17T17:23:23.894922+0000","last_became_active":"2024-08-17T17:22:18.542151+0000","last_became_peered":"2024-08-17T17:22:18.542151+0000","last_unstale":"2024-08-17T17:23:23.894922+0000","last_undegraded":"2024-08-17T17:23:23.894922+0000","last_fullsized":"2024-08-17T17:23:23.894922+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5937,"log_dups_size":0,"ondisk_log_size":5937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:14.269839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389087232,"num_objects":5937,"num_object_clones":0,"num_object_copies":17811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5937,"num_write_kb":379968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":213,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.574008+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:23:16.574008+0000","last_peered":"2024-08-17T17:23:16.574008+0000","last_clean":"2024-08-17T17:23:16.574008+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:23:16.574008+0000","last_undegraded":"2024-08-17T17:23:16.574008+0000","last_fullsized":"2024-08-17T17:23:16.574008+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":327,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.592868+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:23:16.592868+0000","last_peered":"2024-08-17T17:23:16.592868+0000","last_clean":"2024-08-17T17:23:16.592868+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:23:16.592868+0000","last_undegraded":"2024-08-17T17:23:16.592868+0000","last_fullsized":"2024-08-17T17:23:16.592868+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"233'5922","reported_seq":5955,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:25.892694+0000","last_change":"2024-08-17T17:22:18.546366+0000","last_active":"2024-08-17T17:23:25.892694+0000","last_peered":"2024-08-17T17:23:25.892694+0000","last_clean":"2024-08-17T17:23:25.892694+0000","last_became_active":"2024-08-17T17:22:18.545788+0000","last_became_peered":"2024-08-17T17:22:18.545788+0000","last_unstale":"2024-08-17T17:23:25.892694+0000","last_undegraded":"2024-08-17T17:23:25.892694+0000","last_fullsized":"2024-08-17T17:23:25.892694+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:13.373207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388104192,"num_objects":5922,"num_object_clones":0,"num_object_copies":17766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5922,"num_write_kb":379008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":280,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572028+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:23:16.572028+0000","last_peered":"2024-08-17T17:23:16.572028+0000","last_clean":"2024-08-17T17:23:16.572028+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:23:16.572028+0000","last_undegraded":"2024-08-17T17:23:16.572028+0000","last_fullsized":"2024-08-17T17:23:16.572028+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690218+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:23:08.690218+0000","last_peered":"2024-08-17T17:23:08.690218+0000","last_clean":"2024-08-17T17:23:08.690218+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:23:08.690218+0000","last_undegraded":"2024-08-17T17:23:08.690218+0000","last_fullsized":"2024-08-17T17:23:08.690218+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"233'5690","reported_seq":5723,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:23.889729+0000","last_change":"2024-08-17T17:22:18.541297+0000","last_active":"2024-08-17T17:23:23.889729+0000","last_peered":"2024-08-17T17:23:23.889729+0000","last_clean":"2024-08-17T17:23:23.889729+0000","last_became_active":"2024-08-17T17:22:18.541002+0000","last_became_peered":"2024-08-17T17:22:18.541002+0000","last_unstale":"2024-08-17T17:23:23.889729+0000","last_undegraded":"2024-08-17T17:23:23.889729+0000","last_fullsized":"2024-08-17T17:23:23.889729+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:07.861642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372899840,"num_objects":5690,"num_object_clones":0,"num_object_copies":17070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5690,"num_write_kb":364160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":188,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690423+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:23:08.690423+0000","last_peered":"2024-08-17T17:23:08.690423+0000","last_clean":"2024-08-17T17:23:08.690423+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:23:08.690423+0000","last_undegraded":"2024-08-17T17:23:08.690423+0000","last_fullsized":"2024-08-17T17:23:08.690423+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":204,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690193+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:23:08.690193+0000","last_peered":"2024-08-17T17:23:08.690193+0000","last_clean":"2024-08-17T17:23:08.690193+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:23:08.690193+0000","last_undegraded":"2024-08-17T17:23:08.690193+0000","last_fullsized":"2024-08-17T17:23:08.690193+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"233'5663","reported_seq":5682,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.587797+0000","last_change":"2024-08-17T17:22:18.545009+0000","last_active":"2024-08-17T17:23:22.587797+0000","last_peered":"2024-08-17T17:23:22.587797+0000","last_clean":"2024-08-17T17:23:22.587797+0000","last_became_active":"2024-08-17T17:22:18.544525+0000","last_became_peered":"2024-08-17T17:22:18.544525+0000","last_unstale":"2024-08-17T17:23:22.587797+0000","last_undegraded":"2024-08-17T17:23:22.587797+0000","last_fullsized":"2024-08-17T17:23:22.587797+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5663,"log_dups_size":0,"ondisk_log_size":5663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:35.215727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371130368,"num_objects":5663,"num_object_clones":0,"num_object_copies":16989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5663,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":188,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689794+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:23:08.689794+0000","last_peered":"2024-08-17T17:23:08.689794+0000","last_clean":"2024-08-17T17:23:08.689794+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:23:08.689794+0000","last_undegraded":"2024-08-17T17:23:08.689794+0000","last_fullsized":"2024-08-17T17:23:08.689794+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690117+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:23:08.690117+0000","last_peered":"2024-08-17T17:23:08.690117+0000","last_clean":"2024-08-17T17:23:08.690117+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:23:08.690117+0000","last_undegraded":"2024-08-17T17:23:08.690117+0000","last_fullsized":"2024-08-17T17:23:08.690117+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"233'6110","reported_seq":6129,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:27.260554+0000","last_change":"2024-08-17T17:22:18.542069+0000","last_active":"2024-08-17T17:23:27.260554+0000","last_peered":"2024-08-17T17:23:27.260554+0000","last_clean":"2024-08-17T17:23:27.260554+0000","last_became_active":"2024-08-17T17:22:18.541290+0000","last_became_peered":"2024-08-17T17:22:18.541290+0000","last_unstale":"2024-08-17T17:23:27.260554+0000","last_undegraded":"2024-08-17T17:23:27.260554+0000","last_fullsized":"2024-08-17T17:23:27.260554+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6110,"log_dups_size":0,"ondisk_log_size":6110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:04.374938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400424960,"num_objects":6110,"num_object_clones":0,"num_object_copies":18330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":391040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":253,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572136+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:23:16.572136+0000","last_peered":"2024-08-17T17:23:16.572136+0000","last_clean":"2024-08-17T17:23:16.572136+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:23:16.572136+0000","last_undegraded":"2024-08-17T17:23:16.572136+0000","last_fullsized":"2024-08-17T17:23:16.572136+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":209,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572887+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:23:16.572887+0000","last_peered":"2024-08-17T17:23:16.572887+0000","last_clean":"2024-08-17T17:23:16.572887+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:23:16.572887+0000","last_undegraded":"2024-08-17T17:23:16.572887+0000","last_fullsized":"2024-08-17T17:23:16.572887+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"233'5689","reported_seq":5708,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.572464+0000","last_change":"2024-08-17T17:22:18.547163+0000","last_active":"2024-08-17T17:23:22.572464+0000","last_peered":"2024-08-17T17:23:22.572464+0000","last_clean":"2024-08-17T17:23:22.572464+0000","last_became_active":"2024-08-17T17:22:18.546827+0000","last_became_peered":"2024-08-17T17:22:18.546827+0000","last_unstale":"2024-08-17T17:23:22.572464+0000","last_undegraded":"2024-08-17T17:23:22.572464+0000","last_fullsized":"2024-08-17T17:23:22.572464+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:11.384041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372834304,"num_objects":5689,"num_object_clones":0,"num_object_copies":17067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5689,"num_write_kb":364096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":528,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687624+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:23:08.687624+0000","last_peered":"2024-08-17T17:23:08.687624+0000","last_clean":"2024-08-17T17:23:08.687624+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:23:08.687624+0000","last_undegraded":"2024-08-17T17:23:08.687624+0000","last_fullsized":"2024-08-17T17:23:08.687624+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":128,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.582477+0000","last_change":"2024-08-17T17:21:31.878312+0000","last_active":"2024-08-17T17:23:16.582477+0000","last_peered":"2024-08-17T17:23:16.582477+0000","last_clean":"2024-08-17T17:23:16.582477+0000","last_became_active":"2024-08-17T17:21:31.877932+0000","last_became_peered":"2024-08-17T17:21:31.877932+0000","last_unstale":"2024-08-17T17:23:16.582477+0000","last_undegraded":"2024-08-17T17:23:16.582477+0000","last_fullsized":"2024-08-17T17:23:16.582477+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:17.773813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":251,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.571496+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:23:16.571496+0000","last_peered":"2024-08-17T17:23:16.571496+0000","last_clean":"2024-08-17T17:23:16.571496+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:23:16.571496+0000","last_undegraded":"2024-08-17T17:23:16.571496+0000","last_fullsized":"2024-08-17T17:23:16.571496+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":170,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690897+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:23:08.690897+0000","last_peered":"2024-08-17T17:23:08.690897+0000","last_clean":"2024-08-17T17:23:08.690897+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:23:08.690897+0000","last_undegraded":"2024-08-17T17:23:08.690897+0000","last_fullsized":"2024-08-17T17:23:08.690897+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"233'5996","reported_seq":6029,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:26.858643+0000","last_change":"2024-08-17T17:22:18.544196+0000","last_active":"2024-08-17T17:23:26.858643+0000","last_peered":"2024-08-17T17:23:26.858643+0000","last_clean":"2024-08-17T17:23:26.858643+0000","last_became_active":"2024-08-17T17:22:18.543865+0000","last_became_peered":"2024-08-17T17:22:18.543865+0000","last_unstale":"2024-08-17T17:23:26.858643+0000","last_undegraded":"2024-08-17T17:23:26.858643+0000","last_fullsized":"2024-08-17T17:23:26.858643+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5996,"log_dups_size":0,"ondisk_log_size":5996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:48.118547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392953856,"num_objects":5996,"num_object_clones":0,"num_object_copies":17988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5996,"num_write_kb":383744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":217,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687571+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:23:08.687571+0000","last_peered":"2024-08-17T17:23:08.687571+0000","last_clean":"2024-08-17T17:23:08.687571+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:23:08.687571+0000","last_undegraded":"2024-08-17T17:23:08.687571+0000","last_fullsized":"2024-08-17T17:23:08.687571+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":327,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.593383+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:23:16.593383+0000","last_peered":"2024-08-17T17:23:16.593383+0000","last_clean":"2024-08-17T17:23:16.593383+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:23:16.593383+0000","last_undegraded":"2024-08-17T17:23:16.593383+0000","last_fullsized":"2024-08-17T17:23:16.593383+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":338,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.590290+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:23:16.590290+0000","last_peered":"2024-08-17T17:23:16.590290+0000","last_clean":"2024-08-17T17:23:16.590290+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:23:16.590290+0000","last_undegraded":"2024-08-17T17:23:16.590290+0000","last_fullsized":"2024-08-17T17:23:16.590290+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"233'5758","reported_seq":5791,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:23.872725+0000","last_change":"2024-08-17T17:22:18.540626+0000","last_active":"2024-08-17T17:23:23.872725+0000","last_peered":"2024-08-17T17:23:23.872725+0000","last_clean":"2024-08-17T17:23:23.872725+0000","last_became_active":"2024-08-17T17:22:18.540354+0000","last_became_peered":"2024-08-17T17:22:18.540354+0000","last_unstale":"2024-08-17T17:23:23.872725+0000","last_undegraded":"2024-08-17T17:23:23.872725+0000","last_fullsized":"2024-08-17T17:23:23.872725+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:36.914893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377356288,"num_objects":5758,"num_object_clones":0,"num_object_copies":17274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5758,"num_write_kb":368512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":313,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.583714+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:23:16.583714+0000","last_peered":"2024-08-17T17:23:16.583714+0000","last_clean":"2024-08-17T17:23:16.583714+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:23:16.583714+0000","last_undegraded":"2024-08-17T17:23:16.583714+0000","last_fullsized":"2024-08-17T17:23:16.583714+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":293,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570281+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:23:16.570281+0000","last_peered":"2024-08-17T17:23:16.570281+0000","last_clean":"2024-08-17T17:23:16.570281+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:23:16.570281+0000","last_undegraded":"2024-08-17T17:23:16.570281+0000","last_fullsized":"2024-08-17T17:23:16.570281+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":330,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570795+0000","last_change":"2024-08-17T17:21:31.880043+0000","last_active":"2024-08-17T17:23:16.570795+0000","last_peered":"2024-08-17T17:23:16.570795+0000","last_clean":"2024-08-17T17:23:16.570795+0000","last_became_active":"2024-08-17T17:21:31.879787+0000","last_became_peered":"2024-08-17T17:21:31.879787+0000","last_unstale":"2024-08-17T17:23:16.570795+0000","last_undegraded":"2024-08-17T17:23:16.570795+0000","last_fullsized":"2024-08-17T17:23:16.570795+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:29.293588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"233'5632","reported_seq":5651,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.599740+0000","last_change":"2024-08-17T17:22:18.544563+0000","last_active":"2024-08-17T17:23:22.599740+0000","last_peered":"2024-08-17T17:23:22.599740+0000","last_clean":"2024-08-17T17:23:22.599740+0000","last_became_active":"2024-08-17T17:22:18.544243+0000","last_became_peered":"2024-08-17T17:22:18.544243+0000","last_unstale":"2024-08-17T17:23:22.599740+0000","last_undegraded":"2024-08-17T17:23:22.599740+0000","last_fullsized":"2024-08-17T17:23:22.599740+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5632,"log_dups_size":0,"ondisk_log_size":5632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:36.074516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369098752,"num_objects":5632,"num_object_clones":0,"num_object_copies":16896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5632,"num_write_kb":360448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":135,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570325+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:23:16.570325+0000","last_peered":"2024-08-17T17:23:16.570325+0000","last_clean":"2024-08-17T17:23:16.570325+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:23:16.570325+0000","last_undegraded":"2024-08-17T17:23:16.570325+0000","last_fullsized":"2024-08-17T17:23:16.570325+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":188,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689915+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:23:08.689915+0000","last_peered":"2024-08-17T17:23:08.689915+0000","last_clean":"2024-08-17T17:23:08.689915+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:23:08.689915+0000","last_undegraded":"2024-08-17T17:23:08.689915+0000","last_fullsized":"2024-08-17T17:23:08.689915+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":305,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570014+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:23:16.570014+0000","last_peered":"2024-08-17T17:23:16.570014+0000","last_clean":"2024-08-17T17:23:16.570014+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:23:16.570014+0000","last_undegraded":"2024-08-17T17:23:16.570014+0000","last_fullsized":"2024-08-17T17:23:16.570014+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"233'5779","reported_seq":5798,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.989232+0000","last_change":"2024-08-17T17:22:18.542803+0000","last_active":"2024-08-17T17:23:22.989232+0000","last_peered":"2024-08-17T17:23:22.989232+0000","last_clean":"2024-08-17T17:23:22.989232+0000","last_became_active":"2024-08-17T17:22:18.542476+0000","last_became_peered":"2024-08-17T17:22:18.542476+0000","last_unstale":"2024-08-17T17:23:22.989232+0000","last_undegraded":"2024-08-17T17:23:22.989232+0000","last_fullsized":"2024-08-17T17:23:22.989232+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:07:11.840777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378732544,"num_objects":5779,"num_object_clones":0,"num_object_copies":17337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5779,"num_write_kb":369856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572723+0000","last_change":"2024-08-17T17:21:31.870010+0000","last_active":"2024-08-17T17:23:16.572723+0000","last_peered":"2024-08-17T17:23:16.572723+0000","last_clean":"2024-08-17T17:23:16.572723+0000","last_became_active":"2024-08-17T17:21:31.869696+0000","last_became_peered":"2024-08-17T17:21:31.869696+0000","last_unstale":"2024-08-17T17:23:16.572723+0000","last_undegraded":"2024-08-17T17:23:16.572723+0000","last_fullsized":"2024-08-17T17:23:16.572723+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:57.550019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":332,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570174+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:23:16.570174+0000","last_peered":"2024-08-17T17:23:16.570174+0000","last_clean":"2024-08-17T17:23:16.570174+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:23:16.570174+0000","last_undegraded":"2024-08-17T17:23:16.570174+0000","last_fullsized":"2024-08-17T17:23:16.570174+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":294,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570572+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:23:16.570572+0000","last_peered":"2024-08-17T17:23:16.570572+0000","last_clean":"2024-08-17T17:23:16.570572+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:23:16.570572+0000","last_undegraded":"2024-08-17T17:23:16.570572+0000","last_fullsized":"2024-08-17T17:23:16.570572+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"233'5760","reported_seq":5779,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.588830+0000","last_change":"2024-08-17T17:22:18.545052+0000","last_active":"2024-08-17T17:23:22.588830+0000","last_peered":"2024-08-17T17:23:22.588830+0000","last_clean":"2024-08-17T17:23:22.588830+0000","last_became_active":"2024-08-17T17:22:18.544251+0000","last_became_peered":"2024-08-17T17:22:18.544251+0000","last_unstale":"2024-08-17T17:23:22.588830+0000","last_undegraded":"2024-08-17T17:23:22.588830+0000","last_fullsized":"2024-08-17T17:23:22.588830+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:40.762667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377487360,"num_objects":5760,"num_object_clones":0,"num_object_copies":17280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5760,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":304,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570110+0000","last_change":"2024-08-17T17:21:31.870419+0000","last_active":"2024-08-17T17:23:16.570110+0000","last_peered":"2024-08-17T17:23:16.570110+0000","last_clean":"2024-08-17T17:23:16.570110+0000","last_became_active":"2024-08-17T17:21:31.870030+0000","last_became_peered":"2024-08-17T17:21:31.870030+0000","last_unstale":"2024-08-17T17:23:16.570110+0000","last_undegraded":"2024-08-17T17:23:16.570110+0000","last_fullsized":"2024-08-17T17:23:16.570110+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:27.814200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":200,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687206+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:23:08.687206+0000","last_peered":"2024-08-17T17:23:08.687206+0000","last_clean":"2024-08-17T17:23:08.687206+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:23:08.687206+0000","last_undegraded":"2024-08-17T17:23:08.687206+0000","last_fullsized":"2024-08-17T17:23:08.687206+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":327,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.592974+0000","last_change":"2024-08-17T17:21:31.878045+0000","last_active":"2024-08-17T17:23:16.592974+0000","last_peered":"2024-08-17T17:23:16.592974+0000","last_clean":"2024-08-17T17:23:16.592974+0000","last_became_active":"2024-08-17T17:21:31.877807+0000","last_became_peered":"2024-08-17T17:21:31.877807+0000","last_unstale":"2024-08-17T17:23:16.592974+0000","last_undegraded":"2024-08-17T17:23:16.592974+0000","last_fullsized":"2024-08-17T17:23:16.592974+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:18.227640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"233'6022","reported_seq":6041,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:27.291724+0000","last_change":"2024-08-17T17:22:18.541323+0000","last_active":"2024-08-17T17:23:27.291724+0000","last_peered":"2024-08-17T17:23:27.291724+0000","last_clean":"2024-08-17T17:23:27.291724+0000","last_became_active":"2024-08-17T17:22:18.541054+0000","last_became_peered":"2024-08-17T17:22:18.541054+0000","last_unstale":"2024-08-17T17:23:27.291724+0000","last_undegraded":"2024-08-17T17:23:27.291724+0000","last_fullsized":"2024-08-17T17:23:27.291724+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6022,"log_dups_size":0,"ondisk_log_size":6022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:46.052040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394657792,"num_objects":6022,"num_object_clones":0,"num_object_copies":18066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6022,"num_write_kb":385408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":280,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572964+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:23:16.572964+0000","last_peered":"2024-08-17T17:23:16.572964+0000","last_clean":"2024-08-17T17:23:16.572964+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:23:16.572964+0000","last_undegraded":"2024-08-17T17:23:16.572964+0000","last_fullsized":"2024-08-17T17:23:16.572964+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":293,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570652+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:23:16.570652+0000","last_peered":"2024-08-17T17:23:16.570652+0000","last_clean":"2024-08-17T17:23:16.570652+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:23:16.570652+0000","last_undegraded":"2024-08-17T17:23:16.570652+0000","last_fullsized":"2024-08-17T17:23:16.570652+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":278,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570509+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:23:16.570509+0000","last_peered":"2024-08-17T17:23:16.570509+0000","last_clean":"2024-08-17T17:23:16.570509+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:23:16.570509+0000","last_undegraded":"2024-08-17T17:23:16.570509+0000","last_fullsized":"2024-08-17T17:23:16.570509+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"233'5801","reported_seq":5820,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.593081+0000","last_change":"2024-08-17T17:22:18.545136+0000","last_active":"2024-08-17T17:23:22.593081+0000","last_peered":"2024-08-17T17:23:22.593081+0000","last_clean":"2024-08-17T17:23:22.593081+0000","last_became_active":"2024-08-17T17:22:18.544176+0000","last_became_peered":"2024-08-17T17:22:18.544176+0000","last_unstale":"2024-08-17T17:23:22.593081+0000","last_undegraded":"2024-08-17T17:23:22.593081+0000","last_fullsized":"2024-08-17T17:23:22.593081+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":5801,"log_dups_size":0,"ondisk_log_size":5801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:08.787886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380174336,"num_objects":5801,"num_object_clones":0,"num_object_copies":17403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5801,"num_write_kb":371264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":331,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.590541+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:23:16.590541+0000","last_peered":"2024-08-17T17:23:16.590541+0000","last_clean":"2024-08-17T17:23:16.590541+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:23:16.590541+0000","last_undegraded":"2024-08-17T17:23:16.590541+0000","last_fullsized":"2024-08-17T17:23:16.590541+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":200,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690937+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:23:08.690937+0000","last_peered":"2024-08-17T17:23:08.690937+0000","last_clean":"2024-08-17T17:23:08.690937+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:23:08.690937+0000","last_undegraded":"2024-08-17T17:23:08.690937+0000","last_fullsized":"2024-08-17T17:23:08.690937+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689725+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:23:08.689725+0000","last_peered":"2024-08-17T17:23:08.689725+0000","last_clean":"2024-08-17T17:23:08.689725+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:23:08.689725+0000","last_undegraded":"2024-08-17T17:23:08.689725+0000","last_fullsized":"2024-08-17T17:23:08.689725+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":187,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689812+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:23:08.689812+0000","last_peered":"2024-08-17T17:23:08.689812+0000","last_clean":"2024-08-17T17:23:08.689812+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:23:08.689812+0000","last_undegraded":"2024-08-17T17:23:08.689812+0000","last_fullsized":"2024-08-17T17:23:08.689812+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":193,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690385+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:23:08.690385+0000","last_peered":"2024-08-17T17:23:08.690385+0000","last_clean":"2024-08-17T17:23:08.690385+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:23:08.690385+0000","last_undegraded":"2024-08-17T17:23:08.690385+0000","last_fullsized":"2024-08-17T17:23:08.690385+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":293,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570433+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:23:16.570433+0000","last_peered":"2024-08-17T17:23:16.570433+0000","last_clean":"2024-08-17T17:23:16.570433+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:23:16.570433+0000","last_undegraded":"2024-08-17T17:23:16.570433+0000","last_fullsized":"2024-08-17T17:23:16.570433+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687018+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:23:08.687018+0000","last_peered":"2024-08-17T17:23:08.687018+0000","last_clean":"2024-08-17T17:23:08.687018+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:23:08.687018+0000","last_undegraded":"2024-08-17T17:23:08.687018+0000","last_fullsized":"2024-08-17T17:23:08.687018+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":131,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.687339+0000","last_change":"2024-08-17T17:21:31.909108+0000","last_active":"2024-08-17T17:23:08.687339+0000","last_peered":"2024-08-17T17:23:08.687339+0000","last_clean":"2024-08-17T17:23:08.687339+0000","last_became_active":"2024-08-17T17:21:31.908882+0000","last_became_peered":"2024-08-17T17:21:31.908882+0000","last_unstale":"2024-08-17T17:23:08.687339+0000","last_undegraded":"2024-08-17T17:23:08.687339+0000","last_fullsized":"2024-08-17T17:23:08.687339+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:24.197352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":240,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.571144+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:23:16.571144+0000","last_peered":"2024-08-17T17:23:16.571144+0000","last_clean":"2024-08-17T17:23:16.571144+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:23:16.571144+0000","last_undegraded":"2024-08-17T17:23:16.571144+0000","last_fullsized":"2024-08-17T17:23:16.571144+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":337,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.588823+0000","last_change":"2024-08-17T17:21:31.882662+0000","last_active":"2024-08-17T17:23:16.588823+0000","last_peered":"2024-08-17T17:23:16.588823+0000","last_clean":"2024-08-17T17:23:16.588823+0000","last_became_active":"2024-08-17T17:21:31.882374+0000","last_became_peered":"2024-08-17T17:21:31.882374+0000","last_unstale":"2024-08-17T17:23:16.588823+0000","last_undegraded":"2024-08-17T17:23:16.588823+0000","last_fullsized":"2024-08-17T17:23:16.588823+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:31.311847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":252,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570414+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:23:16.570414+0000","last_peered":"2024-08-17T17:23:16.570414+0000","last_clean":"2024-08-17T17:23:16.570414+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:23:16.570414+0000","last_undegraded":"2024-08-17T17:23:16.570414+0000","last_fullsized":"2024-08-17T17:23:16.570414+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690864+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:23:08.690864+0000","last_peered":"2024-08-17T17:23:08.690864+0000","last_clean":"2024-08-17T17:23:08.690864+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:23:08.690864+0000","last_undegraded":"2024-08-17T17:23:08.690864+0000","last_fullsized":"2024-08-17T17:23:08.690864+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":270,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.570594+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:23:16.570594+0000","last_peered":"2024-08-17T17:23:16.570594+0000","last_clean":"2024-08-17T17:23:16.570594+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:23:16.570594+0000","last_undegraded":"2024-08-17T17:23:16.570594+0000","last_fullsized":"2024-08-17T17:23:16.570594+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":328,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.590375+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:23:16.590375+0000","last_peered":"2024-08-17T17:23:16.590375+0000","last_clean":"2024-08-17T17:23:16.590375+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:23:16.590375+0000","last_undegraded":"2024-08-17T17:23:16.590375+0000","last_fullsized":"2024-08-17T17:23:16.590375+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":292,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.569936+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:23:16.569936+0000","last_peered":"2024-08-17T17:23:16.569936+0000","last_clean":"2024-08-17T17:23:16.569936+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:23:16.569936+0000","last_undegraded":"2024-08-17T17:23:16.569936+0000","last_fullsized":"2024-08-17T17:23:16.569936+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":252,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.572528+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:23:16.572528+0000","last_peered":"2024-08-17T17:23:16.572528+0000","last_clean":"2024-08-17T17:23:16.572528+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:23:16.572528+0000","last_undegraded":"2024-08-17T17:23:16.572528+0000","last_fullsized":"2024-08-17T17:23:16.572528+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":206,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689819+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:23:08.689819+0000","last_peered":"2024-08-17T17:23:08.689819+0000","last_clean":"2024-08-17T17:23:08.689819+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:23:08.689819+0000","last_undegraded":"2024-08-17T17:23:08.689819+0000","last_fullsized":"2024-08-17T17:23:08.689819+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":316,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.582552+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:23:16.582552+0000","last_peered":"2024-08-17T17:23:16.582552+0000","last_clean":"2024-08-17T17:23:16.582552+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:23:16.582552+0000","last_undegraded":"2024-08-17T17:23:16.582552+0000","last_fullsized":"2024-08-17T17:23:16.582552+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":319,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.593069+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:23:16.593069+0000","last_peered":"2024-08-17T17:23:16.593069+0000","last_clean":"2024-08-17T17:23:16.593069+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:23:16.593069+0000","last_undegraded":"2024-08-17T17:23:16.593069+0000","last_fullsized":"2024-08-17T17:23:16.593069+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":214,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689778+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:23:08.689778+0000","last_peered":"2024-08-17T17:23:08.689778+0000","last_clean":"2024-08-17T17:23:08.689778+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:23:08.689778+0000","last_undegraded":"2024-08-17T17:23:08.689778+0000","last_fullsized":"2024-08-17T17:23:08.689778+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":87,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690303+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:23:08.690303+0000","last_peered":"2024-08-17T17:23:08.690303+0000","last_clean":"2024-08-17T17:23:08.690303+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:23:08.690303+0000","last_undegraded":"2024-08-17T17:23:08.690303+0000","last_fullsized":"2024-08-17T17:23:08.690303+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":204,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689687+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:23:08.689687+0000","last_peered":"2024-08-17T17:23:08.689687+0000","last_clean":"2024-08-17T17:23:08.689687+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:23:08.689687+0000","last_undegraded":"2024-08-17T17:23:08.689687+0000","last_fullsized":"2024-08-17T17:23:08.689687+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":87,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.690114+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:23:08.690114+0000","last_peered":"2024-08-17T17:23:08.690114+0000","last_clean":"2024-08-17T17:23:08.690114+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:23:08.690114+0000","last_undegraded":"2024-08-17T17:23:08.690114+0000","last_fullsized":"2024-08-17T17:23:08.690114+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":283,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.571406+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:23:16.571406+0000","last_peered":"2024-08-17T17:23:16.571406+0000","last_clean":"2024-08-17T17:23:16.571406+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:23:16.571406+0000","last_undegraded":"2024-08-17T17:23:16.571406+0000","last_fullsized":"2024-08-17T17:23:16.571406+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":269,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.571942+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:23:16.571942+0000","last_peered":"2024-08-17T17:23:16.571942+0000","last_clean":"2024-08-17T17:23:16.571942+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:23:16.571942+0000","last_undegraded":"2024-08-17T17:23:16.571942+0000","last_fullsized":"2024-08-17T17:23:16.571942+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":187,"reported_epoch":232,"state":"active+clean","last_fresh":"2024-08-17T17:23:08.689498+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:23:08.689498+0000","last_peered":"2024-08-17T17:23:08.689498+0000","last_clean":"2024-08-17T17:23:08.689498+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:23:08.689498+0000","last_undegraded":"2024-08-17T17:23:08.689498+0000","last_fullsized":"2024-08-17T17:23:08.689498+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":342,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.588534+0000","last_change":"2024-08-17T17:21:31.882968+0000","last_active":"2024-08-17T17:23:16.588534+0000","last_peered":"2024-08-17T17:23:16.588534+0000","last_clean":"2024-08-17T17:23:16.588534+0000","last_became_active":"2024-08-17T17:21:31.882438+0000","last_became_peered":"2024-08-17T17:21:31.882438+0000","last_unstale":"2024-08-17T17:23:16.588534+0000","last_undegraded":"2024-08-17T17:23:16.588534+0000","last_fullsized":"2024-08-17T17:23:16.588534+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:05.660809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6122962944,"num_objects":93429,"num_object_clones":0,"num_object_copies":280287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":93429,"num_write_kb":5979456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18506055680,"data_stored":18506055680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93429,"ondisk_log_size":93429,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201617,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2372056,"kb_used_data":1462416,"kb_used_omap":1,"kb_used_meta":909630,"kb_avail":91373096,"statfs":{"total":95995035648,"available":93566050304,"internally_reserved":0,"allocated":1497513984,"data_stored":1496362925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":931461464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463253,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3435472,"kb_used_data":2529288,"kb_used_omap":1,"kb_used_meta":906174,"kb_avail":90309680,"statfs":{"total":95995035648,"available":92477112320,"internally_reserved":0,"allocated":2589990912,"data_stored":2588847902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927922520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698491928,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2284028,"kb_used_data":1888692,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":91461124,"statfs":{"total":95995035648,"available":93656190976,"internally_reserved":0,"allocated":1934020608,"data_stored":1932892310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986528,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3967180,"kb_used_data":3070404,"kb_used_omap":1,"kb_used_meta":896766,"kb_avail":89777972,"statfs":{"total":95995035648,"available":91932643328,"internally_reserved":0,"allocated":3144093696,"data_stored":3142954038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918288728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225347,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2344952,"kb_used_data":1857456,"kb_used_omap":4,"kb_used_meta":487483,"kb_avail":91400200,"statfs":{"total":95995035648,"available":93593804800,"internally_reserved":0,"allocated":1902034944,"data_stored":1900910970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":499183040},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075672,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2563008,"kb_used_data":2215544,"kb_used_omap":6,"kb_used_meta":347449,"kb_avail":91182144,"statfs":{"total":95995035648,"available":93370515456,"internally_reserved":0,"allocated":2268717056,"data_stored":2267585685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":355788219},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533565,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1771448,"kb_used_data":1131952,"kb_used_omap":5,"kb_used_meta":639482,"kb_avail":91973704,"statfs":{"total":95995035648,"available":94181072896,"internally_reserved":0,"allocated":1159118848,"data_stored":1157993664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":654829808},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089542,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4475716,"kb_used_data":3858940,"kb_used_omap":5,"kb_used_meta":616762,"kb_avail":89269436,"statfs":{"total":95995035648,"available":91411902464,"internally_reserved":0,"allocated":3951554560,"data_stored":3950419633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":631564347},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3952541696,"data_stored":3952541696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1165099008,"data_stored":1165099008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2283864064,"data_stored":2283864064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1904476160,"data_stored":1904476160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3172728832,"data_stored":3172728832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1905524736,"data_stored":1905524736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2613379072,"data_stored":2613379072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1508442112,"data_stored":1508442112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:23:29.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:23:29.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:29.457 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:29 smithi142 ceph-mon[103614]: pgmap v759: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:23:29.458 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/406303412' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:29.458 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:29.458 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:29 smithi142 ceph-mon[106729]: pgmap v759: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:23:29.458 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/406303412' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:29.458 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:29.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:29.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:29.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:29 smithi152 ceph-mon[93456]: pgmap v759: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:23:29.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/406303412' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:29.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:29.577 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 99199 99183 105.048 66.875 0.0138131 0.0095135 2024-08-17T17:23:29.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:29.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:29.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:29.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:29.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:29.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:29.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:30.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:30.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:30.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:30.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:30.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[103614]: from='client.36357 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[103614]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T17:23:30.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[103614]: pgmap v761: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[106729]: from='client.36357 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[106729]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T17:23:30.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:30 smithi142 ceph-mon[106729]: pgmap v761: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:30.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:30.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:30.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:30.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:30 smithi152 ceph-mon[93456]: from='client.36357 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:30 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:30.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:30 smithi152 ceph-mon[93456]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T17:23:30.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:30 smithi152 ceph-mon[93456]: pgmap v761: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:30.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:23:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:23:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:23:30.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:23:30.577978+0000 min lat: 0.00124672 max lat: 0.4126 avg lat: 0.00957217 2024-08-17T17:23:30.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:23:30.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 15 100273 100258 104.417 67.1875 0.0160122 0.00957217 2024-08-17T17:23:30.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:30.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:30.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:30.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:30.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:30.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:30.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:31.050 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:31.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:31.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:31.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:31.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:31.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:31.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:31.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:31.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:31.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:31.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 101313 101297 103.77 64.9375 0.0211221 0.00963232 2024-08-17T17:23:31.595 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:31.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:31.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:31.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:31.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:31.851 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:23:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:23:31] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.51.0" 2024-08-17T17:23:31.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:31.898 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:31.899 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:31.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:32.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:32.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:23:32.039 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:32.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:32.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:32.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:32.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:32.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:32.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:32.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:32.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:32.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 102334 102318 103.125 63.8125 0.00619126 0.00969199 2024-08-17T17:23:32.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:32.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:32.748 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:32.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:32 smithi152 ceph-mon[93456]: pgmap v762: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:32.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:32 smithi142 ceph-mon[103614]: pgmap v762: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:32 smithi142 ceph-mon[106729]: pgmap v762: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.883 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:32.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:32.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:33.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:33.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:33.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:33.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:33.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:33.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:33.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:33.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:33.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:33.573 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:33.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:33.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 103445 103429 102.59 69.4375 0.0233657 0.00974282 2024-08-17T17:23:33.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:33.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:33.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:33.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:33.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:33.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:33.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:34.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:34.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:23:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:34.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:34.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:34.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:34.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:34.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:34.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:34.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 104498 104482 102.016 65.8125 0.0155723 0.00979676 2024-08-17T17:23:34.585 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:34.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:34.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:34.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:34.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:34.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:34.805 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:34 smithi142 ceph-mon[103614]: pgmap v763: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:34.806 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:34 smithi142 ceph-mon[106729]: pgmap v763: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:34 smithi152 ceph-mon[93456]: pgmap v763: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:23:34.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:34.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:34.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:35.033 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:35.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:35.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:35.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:35.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:35.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:35.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:35.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:35.579 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 105568 105552 101.475 66.875 0.0220374 0.00984954 2024-08-17T17:23:35.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:35.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:35.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:35.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:35.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:35.836 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:35.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:35.864 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:35.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:35.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:36.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:36.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:36.167 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:23:36.170 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:23:36.220 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:36.270 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:36.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:36.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:36.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:36.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:36.578 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 106532 106516 100.85 60.25 0.0111433 0.00991057 2024-08-17T17:23:36.651 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:36 smithi142 ceph-mon[106729]: pgmap v764: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:23:36.651 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:36 smithi142 ceph-mon[106729]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:36.651 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:36 smithi142 ceph-mon[103614]: pgmap v764: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:23:36.651 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:36 smithi142 ceph-mon[103614]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:36.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:36.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:36.799 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:36 smithi152 ceph-mon[93456]: pgmap v764: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:23:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:36 smithi152 ceph-mon[93456]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:36.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:36.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:36.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.067 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:37.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:37.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:37.152 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":764,"stamp":"2024-08-17T17:23:35.499989+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6709450701,"num_objects":102395,"num_object_clones":0,"num_object_copies":307185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102395,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":102647,"num_write_kb":6554063,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102576,"ondisk_log_size":102576,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24829640,"kb_used_data":19655428,"kb_used_omap":35,"kb_used_meta":5174108,"kb_avail":725131576,"statfs":{"total":767960285184,"available":742534733824,"internally_reserved":0,"allocated":20127158272,"data_stored":20118048471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36357,"internal_metadata":5298287099},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":722731008,"num_objects":11028,"num_object_clones":0,"num_object_copies":33084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11028,"num_write_kb":705792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006626"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":278,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.218180+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:23:29.218180+0000","last_peered":"2024-08-17T17:23:29.218180+0000","last_clean":"2024-08-17T17:23:29.218180+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:23:29.218180+0000","last_undegraded":"2024-08-17T17:23:29.218180+0000","last_fullsized":"2024-08-17T17:23:29.218180+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":191,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578259+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:23:16.578259+0000","last_peered":"2024-08-17T17:23:16.578259+0000","last_clean":"2024-08-17T17:23:16.578259+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:23:16.578259+0000","last_undegraded":"2024-08-17T17:23:16.578259+0000","last_fullsized":"2024-08-17T17:23:16.578259+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":171,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578415+0000","last_change":"2024-08-17T17:21:31.903762+0000","last_active":"2024-08-17T17:23:16.578415+0000","last_peered":"2024-08-17T17:23:16.578415+0000","last_clean":"2024-08-17T17:23:16.578415+0000","last_became_active":"2024-08-17T17:21:31.902518+0000","last_became_peered":"2024-08-17T17:21:31.902518+0000","last_unstale":"2024-08-17T17:23:16.578415+0000","last_undegraded":"2024-08-17T17:23:16.578415+0000","last_fullsized":"2024-08-17T17:23:16.578415+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:13.790840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":329,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.225195+0000","last_change":"2024-08-17T17:12:40.341796+0000","last_active":"2024-08-17T17:23:29.225195+0000","last_peered":"2024-08-17T17:23:29.225195+0000","last_clean":"2024-08-17T17:23:29.225195+0000","last_became_active":"2024-08-17T17:12:40.341498+0000","last_became_peered":"2024-08-17T17:12:40.341498+0000","last_unstale":"2024-08-17T17:23:29.225195+0000","last_undegraded":"2024-08-17T17:23:29.225195+0000","last_fullsized":"2024-08-17T17:23:29.225195+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:26.243549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":328,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221396+0000","last_change":"2024-08-17T17:12:40.344688+0000","last_active":"2024-08-17T17:23:29.221396+0000","last_peered":"2024-08-17T17:23:29.221396+0000","last_clean":"2024-08-17T17:23:29.221396+0000","last_became_active":"2024-08-17T17:12:40.343535+0000","last_became_peered":"2024-08-17T17:12:40.343535+0000","last_unstale":"2024-08-17T17:23:29.221396+0000","last_undegraded":"2024-08-17T17:23:29.221396+0000","last_fullsized":"2024-08-17T17:23:29.221396+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:19:20.445608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":172,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.577943+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:23:16.577943+0000","last_peered":"2024-08-17T17:23:16.577943+0000","last_clean":"2024-08-17T17:23:16.577943+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:23:16.577943+0000","last_undegraded":"2024-08-17T17:23:16.577943+0000","last_fullsized":"2024-08-17T17:23:16.577943+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":254,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.228586+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:23:29.228586+0000","last_peered":"2024-08-17T17:23:29.228586+0000","last_clean":"2024-08-17T17:23:29.228586+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:23:29.228586+0000","last_undegraded":"2024-08-17T17:23:29.228586+0000","last_fullsized":"2024-08-17T17:23:29.228586+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":214,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575869+0000","last_change":"2024-08-17T17:21:31.868761+0000","last_active":"2024-08-17T17:23:16.575869+0000","last_peered":"2024-08-17T17:23:16.575869+0000","last_clean":"2024-08-17T17:23:16.575869+0000","last_became_active":"2024-08-17T17:21:31.868430+0000","last_became_peered":"2024-08-17T17:21:31.868430+0000","last_unstale":"2024-08-17T17:23:16.575869+0000","last_undegraded":"2024-08-17T17:23:16.575869+0000","last_fullsized":"2024-08-17T17:23:16.575869+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:35:33.534743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"234'6305","reported_seq":6340,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:31.767836+0000","last_change":"2024-08-17T17:22:18.545386+0000","last_active":"2024-08-17T17:23:31.767836+0000","last_peered":"2024-08-17T17:23:31.767836+0000","last_clean":"2024-08-17T17:23:31.767836+0000","last_became_active":"2024-08-17T17:22:18.545116+0000","last_became_peered":"2024-08-17T17:22:18.545116+0000","last_unstale":"2024-08-17T17:23:31.767836+0000","last_undegraded":"2024-08-17T17:23:31.767836+0000","last_fullsized":"2024-08-17T17:23:31.767836+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:13:37.764971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413204480,"num_objects":6305,"num_object_clones":0,"num_object_copies":18915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6305,"num_write_kb":403520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":190,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578504+0000","last_change":"2024-08-17T17:07:07.433917+0000","last_active":"2024-08-17T17:23:16.578504+0000","last_peered":"2024-08-17T17:23:16.578504+0000","last_clean":"2024-08-17T17:23:16.578504+0000","last_became_active":"2024-08-17T17:07:07.433466+0000","last_became_peered":"2024-08-17T17:07:07.433466+0000","last_unstale":"2024-08-17T17:23:16.578504+0000","last_undegraded":"2024-08-17T17:23:16.578504+0000","last_fullsized":"2024-08-17T17:23:16.578504+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:52.835881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578102+0000","last_change":"2024-08-17T17:21:31.876798+0000","last_active":"2024-08-17T17:23:16.578102+0000","last_peered":"2024-08-17T17:23:16.578102+0000","last_clean":"2024-08-17T17:23:16.578102+0000","last_became_active":"2024-08-17T17:21:31.876370+0000","last_became_peered":"2024-08-17T17:21:31.876370+0000","last_unstale":"2024-08-17T17:23:16.578102+0000","last_undegraded":"2024-08-17T17:23:16.578102+0000","last_fullsized":"2024-08-17T17:23:16.578102+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:33:16.352719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"234'6381","reported_seq":6400,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.601274+0000","last_change":"2024-08-17T17:22:18.544826+0000","last_active":"2024-08-17T17:23:32.601274+0000","last_peered":"2024-08-17T17:23:32.601274+0000","last_clean":"2024-08-17T17:23:32.601274+0000","last_became_active":"2024-08-17T17:22:18.544566+0000","last_became_peered":"2024-08-17T17:22:18.544566+0000","last_unstale":"2024-08-17T17:23:32.601274+0000","last_undegraded":"2024-08-17T17:23:32.601274+0000","last_fullsized":"2024-08-17T17:23:32.601274+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6381,"log_dups_size":0,"ondisk_log_size":6381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:08.783581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418185216,"num_objects":6381,"num_object_clones":0,"num_object_copies":19143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6381,"num_write_kb":408384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":191,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.579026+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:23:16.579026+0000","last_peered":"2024-08-17T17:23:16.579026+0000","last_clean":"2024-08-17T17:23:16.579026+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:23:16.579026+0000","last_undegraded":"2024-08-17T17:23:16.579026+0000","last_fullsized":"2024-08-17T17:23:16.579026+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":205,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575420+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:23:16.575420+0000","last_peered":"2024-08-17T17:23:16.575420+0000","last_clean":"2024-08-17T17:23:16.575420+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:23:16.575420+0000","last_undegraded":"2024-08-17T17:23:16.575420+0000","last_fullsized":"2024-08-17T17:23:16.575420+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"234'6372","reported_seq":6407,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:31.896708+0000","last_change":"2024-08-17T17:22:18.545432+0000","last_active":"2024-08-17T17:23:31.896708+0000","last_peered":"2024-08-17T17:23:31.896708+0000","last_clean":"2024-08-17T17:23:31.896708+0000","last_became_active":"2024-08-17T17:22:18.544999+0000","last_became_peered":"2024-08-17T17:22:18.544999+0000","last_unstale":"2024-08-17T17:23:31.896708+0000","last_undegraded":"2024-08-17T17:23:31.896708+0000","last_fullsized":"2024-08-17T17:23:31.896708+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6372,"log_dups_size":0,"ondisk_log_size":6372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:54.714976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417595392,"num_objects":6372,"num_object_clones":0,"num_object_copies":19116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6372,"num_write_kb":407808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":190,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578662+0000","last_change":"2024-08-17T17:21:31.904014+0000","last_active":"2024-08-17T17:23:16.578662+0000","last_peered":"2024-08-17T17:23:16.578662+0000","last_clean":"2024-08-17T17:23:16.578662+0000","last_became_active":"2024-08-17T17:21:31.903128+0000","last_became_peered":"2024-08-17T17:21:31.903128+0000","last_unstale":"2024-08-17T17:23:16.578662+0000","last_undegraded":"2024-08-17T17:23:16.578662+0000","last_fullsized":"2024-08-17T17:23:16.578662+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:56:59.504309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":188,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575909+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:23:16.575909+0000","last_peered":"2024-08-17T17:23:16.575909+0000","last_clean":"2024-08-17T17:23:16.575909+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:23:16.575909+0000","last_undegraded":"2024-08-17T17:23:16.575909+0000","last_fullsized":"2024-08-17T17:23:16.575909+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"234'6633","reported_seq":6668,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:33.923527+0000","last_change":"2024-08-17T17:22:18.542919+0000","last_active":"2024-08-17T17:23:33.923527+0000","last_peered":"2024-08-17T17:23:33.923527+0000","last_clean":"2024-08-17T17:23:33.923527+0000","last_became_active":"2024-08-17T17:22:18.542151+0000","last_became_peered":"2024-08-17T17:22:18.542151+0000","last_unstale":"2024-08-17T17:23:33.923527+0000","last_undegraded":"2024-08-17T17:23:33.923527+0000","last_fullsized":"2024-08-17T17:23:33.923527+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6633,"log_dups_size":0,"ondisk_log_size":6633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:14.269839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434700288,"num_objects":6633,"num_object_clones":0,"num_object_copies":19899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6633,"num_write_kb":424512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":215,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.218313+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:23:29.218313+0000","last_peered":"2024-08-17T17:23:29.218313+0000","last_clean":"2024-08-17T17:23:29.218313+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:23:29.218313+0000","last_undegraded":"2024-08-17T17:23:29.218313+0000","last_fullsized":"2024-08-17T17:23:29.218313+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":329,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220217+0000","last_change":"2024-08-17T17:07:51.484373+0000","last_active":"2024-08-17T17:23:29.220217+0000","last_peered":"2024-08-17T17:23:29.220217+0000","last_clean":"2024-08-17T17:23:29.220217+0000","last_became_active":"2024-08-17T17:07:51.484090+0000","last_became_peered":"2024-08-17T17:07:51.484090+0000","last_unstale":"2024-08-17T17:23:29.220217+0000","last_undegraded":"2024-08-17T17:23:29.220217+0000","last_fullsized":"2024-08-17T17:23:29.220217+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:10.597875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"234'6248","reported_seq":6283,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:30.825658+0000","last_change":"2024-08-17T17:22:18.546366+0000","last_active":"2024-08-17T17:23:30.825658+0000","last_peered":"2024-08-17T17:23:30.825658+0000","last_clean":"2024-08-17T17:23:30.825658+0000","last_became_active":"2024-08-17T17:22:18.545788+0000","last_became_peered":"2024-08-17T17:22:18.545788+0000","last_unstale":"2024-08-17T17:23:30.825658+0000","last_undegraded":"2024-08-17T17:23:30.825658+0000","last_fullsized":"2024-08-17T17:23:30.825658+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6248,"log_dups_size":0,"ondisk_log_size":6248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:13.373207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409468928,"num_objects":6248,"num_object_clones":0,"num_object_copies":18744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6248,"num_write_kb":399872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":282,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.217522+0000","last_change":"2024-08-17T17:12:40.342453+0000","last_active":"2024-08-17T17:23:29.217522+0000","last_peered":"2024-08-17T17:23:29.217522+0000","last_clean":"2024-08-17T17:23:29.217522+0000","last_became_active":"2024-08-17T17:12:40.341304+0000","last_became_peered":"2024-08-17T17:12:40.341304+0000","last_unstale":"2024-08-17T17:23:29.217522+0000","last_undegraded":"2024-08-17T17:23:29.217522+0000","last_fullsized":"2024-08-17T17:23:29.217522+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:36:31.270665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575627+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:23:16.575627+0000","last_peered":"2024-08-17T17:23:16.575627+0000","last_clean":"2024-08-17T17:23:16.575627+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:23:16.575627+0000","last_undegraded":"2024-08-17T17:23:16.575627+0000","last_fullsized":"2024-08-17T17:23:16.575627+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"234'6394","reported_seq":6429,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:33.895918+0000","last_change":"2024-08-17T17:22:18.541297+0000","last_active":"2024-08-17T17:23:33.895918+0000","last_peered":"2024-08-17T17:23:33.895918+0000","last_clean":"2024-08-17T17:23:33.895918+0000","last_became_active":"2024-08-17T17:22:18.541002+0000","last_became_peered":"2024-08-17T17:22:18.541002+0000","last_unstale":"2024-08-17T17:23:33.895918+0000","last_undegraded":"2024-08-17T17:23:33.895918+0000","last_fullsized":"2024-08-17T17:23:33.895918+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:07.861642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419037184,"num_objects":6394,"num_object_clones":0,"num_object_copies":19182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6394,"num_write_kb":409216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":189,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576040+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:23:16.576040+0000","last_peered":"2024-08-17T17:23:16.576040+0000","last_clean":"2024-08-17T17:23:16.576040+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:23:16.576040+0000","last_undegraded":"2024-08-17T17:23:16.576040+0000","last_fullsized":"2024-08-17T17:23:16.576040+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":205,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576531+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:23:16.576531+0000","last_peered":"2024-08-17T17:23:16.576531+0000","last_clean":"2024-08-17T17:23:16.576531+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:23:16.576531+0000","last_undegraded":"2024-08-17T17:23:16.576531+0000","last_fullsized":"2024-08-17T17:23:16.576531+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"234'6347","reported_seq":6367,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.610205+0000","last_change":"2024-08-17T17:22:18.545009+0000","last_active":"2024-08-17T17:23:32.610205+0000","last_peered":"2024-08-17T17:23:32.610205+0000","last_clean":"2024-08-17T17:23:32.610205+0000","last_became_active":"2024-08-17T17:22:18.544525+0000","last_became_peered":"2024-08-17T17:22:18.544525+0000","last_unstale":"2024-08-17T17:23:32.610205+0000","last_undegraded":"2024-08-17T17:23:32.610205+0000","last_fullsized":"2024-08-17T17:23:32.610205+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:35.215727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415956992,"num_objects":6347,"num_object_clones":0,"num_object_copies":19041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":406208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":189,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575578+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:23:16.575578+0000","last_peered":"2024-08-17T17:23:16.575578+0000","last_clean":"2024-08-17T17:23:16.575578+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:23:16.575578+0000","last_undegraded":"2024-08-17T17:23:16.575578+0000","last_fullsized":"2024-08-17T17:23:16.575578+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":188,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.581364+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:23:16.581364+0000","last_peered":"2024-08-17T17:23:16.581364+0000","last_clean":"2024-08-17T17:23:16.581364+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:23:16.581364+0000","last_undegraded":"2024-08-17T17:23:16.581364+0000","last_fullsized":"2024-08-17T17:23:16.581364+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"234'6455","reported_seq":6474,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.289501+0000","last_change":"2024-08-17T17:22:18.542069+0000","last_active":"2024-08-17T17:23:32.289501+0000","last_peered":"2024-08-17T17:23:32.289501+0000","last_clean":"2024-08-17T17:23:32.289501+0000","last_became_active":"2024-08-17T17:22:18.541290+0000","last_became_peered":"2024-08-17T17:22:18.541290+0000","last_unstale":"2024-08-17T17:23:32.289501+0000","last_undegraded":"2024-08-17T17:23:32.289501+0000","last_fullsized":"2024-08-17T17:23:32.289501+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6455,"log_dups_size":0,"ondisk_log_size":6455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:04.374938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423034880,"num_objects":6455,"num_object_clones":0,"num_object_copies":19365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6455,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":256,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216720+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:23:29.216720+0000","last_peered":"2024-08-17T17:23:29.216720+0000","last_clean":"2024-08-17T17:23:29.216720+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:23:29.216720+0000","last_undegraded":"2024-08-17T17:23:29.216720+0000","last_fullsized":"2024-08-17T17:23:29.216720+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":211,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.228693+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:23:29.228693+0000","last_peered":"2024-08-17T17:23:29.228693+0000","last_clean":"2024-08-17T17:23:29.228693+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:23:29.228693+0000","last_undegraded":"2024-08-17T17:23:29.228693+0000","last_fullsized":"2024-08-17T17:23:29.228693+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"234'6377","reported_seq":6397,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.587599+0000","last_change":"2024-08-17T17:22:18.547163+0000","last_active":"2024-08-17T17:23:32.587599+0000","last_peered":"2024-08-17T17:23:32.587599+0000","last_clean":"2024-08-17T17:23:32.587599+0000","last_became_active":"2024-08-17T17:22:18.546827+0000","last_became_peered":"2024-08-17T17:22:18.546827+0000","last_unstale":"2024-08-17T17:23:32.587599+0000","last_undegraded":"2024-08-17T17:23:32.587599+0000","last_fullsized":"2024-08-17T17:23:32.587599+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6377,"log_dups_size":0,"ondisk_log_size":6377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:11.384041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417923072,"num_objects":6377,"num_object_clones":0,"num_object_copies":19131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6377,"num_write_kb":408128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":529,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.579580+0000","last_change":"2024-08-17T17:07:07.441348+0000","last_active":"2024-08-17T17:23:16.579580+0000","last_peered":"2024-08-17T17:23:16.579580+0000","last_clean":"2024-08-17T17:23:16.579580+0000","last_became_active":"2024-08-17T17:07:07.440923+0000","last_became_peered":"2024-08-17T17:07:07.440923+0000","last_unstale":"2024-08-17T17:23:16.579580+0000","last_undegraded":"2024-08-17T17:23:16.579580+0000","last_fullsized":"2024-08-17T17:23:16.579580+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:48.606374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":130,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.219518+0000","last_change":"2024-08-17T17:21:31.878312+0000","last_active":"2024-08-17T17:23:29.219518+0000","last_peered":"2024-08-17T17:23:29.219518+0000","last_clean":"2024-08-17T17:23:29.219518+0000","last_became_active":"2024-08-17T17:21:31.877932+0000","last_became_peered":"2024-08-17T17:21:31.877932+0000","last_unstale":"2024-08-17T17:23:29.219518+0000","last_undegraded":"2024-08-17T17:23:29.219518+0000","last_fullsized":"2024-08-17T17:23:29.219518+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:17.773813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.217248+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:23:29.217248+0000","last_peered":"2024-08-17T17:23:29.217248+0000","last_clean":"2024-08-17T17:23:29.217248+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:23:29.217248+0000","last_undegraded":"2024-08-17T17:23:29.217248+0000","last_fullsized":"2024-08-17T17:23:29.217248+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":171,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576701+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:23:16.576701+0000","last_peered":"2024-08-17T17:23:16.576701+0000","last_clean":"2024-08-17T17:23:16.576701+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:23:16.576701+0000","last_undegraded":"2024-08-17T17:23:16.576701+0000","last_fullsized":"2024-08-17T17:23:16.576701+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"234'6314","reported_seq":6349,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:31.885778+0000","last_change":"2024-08-17T17:22:18.544196+0000","last_active":"2024-08-17T17:23:31.885778+0000","last_peered":"2024-08-17T17:23:31.885778+0000","last_clean":"2024-08-17T17:23:31.885778+0000","last_became_active":"2024-08-17T17:22:18.543865+0000","last_became_peered":"2024-08-17T17:22:18.543865+0000","last_unstale":"2024-08-17T17:23:31.885778+0000","last_undegraded":"2024-08-17T17:23:31.885778+0000","last_fullsized":"2024-08-17T17:23:31.885778+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6314,"log_dups_size":0,"ondisk_log_size":6314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:48.118547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413794304,"num_objects":6314,"num_object_clones":0,"num_object_copies":18942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6314,"num_write_kb":404096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":218,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578212+0000","last_change":"2024-08-17T17:07:51.485277+0000","last_active":"2024-08-17T17:23:16.578212+0000","last_peered":"2024-08-17T17:23:16.578212+0000","last_clean":"2024-08-17T17:23:16.578212+0000","last_became_active":"2024-08-17T17:07:51.484675+0000","last_became_peered":"2024-08-17T17:07:51.484675+0000","last_unstale":"2024-08-17T17:23:16.578212+0000","last_undegraded":"2024-08-17T17:23:16.578212+0000","last_fullsized":"2024-08-17T17:23:16.578212+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:09.825642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":329,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.219796+0000","last_change":"2024-08-17T17:12:40.342777+0000","last_active":"2024-08-17T17:23:29.219796+0000","last_peered":"2024-08-17T17:23:29.219796+0000","last_clean":"2024-08-17T17:23:29.219796+0000","last_became_active":"2024-08-17T17:12:40.342378+0000","last_became_peered":"2024-08-17T17:12:40.342378+0000","last_unstale":"2024-08-17T17:23:29.219796+0000","last_undegraded":"2024-08-17T17:23:29.219796+0000","last_fullsized":"2024-08-17T17:23:29.219796+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:55.805170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":340,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.219862+0000","last_change":"2024-08-17T17:10:55.312837+0000","last_active":"2024-08-17T17:23:29.219862+0000","last_peered":"2024-08-17T17:23:29.219862+0000","last_clean":"2024-08-17T17:23:29.219862+0000","last_became_active":"2024-08-17T17:10:55.312532+0000","last_became_peered":"2024-08-17T17:10:55.312532+0000","last_unstale":"2024-08-17T17:23:29.219862+0000","last_undegraded":"2024-08-17T17:23:29.219862+0000","last_fullsized":"2024-08-17T17:23:29.219862+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:19.754444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"234'6440","reported_seq":6475,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:33.903351+0000","last_change":"2024-08-17T17:22:18.540626+0000","last_active":"2024-08-17T17:23:33.903351+0000","last_peered":"2024-08-17T17:23:33.903351+0000","last_clean":"2024-08-17T17:23:33.903351+0000","last_became_active":"2024-08-17T17:22:18.540354+0000","last_became_peered":"2024-08-17T17:22:18.540354+0000","last_unstale":"2024-08-17T17:23:33.903351+0000","last_undegraded":"2024-08-17T17:23:33.903351+0000","last_fullsized":"2024-08-17T17:23:33.903351+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6440,"log_dups_size":0,"ondisk_log_size":6440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:36.914893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422051840,"num_objects":6440,"num_object_clones":0,"num_object_copies":19320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6440,"num_write_kb":412160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":315,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216459+0000","last_change":"2024-08-17T17:12:40.342329+0000","last_active":"2024-08-17T17:23:29.216459+0000","last_peered":"2024-08-17T17:23:29.216459+0000","last_clean":"2024-08-17T17:23:29.216459+0000","last_became_active":"2024-08-17T17:12:40.341988+0000","last_became_peered":"2024-08-17T17:12:40.341988+0000","last_unstale":"2024-08-17T17:23:29.216459+0000","last_undegraded":"2024-08-17T17:23:29.216459+0000","last_fullsized":"2024-08-17T17:23:29.216459+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:32.279916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":295,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221098+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:23:29.221098+0000","last_peered":"2024-08-17T17:23:29.221098+0000","last_clean":"2024-08-17T17:23:29.221098+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:23:29.221098+0000","last_undegraded":"2024-08-17T17:23:29.221098+0000","last_fullsized":"2024-08-17T17:23:29.221098+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":332,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220937+0000","last_change":"2024-08-17T17:21:31.880043+0000","last_active":"2024-08-17T17:23:29.220937+0000","last_peered":"2024-08-17T17:23:29.220937+0000","last_clean":"2024-08-17T17:23:29.220937+0000","last_became_active":"2024-08-17T17:21:31.879787+0000","last_became_peered":"2024-08-17T17:21:31.879787+0000","last_unstale":"2024-08-17T17:23:29.220937+0000","last_undegraded":"2024-08-17T17:23:29.220937+0000","last_fullsized":"2024-08-17T17:23:29.220937+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:29.293588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"234'6348","reported_seq":6368,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.601511+0000","last_change":"2024-08-17T17:22:18.544563+0000","last_active":"2024-08-17T17:23:32.601511+0000","last_peered":"2024-08-17T17:23:32.601511+0000","last_clean":"2024-08-17T17:23:32.601511+0000","last_became_active":"2024-08-17T17:22:18.544243+0000","last_became_peered":"2024-08-17T17:22:18.544243+0000","last_unstale":"2024-08-17T17:23:32.601511+0000","last_undegraded":"2024-08-17T17:23:32.601511+0000","last_fullsized":"2024-08-17T17:23:32.601511+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6348,"log_dups_size":0,"ondisk_log_size":6348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:36.074516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416022528,"num_objects":6348,"num_object_clones":0,"num_object_copies":19044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6348,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":137,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.222030+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:23:29.222030+0000","last_peered":"2024-08-17T17:23:29.222030+0000","last_clean":"2024-08-17T17:23:29.222030+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:23:29.222030+0000","last_undegraded":"2024-08-17T17:23:29.222030+0000","last_fullsized":"2024-08-17T17:23:29.222030+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":189,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575937+0000","last_change":"2024-08-17T17:07:07.425869+0000","last_active":"2024-08-17T17:23:16.575937+0000","last_peered":"2024-08-17T17:23:16.575937+0000","last_clean":"2024-08-17T17:23:16.575937+0000","last_became_active":"2024-08-17T17:07:07.425449+0000","last_became_peered":"2024-08-17T17:07:07.425449+0000","last_unstale":"2024-08-17T17:23:16.575937+0000","last_undegraded":"2024-08-17T17:23:16.575937+0000","last_fullsized":"2024-08-17T17:23:16.575937+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:50.086066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":307,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220992+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:23:29.220992+0000","last_peered":"2024-08-17T17:23:29.220992+0000","last_clean":"2024-08-17T17:23:29.220992+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:23:29.220992+0000","last_undegraded":"2024-08-17T17:23:29.220992+0000","last_fullsized":"2024-08-17T17:23:29.220992+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"234'6498","reported_seq":6518,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.917496+0000","last_change":"2024-08-17T17:22:18.542803+0000","last_active":"2024-08-17T17:23:32.917496+0000","last_peered":"2024-08-17T17:23:32.917496+0000","last_clean":"2024-08-17T17:23:32.917496+0000","last_became_active":"2024-08-17T17:22:18.542476+0000","last_became_peered":"2024-08-17T17:22:18.542476+0000","last_unstale":"2024-08-17T17:23:32.917496+0000","last_undegraded":"2024-08-17T17:23:32.917496+0000","last_fullsized":"2024-08-17T17:23:32.917496+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6498,"log_dups_size":0,"ondisk_log_size":6498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:07:11.840777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425852928,"num_objects":6498,"num_object_clones":0,"num_object_copies":19494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6498,"num_write_kb":415872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":232,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.217884+0000","last_change":"2024-08-17T17:21:31.870010+0000","last_active":"2024-08-17T17:23:29.217884+0000","last_peered":"2024-08-17T17:23:29.217884+0000","last_clean":"2024-08-17T17:23:29.217884+0000","last_became_active":"2024-08-17T17:21:31.869696+0000","last_became_peered":"2024-08-17T17:21:31.869696+0000","last_unstale":"2024-08-17T17:23:29.217884+0000","last_undegraded":"2024-08-17T17:23:29.217884+0000","last_fullsized":"2024-08-17T17:23:29.217884+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:57.550019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":334,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221163+0000","last_change":"2024-08-17T17:07:51.486802+0000","last_active":"2024-08-17T17:23:29.221163+0000","last_peered":"2024-08-17T17:23:29.221163+0000","last_clean":"2024-08-17T17:23:29.221163+0000","last_became_active":"2024-08-17T17:07:51.486283+0000","last_became_peered":"2024-08-17T17:07:51.486283+0000","last_unstale":"2024-08-17T17:23:29.221163+0000","last_undegraded":"2024-08-17T17:23:29.221163+0000","last_fullsized":"2024-08-17T17:23:29.221163+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:50.326682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":296,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.222117+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:23:29.222117+0000","last_peered":"2024-08-17T17:23:29.222117+0000","last_clean":"2024-08-17T17:23:29.222117+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:23:29.222117+0000","last_undegraded":"2024-08-17T17:23:29.222117+0000","last_fullsized":"2024-08-17T17:23:29.222117+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"234'6423","reported_seq":6443,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.593276+0000","last_change":"2024-08-17T17:22:18.545052+0000","last_active":"2024-08-17T17:23:32.593276+0000","last_peered":"2024-08-17T17:23:32.593276+0000","last_clean":"2024-08-17T17:23:32.593276+0000","last_became_active":"2024-08-17T17:22:18.544251+0000","last_became_peered":"2024-08-17T17:22:18.544251+0000","last_unstale":"2024-08-17T17:23:32.593276+0000","last_undegraded":"2024-08-17T17:23:32.593276+0000","last_fullsized":"2024-08-17T17:23:32.593276+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:40.762667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420937728,"num_objects":6423,"num_object_clones":0,"num_object_copies":19269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6423,"num_write_kb":411072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":306,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221076+0000","last_change":"2024-08-17T17:21:31.870419+0000","last_active":"2024-08-17T17:23:29.221076+0000","last_peered":"2024-08-17T17:23:29.221076+0000","last_clean":"2024-08-17T17:23:29.221076+0000","last_became_active":"2024-08-17T17:21:31.870030+0000","last_became_peered":"2024-08-17T17:21:31.870030+0000","last_unstale":"2024-08-17T17:23:29.221076+0000","last_undegraded":"2024-08-17T17:23:29.221076+0000","last_fullsized":"2024-08-17T17:23:29.221076+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:27.814200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":201,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578732+0000","last_change":"2024-08-17T17:07:07.434002+0000","last_active":"2024-08-17T17:23:16.578732+0000","last_peered":"2024-08-17T17:23:16.578732+0000","last_clean":"2024-08-17T17:23:16.578732+0000","last_became_active":"2024-08-17T17:07:07.433393+0000","last_became_peered":"2024-08-17T17:07:07.433393+0000","last_unstale":"2024-08-17T17:23:16.578732+0000","last_undegraded":"2024-08-17T17:23:16.578732+0000","last_fullsized":"2024-08-17T17:23:16.578732+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:56.936747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":329,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220319+0000","last_change":"2024-08-17T17:21:31.878045+0000","last_active":"2024-08-17T17:23:29.220319+0000","last_peered":"2024-08-17T17:23:29.220319+0000","last_clean":"2024-08-17T17:23:29.220319+0000","last_became_active":"2024-08-17T17:21:31.877807+0000","last_became_peered":"2024-08-17T17:21:31.877807+0000","last_unstale":"2024-08-17T17:23:29.220319+0000","last_undegraded":"2024-08-17T17:23:29.220319+0000","last_fullsized":"2024-08-17T17:23:29.220319+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:18.227640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"234'6328","reported_seq":6348,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.286671+0000","last_change":"2024-08-17T17:22:18.541323+0000","last_active":"2024-08-17T17:23:32.286671+0000","last_peered":"2024-08-17T17:23:32.286671+0000","last_clean":"2024-08-17T17:23:32.286671+0000","last_became_active":"2024-08-17T17:22:18.541054+0000","last_became_peered":"2024-08-17T17:22:18.541054+0000","last_unstale":"2024-08-17T17:23:32.286671+0000","last_undegraded":"2024-08-17T17:23:32.286671+0000","last_fullsized":"2024-08-17T17:23:32.286671+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:46.052040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414711808,"num_objects":6328,"num_object_clones":0,"num_object_copies":18984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6328,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":283,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.228777+0000","last_change":"2024-08-17T17:12:40.352191+0000","last_active":"2024-08-17T17:23:29.228777+0000","last_peered":"2024-08-17T17:23:29.228777+0000","last_clean":"2024-08-17T17:23:29.228777+0000","last_became_active":"2024-08-17T17:12:40.351471+0000","last_became_peered":"2024-08-17T17:12:40.351471+0000","last_unstale":"2024-08-17T17:23:29.228777+0000","last_undegraded":"2024-08-17T17:23:29.228777+0000","last_fullsized":"2024-08-17T17:23:29.228777+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:11.913270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":295,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221631+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:23:29.221631+0000","last_peered":"2024-08-17T17:23:29.221631+0000","last_clean":"2024-08-17T17:23:29.221631+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:23:29.221631+0000","last_undegraded":"2024-08-17T17:23:29.221631+0000","last_fullsized":"2024-08-17T17:23:29.221631+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":280,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216556+0000","last_change":"2024-08-17T17:12:40.350749+0000","last_active":"2024-08-17T17:23:29.216556+0000","last_peered":"2024-08-17T17:23:29.216556+0000","last_clean":"2024-08-17T17:23:29.216556+0000","last_became_active":"2024-08-17T17:12:40.347753+0000","last_became_peered":"2024-08-17T17:12:40.347753+0000","last_unstale":"2024-08-17T17:23:29.216556+0000","last_undegraded":"2024-08-17T17:23:29.216556+0000","last_fullsized":"2024-08-17T17:23:29.216556+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:22.484815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"234'6506","reported_seq":6526,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:32.609775+0000","last_change":"2024-08-17T17:22:18.545136+0000","last_active":"2024-08-17T17:23:32.609775+0000","last_peered":"2024-08-17T17:23:32.609775+0000","last_clean":"2024-08-17T17:23:32.609775+0000","last_became_active":"2024-08-17T17:22:18.544176+0000","last_became_peered":"2024-08-17T17:22:18.544176+0000","last_unstale":"2024-08-17T17:23:32.609775+0000","last_undegraded":"2024-08-17T17:23:32.609775+0000","last_fullsized":"2024-08-17T17:23:32.609775+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6506,"log_dups_size":0,"ondisk_log_size":6506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:08.787886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426377216,"num_objects":6506,"num_object_clones":0,"num_object_copies":19518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6506,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":333,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221467+0000","last_change":"2024-08-17T17:07:51.487923+0000","last_active":"2024-08-17T17:23:29.221467+0000","last_peered":"2024-08-17T17:23:29.221467+0000","last_clean":"2024-08-17T17:23:29.221467+0000","last_became_active":"2024-08-17T17:07:51.486160+0000","last_became_peered":"2024-08-17T17:07:51.486160+0000","last_unstale":"2024-08-17T17:23:29.221467+0000","last_undegraded":"2024-08-17T17:23:29.221467+0000","last_fullsized":"2024-08-17T17:23:29.221467+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:43.988926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":201,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576509+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:23:16.576509+0000","last_peered":"2024-08-17T17:23:16.576509+0000","last_clean":"2024-08-17T17:23:16.576509+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:23:16.576509+0000","last_undegraded":"2024-08-17T17:23:16.576509+0000","last_fullsized":"2024-08-17T17:23:16.576509+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":188,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575499+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:23:16.575499+0000","last_peered":"2024-08-17T17:23:16.575499+0000","last_clean":"2024-08-17T17:23:16.575499+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:23:16.575499+0000","last_undegraded":"2024-08-17T17:23:16.575499+0000","last_fullsized":"2024-08-17T17:23:16.575499+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":188,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575806+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:23:16.575806+0000","last_peered":"2024-08-17T17:23:16.575806+0000","last_clean":"2024-08-17T17:23:16.575806+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:23:16.575806+0000","last_undegraded":"2024-08-17T17:23:16.575806+0000","last_fullsized":"2024-08-17T17:23:16.575806+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":194,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575792+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:23:16.575792+0000","last_peered":"2024-08-17T17:23:16.575792+0000","last_clean":"2024-08-17T17:23:16.575792+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:23:16.575792+0000","last_undegraded":"2024-08-17T17:23:16.575792+0000","last_fullsized":"2024-08-17T17:23:16.575792+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":295,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.222033+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:23:29.222033+0000","last_peered":"2024-08-17T17:23:29.222033+0000","last_clean":"2024-08-17T17:23:29.222033+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:23:29.222033+0000","last_undegraded":"2024-08-17T17:23:29.222033+0000","last_fullsized":"2024-08-17T17:23:29.222033+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":192,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578504+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:23:16.578504+0000","last_peered":"2024-08-17T17:23:16.578504+0000","last_clean":"2024-08-17T17:23:16.578504+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:23:16.578504+0000","last_undegraded":"2024-08-17T17:23:16.578504+0000","last_fullsized":"2024-08-17T17:23:16.578504+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":132,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.578872+0000","last_change":"2024-08-17T17:21:31.909108+0000","last_active":"2024-08-17T17:23:16.578872+0000","last_peered":"2024-08-17T17:23:16.578872+0000","last_clean":"2024-08-17T17:23:16.578872+0000","last_became_active":"2024-08-17T17:21:31.908882+0000","last_became_peered":"2024-08-17T17:21:31.908882+0000","last_unstale":"2024-08-17T17:23:16.578872+0000","last_undegraded":"2024-08-17T17:23:16.578872+0000","last_fullsized":"2024-08-17T17:23:16.578872+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:24.197352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":242,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216819+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:23:29.216819+0000","last_peered":"2024-08-17T17:23:29.216819+0000","last_clean":"2024-08-17T17:23:29.216819+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:23:29.216819+0000","last_undegraded":"2024-08-17T17:23:29.216819+0000","last_fullsized":"2024-08-17T17:23:29.216819+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":339,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220293+0000","last_change":"2024-08-17T17:21:31.882662+0000","last_active":"2024-08-17T17:23:29.220293+0000","last_peered":"2024-08-17T17:23:29.220293+0000","last_clean":"2024-08-17T17:23:29.220293+0000","last_became_active":"2024-08-17T17:21:31.882374+0000","last_became_peered":"2024-08-17T17:21:31.882374+0000","last_unstale":"2024-08-17T17:23:29.220293+0000","last_undegraded":"2024-08-17T17:23:29.220293+0000","last_fullsized":"2024-08-17T17:23:29.220293+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:31.311847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":254,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216469+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:23:29.216469+0000","last_peered":"2024-08-17T17:23:29.216469+0000","last_clean":"2024-08-17T17:23:29.216469+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:23:29.216469+0000","last_undegraded":"2024-08-17T17:23:29.216469+0000","last_fullsized":"2024-08-17T17:23:29.216469+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":205,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576467+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:23:16.576467+0000","last_peered":"2024-08-17T17:23:16.576467+0000","last_clean":"2024-08-17T17:23:16.576467+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:23:16.576467+0000","last_undegraded":"2024-08-17T17:23:16.576467+0000","last_fullsized":"2024-08-17T17:23:16.576467+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216633+0000","last_change":"2024-08-17T17:12:40.351262+0000","last_active":"2024-08-17T17:23:29.216633+0000","last_peered":"2024-08-17T17:23:29.216633+0000","last_clean":"2024-08-17T17:23:29.216633+0000","last_became_active":"2024-08-17T17:12:40.348559+0000","last_became_peered":"2024-08-17T17:12:40.348559+0000","last_unstale":"2024-08-17T17:23:29.216633+0000","last_undegraded":"2024-08-17T17:23:29.216633+0000","last_fullsized":"2024-08-17T17:23:29.216633+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:04.257228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":330,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.221324+0000","last_change":"2024-08-17T17:12:40.344694+0000","last_active":"2024-08-17T17:23:29.221324+0000","last_peered":"2024-08-17T17:23:29.221324+0000","last_clean":"2024-08-17T17:23:29.221324+0000","last_became_active":"2024-08-17T17:12:40.344305+0000","last_became_peered":"2024-08-17T17:12:40.344305+0000","last_unstale":"2024-08-17T17:23:29.221324+0000","last_undegraded":"2024-08-17T17:23:29.221324+0000","last_fullsized":"2024-08-17T17:23:29.221324+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:50.542188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":294,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220948+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:23:29.220948+0000","last_peered":"2024-08-17T17:23:29.220948+0000","last_clean":"2024-08-17T17:23:29.220948+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:23:29.220948+0000","last_undegraded":"2024-08-17T17:23:29.220948+0000","last_fullsized":"2024-08-17T17:23:29.220948+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":254,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.216538+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:23:29.216538+0000","last_peered":"2024-08-17T17:23:29.216538+0000","last_clean":"2024-08-17T17:23:29.216538+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:23:29.216538+0000","last_undegraded":"2024-08-17T17:23:29.216538+0000","last_fullsized":"2024-08-17T17:23:29.216538+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":207,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576054+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:23:16.576054+0000","last_peered":"2024-08-17T17:23:16.576054+0000","last_clean":"2024-08-17T17:23:16.576054+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:23:16.576054+0000","last_undegraded":"2024-08-17T17:23:16.576054+0000","last_fullsized":"2024-08-17T17:23:16.576054+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":318,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.219082+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:23:29.219082+0000","last_peered":"2024-08-17T17:23:29.219082+0000","last_clean":"2024-08-17T17:23:29.219082+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:23:29.219082+0000","last_undegraded":"2024-08-17T17:23:29.219082+0000","last_fullsized":"2024-08-17T17:23:29.219082+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":321,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.220530+0000","last_change":"2024-08-17T17:07:07.423715+0000","last_active":"2024-08-17T17:23:29.220530+0000","last_peered":"2024-08-17T17:23:29.220530+0000","last_clean":"2024-08-17T17:23:29.220530+0000","last_became_active":"2024-08-17T17:07:07.423429+0000","last_became_peered":"2024-08-17T17:07:07.423429+0000","last_unstale":"2024-08-17T17:23:29.220530+0000","last_undegraded":"2024-08-17T17:23:29.220530+0000","last_fullsized":"2024-08-17T17:23:29.220530+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:13.132943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":215,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575739+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:23:16.575739+0000","last_peered":"2024-08-17T17:23:16.575739+0000","last_clean":"2024-08-17T17:23:16.575739+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:23:16.575739+0000","last_undegraded":"2024-08-17T17:23:16.575739+0000","last_fullsized":"2024-08-17T17:23:16.575739+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":88,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575708+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:23:16.575708+0000","last_peered":"2024-08-17T17:23:16.575708+0000","last_clean":"2024-08-17T17:23:16.575708+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:23:16.575708+0000","last_undegraded":"2024-08-17T17:23:16.575708+0000","last_fullsized":"2024-08-17T17:23:16.575708+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.575455+0000","last_change":"2024-08-17T17:12:40.335991+0000","last_active":"2024-08-17T17:23:16.575455+0000","last_peered":"2024-08-17T17:23:16.575455+0000","last_clean":"2024-08-17T17:23:16.575455+0000","last_became_active":"2024-08-17T17:12:40.335692+0000","last_became_peered":"2024-08-17T17:12:40.335692+0000","last_unstale":"2024-08-17T17:23:16.575455+0000","last_undegraded":"2024-08-17T17:23:16.575455+0000","last_fullsized":"2024-08-17T17:23:16.575455+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:13.303857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":88,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576396+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:23:16.576396+0000","last_peered":"2024-08-17T17:23:16.576396+0000","last_clean":"2024-08-17T17:23:16.576396+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:23:16.576396+0000","last_undegraded":"2024-08-17T17:23:16.576396+0000","last_fullsized":"2024-08-17T17:23:16.576396+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":285,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.217127+0000","last_change":"2024-08-17T17:12:40.343212+0000","last_active":"2024-08-17T17:23:29.217127+0000","last_peered":"2024-08-17T17:23:29.217127+0000","last_clean":"2024-08-17T17:23:29.217127+0000","last_became_active":"2024-08-17T17:12:40.342909+0000","last_became_peered":"2024-08-17T17:12:40.342909+0000","last_unstale":"2024-08-17T17:23:29.217127+0000","last_undegraded":"2024-08-17T17:23:29.217127+0000","last_fullsized":"2024-08-17T17:23:29.217127+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:24.931427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":271,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.217667+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:23:29.217667+0000","last_peered":"2024-08-17T17:23:29.217667+0000","last_clean":"2024-08-17T17:23:29.217667+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:23:29.217667+0000","last_undegraded":"2024-08-17T17:23:29.217667+0000","last_fullsized":"2024-08-17T17:23:29.217667+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":188,"reported_epoch":233,"state":"active+clean","last_fresh":"2024-08-17T17:23:16.576138+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:23:16.576138+0000","last_peered":"2024-08-17T17:23:16.576138+0000","last_clean":"2024-08-17T17:23:16.576138+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:23:16.576138+0000","last_undegraded":"2024-08-17T17:23:16.576138+0000","last_fullsized":"2024-08-17T17:23:16.576138+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":344,"reported_epoch":234,"state":"active+clean","last_fresh":"2024-08-17T17:23:29.219907+0000","last_change":"2024-08-17T17:21:31.882968+0000","last_active":"2024-08-17T17:23:29.219907+0000","last_peered":"2024-08-17T17:23:29.219907+0000","last_clean":"2024-08-17T17:23:29.219907+0000","last_became_active":"2024-08-17T17:21:31.882438+0000","last_became_peered":"2024-08-17T17:21:31.882438+0000","last_unstale":"2024-08-17T17:23:29.219907+0000","last_undegraded":"2024-08-17T17:23:29.219907+0000","last_fullsized":"2024-08-17T17:23:29.219907+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:05.660809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6708854784,"num_objects":102369,"num_object_clones":0,"num_object_copies":307107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":102369,"num_write_kb":6551616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20186267648,"data_stored":20186267648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102369,"ondisk_log_size":102369,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201619,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2548644,"kb_used_data":1643164,"kb_used_omap":1,"kb_used_meta":905470,"kb_avail":91196508,"statfs":{"total":95995035648,"available":93385224192,"internally_reserved":0,"allocated":1682599936,"data_stored":1681444837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927201624},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463255,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3729884,"kb_used_data":2840340,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90015268,"statfs":{"total":95995035648,"available":92175634432,"internally_reserved":0,"allocated":2908508160,"data_stored":2907361110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698491930,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2480328,"kb_used_data":2089152,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":91264824,"statfs":{"total":95995035648,"available":93455179776,"internally_reserved":0,"allocated":2139291648,"data_stored":2138159310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986529,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4133208,"kb_used_data":3244752,"kb_used_omap":1,"kb_used_meta":888446,"kb_avail":89611944,"statfs":{"total":95995035648,"available":91762630656,"internally_reserved":0,"allocated":3322626048,"data_stored":3321482350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909769048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225349,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2566276,"kb_used_data":2082940,"kb_used_omap":4,"kb_used_meta":483323,"kb_avail":91178876,"statfs":{"total":95995035648,"available":93367169024,"internally_reserved":0,"allocated":2132930560,"data_stored":2131802546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":494923200},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075674,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2824396,"kb_used_data":2481092,"kb_used_omap":6,"kb_used_meta":343289,"kb_avail":90920756,"statfs":{"total":95995035648,"available":93102854144,"internally_reserved":0,"allocated":2540638208,"data_stored":2539502797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":351528379},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533566,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840200,"kb_used_data":1196540,"kb_used_omap":5,"kb_used_meta":643642,"kb_avail":91904952,"statfs":{"total":95995035648,"available":94110670848,"internally_reserved":0,"allocated":1225256960,"data_stored":1224127736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":659089648},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089543,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4706704,"kb_used_data":4077448,"kb_used_omap":5,"kb_used_meta":629242,"kb_avail":89038448,"statfs":{"total":95995035648,"available":91175370752,"internally_reserved":0,"allocated":4175306752,"data_stored":4174167785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":644343867},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4172087296,"data_stored":4172087296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1228931072,"data_stored":1228931072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2553937920,"data_stored":2553937920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2137849856,"data_stored":2137849856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3343515648,"data_stored":3343515648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2129788928,"data_stored":2129788928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2931621888,"data_stored":2931621888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1688535040,"data_stored":1688535040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:23:37.155 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:23:37.156 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:23:37.156 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:23:37.156 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.448872 2024-08-17T17:23:37.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 0.44887183675860964 2024-08-17T17:23:37.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:37.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:37.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:37.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:37.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:37.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:37.580 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 107567 107551 100.308 64.6875 0.0151591 0.00996392 2024-08-17T17:23:37.618 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:37.618 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:37.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:37.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:37.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:37.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:37.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:37.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:38.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:38.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:38.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:38.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:38.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:38.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:38.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:38.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:23:38.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:38.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:38.580 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 108594 108578 99.7771 64.1875 0.0217228 0.0100174 2024-08-17T17:23:38.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:38.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:38.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:38.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:38.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:38.786 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:38 smithi142 ceph-mon[106729]: pgmap v765: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-17T17:23:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:38 smithi142 ceph-mon[103614]: pgmap v765: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-17T17:23:38.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:38.912 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:38.924 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:38 smithi152 ceph-mon[93456]: pgmap v765: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-17T17:23:39.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:39.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:39.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:39.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:39.297 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:39.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:39.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:39.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:39.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:39.580 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 109617 109601 99.2576 63.9375 0.0170774 0.0100699 2024-08-17T17:23:39.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:39.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:39.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:39.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:39.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:39.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:39.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:40.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:40.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:40.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:40.166 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:40.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:40.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:40.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:40.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:23:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:23:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:23:40.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:40.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:40.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:40.581 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 110667 110651 98.7752 65.625 0.00541581 0.0101188 2024-08-17T17:23:40.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:40.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:40.671 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:40 smithi142 ceph-mon[106729]: pgmap v766: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:40.672 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:40 smithi142 ceph-mon[103614]: pgmap v766: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:40.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:40.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:40 smithi152 ceph-mon[93456]: pgmap v766: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:40.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:40.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:40.756 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:40.756 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:40.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:40.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:40.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:41.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:41.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:41.156 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:41.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:41.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:41.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:41.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:41.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:41.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:41.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:41.581 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 111573 111557 98.1816 56.625 0.00914332 0.0101793 2024-08-17T17:23:41.596 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:23:41.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:41.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:41.757 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:41.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:41.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:41.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:41.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1351268552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:41.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:23:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:23:41] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-08-17T17:23:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:41.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1351268552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:41.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:41.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:41.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:42.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1351268552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:42.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:42.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:23:42.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:42.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:42.378 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:42.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:42.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:42.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:42.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:42.584 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:42.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 112571 112555 97.6808 62.375 0.0116943 0.0102325 2024-08-17T17:23:42.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:42.660 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[103614]: pgmap v767: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-08-17T17:23:42.660 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:42.660 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1351268552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:42.660 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[103614]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T17:23:42.660 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[106729]: pgmap v767: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1351268552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:42.661 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:42 smithi142 ceph-mon[106729]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T17:23:42.692 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:42.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:42.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:42.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:42.859 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:42 smithi152 ceph-mon[93456]: pgmap v767: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-08-17T17:23:42.859 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:42 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:42.859 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1351268552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:42.859 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:42 smithi152 ceph-mon[93456]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T17:23:42.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:42.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:43.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:43.107 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:43.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:43.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:43.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:43.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:43.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:43.458 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:23:43.458 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:23:43.458 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:43.458 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:43.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:43.492 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:23:43.492 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:43.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:43.555 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:23:43.555 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:43.582 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:23:43.582 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:43.584 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 113514 113498 97.15 58.9375 0.0143321 0.0102886 2024-08-17T17:23:43.620 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:23:43.620 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:43.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:43.661 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:23:43.661 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:43.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:43.689 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:23:43.689 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:43.711 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2219862765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:43.711 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2219862765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:43.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:43.795 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2219862765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:43.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:43.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:43.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:43.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:44.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:44.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:44.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:44.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:44.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:44.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:44.470 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:23:44.470 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:23:44.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:23:44.470 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73423 9.10 9.364f5070 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.638087+0000", 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.82811786500002, 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96902246400000003, 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73423 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.471 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638087+0000", 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638087+0000", 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638085+0000", 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638219+0000", 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013416799999999999 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638229+0000", 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5980000000000003e-06 2024-08-17T17:23:44.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638234+0000", 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0200000000000002e-06 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638275+0000", 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0731e-05 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638281+0000", 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.657e-06 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607099+0000", 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96881813800000005 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607109+0000", 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7890000000000006e-06 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73452 9.10 9.1b94c10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.654997+0000", 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.81120768900001, 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95221170799999999, 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73452 2024-08-17T17:23:44.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.654997+0000", 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.654997+0000", 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.654995+0000", 2024-08-17T17:23:44.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655078+0000", 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2665999999999998e-05 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655090+0000", 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1800000000000001e-05 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655094+0000", 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3970000000000004e-06 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655139+0000", 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4725000000000003e-05 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655146+0000", 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6240000000000002e-06 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607202+0000", 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95205562600000004 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607209+0000", 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4579999999999998e-06 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73460 9.10 9.6fcedb10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.659057+0000", 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.80714764200002, 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94826010100000002, 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.481 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73460 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659057+0000", 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659057+0000", 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659055+0000", 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659082+0000", 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.728e-05 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659096+0000", 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3103e-05 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659102+0000", 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9460000000000003e-06 2024-08-17T17:23:44.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659157+0000", 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5708000000000003e-05 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659164+0000", 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9949999999999999e-06 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607305+0000", 2024-08-17T17:23:44.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94814032699999995 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607317+0000", 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2401e-05 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73489 9.10 9.408e6a50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.486 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.677505+0000", 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.78869987799999, 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92987554699999997, 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73489 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677505+0000", 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677505+0000", 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677503+0000", 2024-08-17T17:23:44.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677640+0000", 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013677000000000001 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677649+0000", 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0380000000000005e-06 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677654+0000", 2024-08-17T17:23:44.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9400000000000001e-06 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677702+0000", 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7562999999999998e-05 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677709+0000", 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5299999999999999e-06 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607348+0000", 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92963922200000004 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:23:44.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1744e-05 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73517 9.10 9.fb63b790 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.691458+0000", 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.774746843, 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91600119800000002, 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73517 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691458+0000", 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691458+0000", 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691456+0000", 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691544+0000", 2024-08-17T17:23:44.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7583999999999994e-05 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691558+0000", 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4141000000000001e-05 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691564+0000", 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3330000000000002e-06 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691616+0000", 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1677e-05 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691623+0000", 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4479999999999997e-06 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607426+0000", 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91580259200000003 2024-08-17T17:23:44.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607459+0000", 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2879999999999997e-05 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73531 9.10 9.d7c06550 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.698407+0000", 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.767797841, 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.909076039, 2024-08-17T17:23:44.497 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73531 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698407+0000", 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698407+0000", 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698405+0000", 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698479+0000", 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4041000000000005e-05 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698493+0000", 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3409e-05 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698499+0000", 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.72e-06 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698544+0000", 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4651000000000003e-05 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.698551+0000", 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9800000000000001e-06 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607475+0000", 2024-08-17T17:23:44.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90892428199999997 2024-08-17T17:23:44.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607483+0000", 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4850000000000003e-06 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73543 9.10 9.f9329950 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.706051+0000", 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.76015399400001, 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90187487499999996, 2024-08-17T17:23:44.504 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73543 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706051+0000", 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706051+0000", 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706049+0000", 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706104+0000", 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4904999999999997e-05 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706118+0000", 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3998999999999999e-05 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706123+0000", 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0340000000000004e-06 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706180+0000", 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6501e-05 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.706188+0000", 2024-08-17T17:23:44.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1019999999999997e-06 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607921+0000", 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90173358800000003 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1289999999999998e-06 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.509 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73564 9.10 9.15031930 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.719151+0000", 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.74705346600001, 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88878655600000001, 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73564 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719151+0000", 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719151+0000", 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719149+0000", 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719180+0000", 2024-08-17T17:23:44.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0879e-05 2024-08-17T17:23:44.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719197+0000", 2024-08-17T17:23:44.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7158000000000001e-05 2024-08-17T17:23:44.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719204+0000", 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6320000000000002e-06 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719262+0000", 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7980999999999999e-05 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.719270+0000", 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5560000000000008e-06 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607934+0000", 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88866431400000001 2024-08-17T17:23:44.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607938+0000", 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2310000000000001e-06 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73583 9.10 9.c7df69d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.728968+0000", 2024-08-17T17:23:44.516 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 328.73723601900002, 2024-08-17T17:23:44.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87898079399999995, 2024-08-17T17:23:44.517 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.517 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:23:44.517 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.517 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.517 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73583 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.728968+0000", 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.728968+0000", 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.728967+0000", 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.729020+0000", 2024-08-17T17:23:44.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3013000000000002e-05 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.729038+0000", 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8298e-05 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.729045+0000", 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.601e-06 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.729097+0000", 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2024999999999997e-05 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.729104+0000", 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1250000000000004e-06 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607946+0000", 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87884240400000002 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607949+0000", 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0189999999999998e-06 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:130930 9.10 9:0f3273ea:::benchmark_data_smithi142_157452_object130929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.632116+0000", 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.83408805599998, 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.009814414, 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130930 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632116+0000", 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632116+0000", 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632115+0000", 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632293+0000", 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017897599999999999 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632310+0000", 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6932e-05 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632318+0000", 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1160000000000001e-06 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.634785+0000", 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024670349999999998 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.634794+0000", 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3039999999999994e-06 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.637459+0000", 2024-08-17T17:23:44.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.002664835 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.637550+0000", 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1077999999999998e-05 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.637640+0000", 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0031999999999995e-05 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641657+0000", 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004017222 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641666+0000", 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7329999999999992e-06 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641843+0000", 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000177565 2024-08-17T17:23:44.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641894+0000", 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0785999999999998e-05 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641931+0000", 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6743999999999999e-05 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:130938 9.10 9:0e8a7329:::benchmark_data_smithi142_157452_object130937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.637870+0000", 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.828334837, 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.005439572, 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130938 2024-08-17T17:23:44.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637870+0000", 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637870+0000", 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637868+0000", 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637959+0000", 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0857000000000001e-05 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637975+0000", 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6239000000000002e-05 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637982+0000", 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4250000000000003e-06 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.638046+0000", 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4596000000000003e-05 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.638058+0000", 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1234e-05 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.639258+0000", 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001200818 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.639345+0000", 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6588000000000005e-05 2024-08-17T17:23:44.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.639453+0000", 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107642 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641780+0000", 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002326846 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641943+0000", 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000162941 2024-08-17T17:23:44.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643245+0000", 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001302847 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643274+0000", 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8538e-05 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643309+0000", 2024-08-17T17:23:44.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5363e-05 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:130971 9.10 9:0d797390:::benchmark_data_smithi142_157452_object130970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.668996+0000", 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.79720856099999, 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97444199300000001, 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.537 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130971 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.668996+0000", 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.668996+0000", 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.668994+0000", 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669035+0000", 2024-08-17T17:23:44.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1689e-05 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669055+0000", 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0049000000000001e-05 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669063+0000", 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2690000000000002e-06 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669125+0000", 2024-08-17T17:23:44.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2682999999999996e-05 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669136+0000", 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0825999999999999e-05 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.640571+0000", 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97143519 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.640640+0000", 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8612e-05 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.640712+0000", 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2284000000000001e-05 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643217+0000", 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025048829999999998 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643226+0000", 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6920000000000008e-06 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643354+0000", 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012790800000000001 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643400+0000", 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6555999999999999e-05 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643438+0000", 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7691000000000002e-05 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.544 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:131062 9.10 9:0d6d8fba:::benchmark_data_smithi142_157452_object131061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.757047+0000", 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.70915704599997, 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89274083800000004, 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.545 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131062 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757047+0000", 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757047+0000", 2024-08-17T17:23:44.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757045+0000", 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999971 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757139+0000", 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4112999999999998e-05 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757162+0000", 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2869000000000001e-05 2024-08-17T17:23:44.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757171+0000", 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9679999999999995e-06 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757226+0000", 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5108999999999997e-05 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757234+0000", 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.678e-06 2024-08-17T17:23:44.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641972+0000", 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88473843399999996 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.642048+0000", 2024-08-17T17:23:44.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5779999999999996e-05 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.642121+0000", 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3702999999999996e-05 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.646131+0000", 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040091509999999999 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.646177+0000", 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6396000000000001e-05 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:23:44.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.649711+0000", 2024-08-17T17:23:44.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035344040000000001 2024-08-17T17:23:44.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.649758+0000", 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.668e-05 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.649788+0000", 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0202999999999999e-05 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:131065 9.10 9:09ccfd74:::benchmark_data_smithi142_157452_object131064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.757189+0000", 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.70901542299998, 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89725090200000002, 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131065 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757189+0000", 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757189+0000", 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757188+0000", 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757224+0000", 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5222999999999998e-05 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757232+0000", 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7299999999999994e-06 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757238+0000", 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4060000000000004e-06 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757322+0000", 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4386000000000001e-05 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.757328+0000", 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9020000000000004e-06 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643470+0000", 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88614155100000003 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643539+0000", 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9527e-05 2024-08-17T17:23:44.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643623+0000", 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3320999999999997e-05 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.646207+0000", 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025843350000000001 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.646215+0000", 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9109999999999993e-06 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.654345+0000", 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081302479999999996 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.654397+0000", 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1657999999999999e-05 2024-08-17T17:23:44.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.654440+0000", 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3325000000000003e-05 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:131109 9.10 9:08c40ada:::benchmark_data_smithi142_157452_object131108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.797077+0000", 2024-08-17T17:23:44.567 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.66912710299999, 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86108629400000003, 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131109 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797077+0000", 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797077+0000", 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797076+0000", 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797201+0000", 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000125 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797221+0000", 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0577999999999998e-05 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797228+0000", 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7490000000000001e-06 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797283+0000", 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4602999999999999e-05 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.797294+0000", 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.102e-05 2024-08-17T17:23:44.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.644548+0000", 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84725410199999995 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.644666+0000", 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011785099999999999 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.644840+0000", 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000174007 2024-08-17T17:23:44.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.649797+0000", 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049576949999999998 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.649833+0000", 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5831000000000003e-05 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.658079+0000", 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082455549999999999 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.658126+0000", 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7324000000000003e-05 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.658164+0000", 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7629999999999997e-05 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.574 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:26872 10.1 10:8481d32e:::benchmark_data_smithi142_192453_object26871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:39.279132+0000", 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 65.187072310000005, 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13717700999999999, 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 26872 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.575 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279132+0000", 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279132+0000", 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279131+0000", 2024-08-17T17:23:44.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279172+0000", 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.074e-05 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279182+0000", 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8320000000000001e-06 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279187+0000", 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.792e-06 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.282948+0000", 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037616519999999999 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.283048+0000", 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9664e-05 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.283163+0000", 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114701 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.287562+0000", 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043993310000000002 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.287726+0000", 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000164384 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.416205+0000", 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.128478974 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.416270+0000", 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4527000000000001e-05 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.416309+0000", 2024-08-17T17:23:44.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9462000000000003e-05 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:26875 10.1 10:8339c769:::benchmark_data_smithi142_192453_object26874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:39.279192+0000", 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 65.187013007999994, 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13801306999999999, 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.581 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 26875 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279192+0000", 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279192+0000", 2024-08-17T17:23:44.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279191+0000", 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279220+0000", 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8575999999999999e-05 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279227+0000", 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1040000000000001e-06 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.279231+0000", 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0910000000000003e-06 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.284596+0000", 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005365608 2024-08-17T17:23:44.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.284703+0000", 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106389 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.284882+0000", 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017878499999999999 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.287606+0000", 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027240559999999999 2024-08-17T17:23:44.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.287730+0000", 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012473399999999999 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.417007+0000", 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.12927609800000001 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.417170+0000", 2024-08-17T17:23:44.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016381800000000001 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:39.417205+0000", 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4220000000000001e-05 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:46041 10.e 10:727e96bd:::benchmark_data_smithi142_192453_object46040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:49.574886+0000", 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.891319013, 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18883924599999999, 2024-08-17T17:23:44.587 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 46041 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574886+0000", 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574886+0000", 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574884+0000", 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574911+0000", 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7141e-05 2024-08-17T17:23:44.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574920+0000", 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0100000000000001e-06 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574925+0000", 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7690000000000004e-06 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.574990+0000", 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5427999999999998e-05 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.575099+0000", 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010870399999999999 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.575216+0000", 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116859 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.577212+0000", 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019965650000000001 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.579061+0000", 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001848707 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.763642+0000", 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18458048099999999 2024-08-17T17:23:44.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.763690+0000", 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.833e-05 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.763725+0000", 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4761999999999997e-05 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:46068 10.e 10:7ee4241d:::benchmark_data_smithi142_192453_object46067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:49.592661+0000", 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.873543949000002, 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.17116077199999999, 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 46068 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.594 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592661+0000", 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592661+0000", 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592660+0000", 2024-08-17T17:23:44.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592710+0000", 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0102000000000002e-05 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592720+0000", 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9690000000000006e-06 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592724+0000", 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.313e-06 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592761+0000", 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6569999999999997e-05 2024-08-17T17:23:44.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592846+0000", 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5284999999999994e-05 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.592940+0000", 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3609000000000004e-05 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.594228+0000", 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012885259999999999 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.594633+0000", 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00040531699999999998 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.763755+0000", 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.169121208 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.763791+0000", 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5902000000000003e-05 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:49.763821+0000", 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0809999999999998e-05 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.600 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:87779 10.1 10:867a5498:::benchmark_data_smithi142_192453_object87778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:19.145247+0000", 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 25.320957078999999, 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13832661500000001, 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:44.601 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 87779 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145247+0000", 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145247+0000", 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:44.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145246+0000", 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145311+0000", 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4980000000000005e-05 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145326+0000", 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4535000000000001e-05 2024-08-17T17:23:44.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145331+0000", 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3260000000000003e-06 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145379+0000", 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7825999999999999e-05 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:44.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145499+0000", 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120548 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.145636+0000", 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013705100000000001 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.148670+0000", 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030339070000000002 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.151105+0000", 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024346720000000001 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.283464+0000", 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13235953 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.283534+0000", 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0018999999999995e-05 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:44.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:19.283574+0000", 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9641999999999998e-05 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:44.607 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:23:44.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:44.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:44.609 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 15 114455 114440 96.6328 58.875 0.0129905 0.0103435 2024-08-17T17:23:44.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:44.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:44.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:44 smithi142 ceph-mon[106729]: pgmap v769: 89 pgs: 89 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:44 smithi142 ceph-mon[103614]: pgmap v769: 89 pgs: 89 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:44.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:44.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:44 smithi152 ceph-mon[93456]: pgmap v769: 89 pgs: 89 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:44.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:45.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:45.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:45.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:45.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:45.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:45.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:45.471 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:45.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:45.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:23:45.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:45.584 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 115326 115310 96.0692 54.375 0.0209513 0.0104044 2024-08-17T17:23:45.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:45.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:45.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:45.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:45.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:45.880 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:45.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:45.922 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:45.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:46.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:46.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:46.147 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:46.249 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:46.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:46.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:46.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:46.431 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:46.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:46.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:46.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:46.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:46.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 116216 116200 95.5371 55.625 0.00858268 0.0104622 2024-08-17T17:23:46.617 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.0 primary-affinity to 0.448872 (8294172) 2024-08-17T17:23:46.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:46.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:46.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:46 smithi152 ceph-mon[93456]: pgmap v770: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4152386136' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.44887183675860964} : dispatch 2024-08-17T17:23:46.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:46 smithi142 ceph-mon[106729]: pgmap v770: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4152386136' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.44887183675860964} : dispatch 2024-08-17T17:23:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:46 smithi142 ceph-mon[103614]: pgmap v770: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4152386136' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.44887183675860964} : dispatch 2024-08-17T17:23:46.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:46.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:47.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:47.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:47.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:47.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:47.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:47.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:47.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:47.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:47.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:47.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:47.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:47.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:47.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:47.584 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 116392 116376 94.4393 11 0.00377902 0.0104664 2024-08-17T17:23:47.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:47.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:47.733 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4152386136' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.44887183675860964}]': finished 2024-08-17T17:23:47.733 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:47 smithi152 ceph-mon[93456]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T17:23:47.734 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:47 smithi152 ceph-mon[93456]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:116483 10.e e237/222 10:7496edad:::benchmark_data_smithi142_192453_object116482:head v 237'7251, mlcod=237'7249)", 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:47.763148+0000", 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0022656669999999999, 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022932249999999999, 2024-08-17T17:23:47.767 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763148+0000", 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:47.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763148+0000", 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763147+0000", 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763171+0000", 2024-08-17T17:23:47.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4082000000000001e-05 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763182+0000", 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0577999999999999e-05 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763187+0000", 2024-08-17T17:23:47.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9899999999999997e-06 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763253+0000", 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6122000000000002e-05 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763273+0000", 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9962e-05 2024-08-17T17:23:47.771 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116486 10.b 10:d4cd42c7:::benchmark_data_smithi142_192453_object116485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:47.763232+0000", 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0021809960000000001, 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022753920000000002, 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:47.772 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116486 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763232+0000", 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763232+0000", 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763231+0000", 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:47.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763276+0000", 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5518e-05 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763291+0000", 2024-08-17T17:23:47.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4238e-05 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763297+0000", 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9619999999999998e-06 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763341+0000", 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4008000000000003e-05 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763449+0000", 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108656 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763568+0000", 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011916799999999999 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:47.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116487 10.7 10:e841a431:::benchmark_data_smithi142_192453_object116486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:47.763311+0000", 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0021023309999999998, 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002267748, 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:47.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116487 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763311+0000", 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763311+0000", 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763310+0000", 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763342+0000", 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1714000000000002e-05 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763350+0000", 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0399999999999993e-06 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763355+0000", 2024-08-17T17:23:47.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5870000000000003e-06 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763441+0000", 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5928999999999997e-05 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763562+0000", 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012117600000000001 2024-08-17T17:23:47.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.763697+0000", 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013496499999999999 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116488 10.0 10:06b21cc9:::benchmark_data_smithi142_192453_object116487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2024-08-17T17:23:47.783 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:47.765000+0000", 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00041343899999999998, 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064678000000000001, 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116488 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:23:47.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765000+0000", 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765000+0000", 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:23:47.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.764999+0000", 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765112+0000", 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113293 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:23:47.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765128+0000", 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5702999999999999e-05 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765133+0000", 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5380000000000002e-06 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765190+0000", 2024-08-17T17:23:47.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7154999999999997e-05 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765295+0000", 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104949 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.765412+0000", 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011673 2024-08-17T17:23:47.788 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.789 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:23:47.789 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.789 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:23:47.789 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:23:47.789 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 4 2024-08-17T17:23:47.789 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:23:47.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:47.853 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4152386136' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.44887183675860964}]': finished 2024-08-17T17:23:47.853 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:47 smithi142 ceph-mon[106729]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T17:23:47.853 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:47 smithi142 ceph-mon[106729]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T17:23:47.854 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4152386136' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.44887183675860964}]': finished 2024-08-17T17:23:47.854 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:47 smithi142 ceph-mon[103614]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T17:23:47.854 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:47 smithi142 ceph-mon[103614]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T17:23:47.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:47.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:47.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:48.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:48.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:48.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:48.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:48.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:48.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:48.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:48.421 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:48.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:48.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:48.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:23:48.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 117324 117308 93.9754 58.25 0.00682966 0.0106359 2024-08-17T17:23:48.653 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:48.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:48.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:48 smithi152 ceph-mon[93456]: pgmap v772: 89 pgs: 2 peering, 87 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s 2024-08-17T17:23:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:48 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:23:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:48.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:48.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[106729]: pgmap v772: 89 pgs: 2 peering, 87 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[103614]: pgmap v772: 89 pgs: 2 peering, 87 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:48.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:49.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:49.054 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:49.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:49.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:49.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:49.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:49.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:49.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:49.433 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:49.535 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:49.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:49.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 118214 118198 93.4899 55.625 0.0136785 0.0106912 2024-08-17T17:23:49.670 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:49.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:49.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:49.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:23:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:23:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:49.823 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:23:49.823 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:23:49.823 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:49.824 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:23:49.824 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:23:49.824 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:49.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:49.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:49.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:49.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:50.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:50.089 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:50.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:50.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:50.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:50.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:23:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:23:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:23:50.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:50.382 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:50.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:50.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:50.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:50.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:23:50.585321+0000 min lat: 0.00124672 max lat: 1.08686 avg lat: 0.0107387 2024-08-17T17:23:50.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:23:50.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 119178 119162 93.0744 60.25 0.00619473 0.0107387 2024-08-17T17:23:50.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:50.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:50.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:50.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:50.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:50 smithi142 ceph-mon[106729]: pgmap v774: 89 pgs: 2 peering, 87 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16 KiB/s, 0 objects/s recovering 2024-08-17T17:23:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3966591472' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:50 smithi142 ceph-mon[103614]: pgmap v774: 89 pgs: 2 peering, 87 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16 KiB/s, 0 objects/s recovering 2024-08-17T17:23:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3966591472' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:50.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:50.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:50 smithi152 ceph-mon[93456]: pgmap v774: 89 pgs: 2 peering, 87 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16 KiB/s, 0 objects/s recovering 2024-08-17T17:23:50.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3966591472' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:50.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:51.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:51.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:51.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:51.103 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:23:51.103 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:23:51.104 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:23:51.104 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:23:51.104 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:23:51.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:51.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:51.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:51.252 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:23:51.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:51.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:51.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:51.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:51.585 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 120108 120092 92.6429 58.125 0.0117484 0.0107887 2024-08-17T17:23:51.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:51.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:51.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:51.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:51 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3966591472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:51 smithi152 ceph-mon[93456]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T17:23:51.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:51.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:23:51] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-08-17T17:23:51.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:51.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3966591472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:51.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-mon[106729]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T17:23:51.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:51.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3966591472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:51.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:51 smithi142 ceph-mon[103614]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T17:23:51.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:52.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:52.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:52.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:52.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:52.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:52.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:52.415 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:52.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:52.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:52.584 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:23:52.584 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:23:52.584 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:23:52.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 0 2024-08-17T17:23:52.586 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 121072 121056 92.2474 60.25 0.0210967 0.0108349 2024-08-17T17:23:52.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:52 smithi142 ceph-mon[103614]: pgmap v776: 89 pgs: 2 peering, 87 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 21 KiB/s, 0 objects/s recovering 2024-08-17T17:23:52.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:52 smithi142 ceph-mon[106729]: pgmap v776: 89 pgs: 2 peering, 87 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 21 KiB/s, 0 objects/s recovering 2024-08-17T17:23:52.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:52.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:52.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:52.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:52.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:52.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:52.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:52 smithi152 ceph-mon[93456]: pgmap v776: 89 pgs: 2 peering, 87 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 21 KiB/s, 0 objects/s recovering 2024-08-17T17:23:53.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:53.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:53.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:53.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:53.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:23:53.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:23:53.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:23:53.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:23:53.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:23:53.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:53.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:53.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:53.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:53.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:53.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:53.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:53.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:53.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:53.586 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 121989 121973 91.8265 57.3125 0.0171969 0.0108852 2024-08-17T17:23:53.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:23:53.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:53.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:53.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:53.815 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:53.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/541795322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/541795322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/541795322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:53.917 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:53.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:53.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:54.012 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:54.061 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:54.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:54.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:54.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:54.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:54.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:54.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:54.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:54.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:54.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:54.586 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 122932 122916 91.435 58.9375 0.0209371 0.0109314 2024-08-17T17:23:54.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:54.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:54.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:54.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:54.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:54 smithi152 ceph-mon[93456]: pgmap v777: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T17:23:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:54 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:23:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:54 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:23:54.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:54.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:55.012 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:54 smithi142 ceph-mon[103614]: pgmap v777: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T17:23:55.012 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:54 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:23:55.012 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:54 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:23:55.012 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:54 smithi142 ceph-mon[106729]: pgmap v777: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T17:23:55.012 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:54 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:23:55.013 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:54 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:23:55.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:55.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:55.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:23:55.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:55.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:55.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:55.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:55.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:55.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:55.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.523 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:55.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:55.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 123885 123869 91.0592 59.5625 0.0210056 0.0109765 2024-08-17T17:23:55.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:55.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:55.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:55.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:55.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:55.899 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:55.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:55.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:55.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:56.037 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:56.037 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:56.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:56.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:56.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:56.129 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:56.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:56.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:56.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:56.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:56.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:56.527 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:56.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 124728 124712 90.613 52.6875 0.00487739 0.0110301 2024-08-17T17:23:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[106729]: pgmap v778: 89 pgs: 89 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 16 KiB/s, 0 objects/s recovering 2024-08-17T17:23:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3612440890' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[106729]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:23:56.435+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e238 definitely_dead 0 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[103614]: pgmap v778: 89 pgs: 89 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 16 KiB/s, 0 objects/s recovering 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3612440890' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:23:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:56 smithi142 ceph-mon[103614]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:23:56.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:56.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:56.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:56 smithi152 ceph-mon[93456]: pgmap v778: 89 pgs: 89 active+clean; 7.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 16 KiB/s, 0 objects/s recovering 2024-08-17T17:23:56.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:23:56.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:56.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3612440890' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:23:56.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:56 smithi152 ceph-mon[93456]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:23:56.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:23:56.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:56.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:56.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:56.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:56.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:57.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:57.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:57.158 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:57.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:57.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:57.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:57.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:57.423 INFO:teuthology.orchestra.run.smithi142.stderr:marked out osd.0. 2024-08-17T17:23:57.504 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:57.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:57.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 125451 125435 90.0909 45.1875 0.00639338 0.011075 2024-08-17T17:23:57.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.650 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:57.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:57.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:57.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:57.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:57.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:57.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:57.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:58.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:58.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:58.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:58.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:58.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:58.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:58.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:58.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:58.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:58.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3612440890' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:23:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:58 smithi152 ceph-mon[93456]: osdmap e239: 8 total, 8 up, 7 in 2024-08-17T17:23:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:58 smithi152 ceph-mon[93456]: pgmap v780: 89 pgs: 89 active+clean; 7.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s 2024-08-17T17:23:58.589 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 125451 125435 89.0672 0 - 0.011075 2024-08-17T17:23:58.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:58.640 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:58.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:58.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:58.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:58.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3612440890' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:23:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:58 smithi142 ceph-mon[106729]: osdmap e239: 8 total, 8 up, 7 in 2024-08-17T17:23:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:58 smithi142 ceph-mon[106729]: pgmap v780: 89 pgs: 89 active+clean; 7.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s 2024-08-17T17:23:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3612440890' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:23:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:58 smithi142 ceph-mon[103614]: osdmap e239: 8 total, 8 up, 7 in 2024-08-17T17:23:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:58 smithi142 ceph-mon[103614]: pgmap v780: 89 pgs: 89 active+clean; 7.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s 2024-08-17T17:23:58.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:58.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:58.994 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:59.019 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:59.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:59.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:59.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:59.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:59.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:59.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:59.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:59.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:59.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:59.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:59.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 125451 125435 88.0666 0 - 0.011075 2024-08-17T17:23:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:23:59 smithi142 ceph-mon[106729]: osdmap e240: 8 total, 8 up, 7 in 2024-08-17T17:23:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:23:59 smithi142 ceph-mon[103614]: osdmap e240: 8 total, 8 up, 7 in 2024-08-17T17:23:59.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:59.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:59.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:59.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:59.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:23:59 smithi152 ceph-mon[93456]: osdmap e240: 8 total, 8 up, 7 in 2024-08-17T17:23:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:23:59.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:59.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:59.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:23:59.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:00.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:00.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:00.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:00.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:00.334 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:00.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.409 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:00.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:00.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:00.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:00.546 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:00 smithi152 ceph-mon[93456]: osdmap e241: 8 total, 8 up, 7 in 2024-08-17T17:24:00.546 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:00 smithi152 ceph-mon[93456]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 238'7944 2024-08-17T17:24:00.546 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:00 smithi152 ceph-mon[93456]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 238'7875 2024-08-17T17:24:00.546 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:00 smithi152 ceph-mon[93456]: pgmap v783: 89 pgs: 2 remapped+peering, 12 peering, 75 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 55 MiB/s wr, 875 op/s; 94 B/s, 0 objects/s recovering 2024-08-17T17:24:00.546 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:24:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:24:00.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:24:00.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 90 6 125745 125739 87.2993 6.33333 0.0492847 0.0114484 2024-08-17T17:24:00.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:00.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:00.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:00.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[106729]: osdmap e241: 8 total, 8 up, 7 in 2024-08-17T17:24:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[106729]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 238'7944 2024-08-17T17:24:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[106729]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 238'7875 2024-08-17T17:24:00.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[106729]: pgmap v783: 89 pgs: 2 remapped+peering, 12 peering, 75 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 55 MiB/s wr, 875 op/s; 94 B/s, 0 objects/s recovering 2024-08-17T17:24:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[103614]: osdmap e241: 8 total, 8 up, 7 in 2024-08-17T17:24:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[103614]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 238'7944 2024-08-17T17:24:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[103614]: 10.4 starting backfill to osd.1 from (0'0,0'0] MAX to 238'7875 2024-08-17T17:24:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:00 smithi142 ceph-mon[103614]: pgmap v783: 89 pgs: 2 remapped+peering, 12 peering, 75 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 55 MiB/s wr, 875 op/s; 94 B/s, 0 objects/s recovering 2024-08-17T17:24:00.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:00.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:01.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:01.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:01.062 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:01.062 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:01.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:01.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:01.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:01.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:01.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:01.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:01.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:01.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:01.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0287 2024-08-17T17:24:01.587 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 125745 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 87.295 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 44.8164 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 254.375 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1396 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 717.069 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 4070 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 0 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0114499 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0306919 2024-08-17T17:24:01.588 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 2.50142 2024-08-17T17:24:01.589 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00124672 2024-08-17T17:24:01.605 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:24:01.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:01.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:01.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:01 smithi152 ceph-mon[93456]: osdmap e242: 8 total, 8 up, 7 in 2024-08-17T17:24:01.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:24:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:01 smithi142 ceph-mon[106729]: osdmap e242: 8 total, 8 up, 7 in 2024-08-17T17:24:01.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:01 smithi142 ceph-mon[103614]: osdmap e242: 8 total, 8 up, 7 in 2024-08-17T17:24:01.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:01.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:01.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:01.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:02.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:02.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:02.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:24:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:24:01] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-08-17T17:24:02.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:02.240 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:02.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:02.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:02.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:02.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:02.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[106729]: osdmap e243: 8 total, 8 up, 7 in 2024-08-17T17:24:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[106729]: pgmap v786: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 12 peering, 74 active+clean; 7.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 667 op/s; 7825/376323 objects degraded (2.079%); 142 B/s, 1 objects/s recovering 2024-08-17T17:24:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:02.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3468043974' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[103614]: osdmap e243: 8 total, 8 up, 7 in 2024-08-17T17:24:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[103614]: pgmap v786: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 12 peering, 74 active+clean; 7.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 667 op/s; 7825/376323 objects degraded (2.079%); 142 B/s, 1 objects/s recovering 2024-08-17T17:24:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3468043974' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:02.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:02.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:02.703 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:24:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:02 smithi152 ceph-mon[93456]: osdmap e243: 8 total, 8 up, 7 in 2024-08-17T17:24:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:02 smithi152 ceph-mon[93456]: pgmap v786: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 12 peering, 74 active+clean; 7.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 667 op/s; 7825/376323 objects degraded (2.079%); 142 B/s, 1 objects/s recovering 2024-08-17T17:24:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:02.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3468043974' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:02.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:02.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:02.931 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:02.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:03.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:03.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:03.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:03.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:03.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:03.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:03.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:03.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:03.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:03.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:03 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 7825/376323 objects degraded (2.079%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:24:03.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:03 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:03.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3468043974' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:03.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:03 smithi152 ceph-mon[93456]: osdmap e244: 8 total, 8 up, 7 in 2024-08-17T17:24:03.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:03.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:03.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 7825/376323 objects degraded (2.079%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:24:03.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:03.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3468043974' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:03.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[106729]: osdmap e244: 8 total, 8 up, 7 in 2024-08-17T17:24:03.751 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 7825/376323 objects degraded (2.079%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:24:03.751 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:03.751 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3468043974' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:03.751 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:03 smithi142 ceph-mon[103614]: osdmap e244: 8 total, 8 up, 7 in 2024-08-17T17:24:03.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:03.814 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:03.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:03.917 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:03.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:04.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:04.086 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:24:04.086 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:04.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:04.122 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:24:04.122 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:04.150 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:24:04.150 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:04.174 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:24:04.174 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:04.208 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:24:04.208 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:04.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:04.257 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:24:04.257 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:04.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:04.287 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:04.287 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:04.302 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:24:04.302 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:24:04.302 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2024-08-17T17:24:04.302 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:24:04.303 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:24:04.303 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:04.341 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:24:04.342 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:04.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:04.402 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:04.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.554 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:04.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:04.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:04 smithi142 ceph-mon[103614]: pgmap v788: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 79 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 250 B/s rd, 5.4 MiB/s wr, 174 op/s; 39181/375996 objects degraded (10.421%); 23583/375996 objects misplaced (6.272%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:04.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2705002601' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:04.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:04 smithi142 ceph-mon[106729]: pgmap v788: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 79 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 250 B/s rd, 5.4 MiB/s wr, 174 op/s; 39181/375996 objects degraded (10.421%); 23583/375996 objects misplaced (6.272%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:04.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2705002601' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:04.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:04.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:04.797 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:04 smithi152 ceph-mon[93456]: pgmap v788: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 79 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 250 B/s rd, 5.4 MiB/s wr, 174 op/s; 39181/375996 objects degraded (10.421%); 23583/375996 objects misplaced (6.272%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:24:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2705002601' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:04.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:04.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:04.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:04.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:24:05.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:05.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:05.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:05.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:05.283 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:05.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:05.383 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:05.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:05.441 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:05.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:24:05.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:05.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:05.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:05.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:05.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:05.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:05.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:05.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:06.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:06.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:06.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:06.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:06.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:06.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:06.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:06.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.489 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:06.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:06.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:06.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:06.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:06.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:06.759 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:06 smithi152 ceph-mon[93456]: pgmap v789: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 170 B/s rd, 4.5 MiB/s wr, 188 op/s; 47197/375216 objects degraded (12.579%); 31245/375216 objects misplaced (8.327%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:24:06.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:06 smithi142 ceph-mon[106729]: pgmap v789: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 170 B/s rd, 4.5 MiB/s wr, 188 op/s; 47197/375216 objects degraded (12.579%); 31245/375216 objects misplaced (8.327%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:24:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:06 smithi142 ceph-mon[103614]: pgmap v789: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 170 B/s rd, 4.5 MiB/s wr, 188 op/s; 47197/375216 objects degraded (12.579%); 31245/375216 objects misplaced (8.327%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:24:06.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:06.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:07.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:07.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:07.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:07.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:07.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:07.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:07.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:07.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:07.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:07.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:07.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:07.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:07.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:07.832 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:24:07.832 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:24:07.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:24:07.832 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138547 9.13 9:cb7f8172:::benchmark_data_smithi142_157452_object6426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.660034+0000", 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.16668306600002, 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2183835709999999, 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138547 2024-08-17T17:24:07.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660034+0000", 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660034+0000", 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660033+0000", 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660040+0000", 2024-08-17T17:24:07.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9619999999999996e-06 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660053+0000", 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3118999999999999e-05 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660057+0000", 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5719999999999996e-06 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660095+0000", 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7548e-05 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660196+0000", 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100862 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.660288+0000", 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1793999999999996e-05 2024-08-17T17:24:07.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.666760+0000", 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064725069999999997 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.672804+0000", 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060435369999999999 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.672813+0000", 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4760000000000004e-06 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664712+0000", 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99189900399999997 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664744+0000", 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2020000000000002e-05 2024-08-17T17:24:07.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.857732+0000", 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.19298796 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.857862+0000", 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012966 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.878361+0000", 2024-08-17T17:24:07.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020498695000000001 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.878418+0000", 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7278000000000003e-05 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138562 9.13 9:c969e319:::benchmark_data_smithi142_157452_object6441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.841 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.677884+0000", 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.14883316300001, 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3334073970000002, 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138562 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677884+0000", 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677884+0000", 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677882+0000", 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:24:07.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677889+0000", 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3479999999999998e-06 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677904+0000", 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4928e-05 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677911+0000", 2024-08-17T17:24:07.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2420000000000003e-06 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677958+0000", 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6607000000000003e-05 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.677968+0000", 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0193999999999999e-05 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664786+0000", 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98681752700000003 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664793+0000", 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.857891+0000", 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.193098054 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.858593+0000", 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070260500000000003 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.882058+0000", 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023464589000000001 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.882156+0000", 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7842000000000003e-05 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.882300+0000", 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014423499999999999 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.892690+0000", 2024-08-17T17:24:07.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010390483000000001 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.008564+0000", 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.115873302 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.008628+0000", 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3874000000000005e-05 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.008690+0000", 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2830000000000007e-05 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.011292+0000", 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026010510000000001 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138597 9.13 9:cb0adba0:::benchmark_data_smithi142_157452_object6476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.731949+0000", 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.09476825399997, 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2811760080000001, 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138597 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.731949+0000", 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.731949+0000", 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.731948+0000", 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.731953+0000", 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.181e-06 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.731966+0000", 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3535e-05 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.731972+0000", 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7379999999999999e-06 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.732029+0000", 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6341000000000001e-05 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.732038+0000", 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7049999999999994e-06 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664825+0000", 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93278677700000001 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664831+0000", 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4709999999999998e-06 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.858640+0000", 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1938097919999999 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.859344+0000", 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070333600000000004 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.890522+0000", 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031178096999999998 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.890591+0000", 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9461000000000005e-05 2024-08-17T17:24:07.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.890678+0000", 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6731999999999996e-05 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.900347+0000", 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.009669525 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.011341+0000", 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.110993172 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.011380+0000", 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9406000000000002e-05 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.011437+0000", 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6464e-05 2024-08-17T17:24:07.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.013125+0000", 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001688529 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138625 9.13 9:cb5ad8d0:::benchmark_data_smithi142_157452_object6504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.755503+0000", 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.07121384999999, 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2590610789999999, 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.859 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138625 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755503+0000", 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755503+0000", 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755503+0000", 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755507+0000", 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2400000000000001e-06 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755521+0000", 2024-08-17T17:24:07.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3946e-05 2024-08-17T17:24:07.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755528+0000", 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.562e-06 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755587+0000", 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9089999999999998e-05 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755596+0000", 2024-08-17T17:24:07.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8660000000000001e-06 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664858+0000", 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90926198999999996 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664866+0000", 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4580000000000004e-06 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.859386+0000", 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1945201729999999 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.860215+0000", 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00082888799999999996 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.907449+0000", 2024-08-17T17:24:07.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.047233891 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.907523+0000", 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4425999999999995e-05 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.907615+0000", 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2029999999999998e-05 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.923389+0000", 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015774098 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.013175+0000", 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089785522000000006 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.013219+0000", 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4317000000000003e-05 2024-08-17T17:24:07.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.013272+0000", 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3052e-05 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.014565+0000", 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012922000000000001 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.868 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138627 9.13 9:caf758ce:::benchmark_data_smithi142_157452_object6506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.755623+0000", 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.071094518, 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2602539099999999, 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138627 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.869 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755623+0000", 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755623+0000", 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755622+0000", 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755626+0000", 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.428e-06 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755633+0000", 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.097e-06 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755636+0000", 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2200000000000001e-06 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755669+0000", 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2713999999999999e-05 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.755675+0000", 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8069999999999996e-06 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664891+0000", 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90921616800000005 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664895+0000", 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1910000000000002e-06 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.860257+0000", 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1953617969999999 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.860798+0000", 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00054179800000000002 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.915604+0000", 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.054805622999999998 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.915675+0000", 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0457999999999995e-05 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.915765+0000", 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0779000000000006e-05 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.931361+0000", 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015596099 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.014594+0000", 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083232837000000004 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.014639+0000", 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4660000000000003e-05 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.014691+0000", 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2015999999999998e-05 2024-08-17T17:24:07.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.015877+0000", 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001185712 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138633 9.13 9:cee85b9c:::benchmark_data_smithi142_157452_object6512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.761826+0000", 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.06489113499998, 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2558061010000001, 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.879 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138633 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761826+0000", 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761826+0000", 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761825+0000", 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761831+0000", 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7969999999999997e-06 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761842+0000", 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1164e-05 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761848+0000", 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0299999999999999e-06 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761898+0000", 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9944999999999997e-05 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.761905+0000", 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8190000000000003e-06 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664922+0000", 2024-08-17T17:24:07.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90301752700000004 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664929+0000", 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9439999999999999e-06 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.860831+0000", 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.195901648 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.861442+0000", 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00061123600000000003 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.923439+0000", 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.061997504000000002 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.923548+0000", 2024-08-17T17:24:07.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010867200000000001 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.923696+0000", 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014794400000000001 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.941356+0000", 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017659993999999998 2024-08-17T17:24:07.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.015928+0000", 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.074571719999999994 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.015959+0000", 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1273999999999999e-05 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.015988+0000", 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9068e-05 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.017632+0000", 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001644148 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.888 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138640 9.13 9:ca561b28:::benchmark_data_smithi142_157452_object6519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.769476+0000", 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.05724125199998, 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.250535003, 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138640 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769476+0000", 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769476+0000", 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769474+0000", 2024-08-17T17:24:07.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769482+0000", 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7889999999999995e-06 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769497+0000", 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4928e-05 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769504+0000", 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6200000000000001e-06 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769567+0000", 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3634999999999996e-05 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.769578+0000", 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0956e-05 2024-08-17T17:24:07.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664953+0000", 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89537484099999998 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664957+0000", 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6519999999999999e-06 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.861488+0000", 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.196530928 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.862078+0000", 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00059009999999999998 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.931400+0000", 2024-08-17T17:24:07.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.069322782999999999 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.931509+0000", 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108416 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.931640+0000", 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130696 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.953660+0000", 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022019984999999999 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.017690+0000", 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.064030765000000003 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.017732+0000", 2024-08-17T17:24:07.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1933e-05 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.017767+0000", 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4493000000000003e-05 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.020011+0000", 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022442999999999999 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.897 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138696 9.13 9:cab3dd44:::benchmark_data_smithi142_157452_object6575:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.823830+0000", 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.00288688900002, 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1982261809999999, 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.898 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138696 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823830+0000", 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823830+0000", 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823829+0000", 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823835+0000", 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0480000000000004e-06 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823846+0000", 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0888e-05 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823860+0000", 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3930999999999999e-05 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823913+0000", 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3049e-05 2024-08-17T17:24:07.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.823923+0000", 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5810000000000003e-06 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664987+0000", 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.841064849 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.664996+0000", 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0599999999999997e-06 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.862157+0000", 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1971602510000001 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.864877+0000", 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027206639999999998 2024-08-17T17:24:07.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.941389+0000", 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.076511705999999999 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.941496+0000", 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107046 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.941650+0000", 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000153459 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.960945+0000", 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019294898000000001 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.020057+0000", 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.059112194999999999 2024-08-17T17:24:07.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.020087+0000", 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9957000000000001e-05 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.020134+0000", 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6978999999999999e-05 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.022057+0000", 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019229150000000001 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138721 9.13 9:c8ddf220:::benchmark_data_smithi142_157452_object6600:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.852564+0000", 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.97415342199997, 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.171875139, 2024-08-17T17:24:07.907 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138721 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852564+0000", 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852564+0000", 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852563+0000", 2024-08-17T17:24:07.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:07.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852568+0000", 2024-08-17T17:24:07.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7250000000000002e-06 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852580+0000", 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1436e-05 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852584+0000", 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.583e-06 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852633+0000", 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8177000000000003e-05 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.852639+0000", 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6019999999999996e-06 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665019+0000", 2024-08-17T17:24:07.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81238022300000001 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665035+0000", 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5163e-05 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.864938+0000", 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.199903411 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.865619+0000", 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00068072300000000005 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.951249+0000", 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.085630730000000002 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.951326+0000", 2024-08-17T17:24:07.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6914000000000003e-05 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.951417+0000", 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0327000000000004e-05 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.968977+0000", 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017560414999999999 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.022102+0000", 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.053125196 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.022137+0000", 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4310999999999997e-05 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.022186+0000", 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9478000000000003e-05 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.024439+0000", 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022529099999999999 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138728 9.13 9:cce38b77:::benchmark_data_smithi142_157452_object6607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.856953+0000", 2024-08-17T17:24:07.917 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.96976415900002, 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1698078820000002, 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138728 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.856953+0000", 2024-08-17T17:24:07.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.856953+0000", 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.856952+0000", 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.856958+0000", 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7209999999999999e-06 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.856967+0000", 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1050000000000001e-06 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.856972+0000", 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2580000000000003e-06 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.857040+0000", 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7832999999999999e-05 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.857048+0000", 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4530000000000002e-06 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665066+0000", 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80801820499999999 2024-08-17T17:24:07.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665075+0000", 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5189999999999992e-06 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.865666+0000", 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.200591008 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.866452+0000", 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00078568599999999996 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.958612+0000", 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.092159938999999996 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.958724+0000", 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112675 2024-08-17T17:24:07.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.958884+0000", 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000159772 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.975439+0000", 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016555186 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.024490+0000", 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.049051229000000002 2024-08-17T17:24:07.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.925 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:07 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47197/372654 objects degraded (12.665%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.024522+0000", 2024-08-17T17:24:07.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1353000000000003e-05 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.024569+0000", 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6742000000000001e-05 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.026761+0000", 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021924739999999998 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.927 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138733 9.13 9:cf2aa91c:::benchmark_data_smithi142_157452_object6612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.863486+0000", 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.963231511, 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1657482350000001, 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138733 2024-08-17T17:24:07.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863486+0000", 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863486+0000", 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863484+0000", 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863490+0000", 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4580000000000001e-06 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863504+0000", 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4031e-05 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863510+0000", 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3790000000000003e-06 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863552+0000", 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2435999999999998e-05 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.863562+0000", 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4299999999999995e-06 2024-08-17T17:24:07.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665099+0000", 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80153688599999995 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665103+0000", 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8700000000000002e-06 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.866495+0000", 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2013927200000001 2024-08-17T17:24:07.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.867832+0000", 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001336818 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.966655+0000", 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.098822572999999997 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.966727+0000", 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1959999999999995e-05 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.966815+0000", 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8166999999999998e-05 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.980708+0000", 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013892785 2024-08-17T17:24:07.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.026807+0000", 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046099020999999997 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.026836+0000", 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9345000000000001e-05 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.026892+0000", 2024-08-17T17:24:07.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6376999999999999e-05 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.029234+0000", 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023415599999999999 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138754 9.13 9:c9cf6378:::benchmark_data_smithi142_157452_object6633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:13.886121+0000", 2024-08-17T17:24:07.936 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.94059607399998, 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1454613829999998, 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138754 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886121+0000", 2024-08-17T17:24:07.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886121+0000", 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886119+0000", 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886127+0000", 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3699999999999995e-06 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886141+0000", 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3473e-05 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886146+0000", 2024-08-17T17:24:07.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3360000000000002e-06 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886205+0000", 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8752999999999998e-05 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:13.886216+0000", 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1285e-05 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665130+0000", 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.77891421400000005 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665140+0000", 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6460000000000005e-06 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.867876+0000", 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2027360199999999 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.868516+0000", 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064027599999999995 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.973281+0000", 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.104765171 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.973347+0000", 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5763e-05 2024-08-17T17:24:07.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.973428+0000", 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0487000000000004e-05 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.983567+0000", 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010139574 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.029286+0000", 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.045718479999999999 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.029326+0000", 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0408999999999999e-05 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.029398+0000", 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1653e-05 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.031583+0000", 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021849399999999998 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138883 9.13 9:ce058e95:::benchmark_data_smithi142_157452_object6762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:14.046747+0000", 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.7799698, 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9873402250000001, 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138883 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046747+0000", 2024-08-17T17:24:07.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046747+0000", 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046746+0000", 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046752+0000", 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0739999999999998e-06 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046766+0000", 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3898e-05 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046773+0000", 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7839999999999998e-06 2024-08-17T17:24:07.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046832+0000", 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9043999999999999e-05 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.046841+0000", 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4600000000000003e-06 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665161+0000", 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.61832039299999997 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665165+0000", 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6279999999999998e-06 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.868558+0000", 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2033934689999999 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.869252+0000", 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069352900000000002 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.985914+0000", 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.116662275 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.985980+0000", 2024-08-17T17:24:07.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6272000000000005e-05 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.986072+0000", 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1778000000000001e-05 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.002294+0000", 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016221869 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.031628+0000", 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0293339 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.031656+0000", 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8524e-05 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.031707+0000", 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0399999999999999e-05 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.034088+0000", 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023810609999999999 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138908 9.13 9:c9efe954:::benchmark_data_smithi142_157452_object6787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:14.084636+0000", 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.74208129099998, 2024-08-17T17:24:07.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9519310700000001, 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138908 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084636+0000", 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084636+0000", 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084634+0000", 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084641+0000", 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1010000000000003e-06 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084659+0000", 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8196e-05 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084667+0000", 2024-08-17T17:24:07.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6680000000000001e-06 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084725+0000", 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7485000000000003e-05 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.084733+0000", 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5250000000000005e-06 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665193+0000", 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.58046011200000003 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665203+0000", 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3680000000000008e-06 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.869303+0000", 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2041002030000001 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.869893+0000", 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00059004700000000001 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.993106+0000", 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.123213409 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.993190+0000", 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3763000000000004e-05 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.993295+0000", 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010454 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.006103+0000", 2024-08-17T17:24:07.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012808218999999999 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.034140+0000", 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028037069000000001 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.034178+0000", 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7800999999999999e-05 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.034235+0000", 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6832000000000001e-05 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.036567+0000", 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023325260000000001 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.963 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:138955 9.13 9:ce293646:::benchmark_data_smithi142_157452_object6834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:14.174720+0000", 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.65199772099999, 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8639779089999999, 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138955 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174720+0000", 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174720+0000", 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174718+0000", 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174724+0000", 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.319e-06 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174737+0000", 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2968e-05 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174743+0000", 2024-08-17T17:24:07.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5899999999999998e-06 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174796+0000", 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3520000000000003e-05 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.174807+0000", 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0225e-05 2024-08-17T17:24:07.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665225+0000", 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.49041861399999997 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665229+0000", 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.653e-06 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.869945+0000", 2024-08-17T17:24:07.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2047161179999999 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.870565+0000", 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062053399999999995 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.999844+0000", 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.129278065 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.999923+0000", 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9688e-05 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.000017+0000", 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3412000000000006e-05 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.008510+0000", 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084936839999999996 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.036616+0000", 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028105662999999999 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.036649+0000", 2024-08-17T17:24:07.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3249e-05 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.036690+0000", 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0865000000000003e-05 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.038697+0000", 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020073420000000001 2024-08-17T17:24:07.972 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:139005 9.13 9:cd78a28f:::benchmark_data_smithi142_157452_object6884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:14.344100+0000", 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.48261680000002, 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6963900540000001, 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.973 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 139005 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344100+0000", 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344100+0000", 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344100+0000", 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344104+0000", 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1500000000000001e-06 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344114+0000", 2024-08-17T17:24:07.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7419999999999992e-06 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344118+0000", 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1570000000000002e-06 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344190+0000", 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1790000000000002e-05 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.344196+0000", 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0859999999999998e-06 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665250+0000", 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32105478500000001 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:14.665255+0000", 2024-08-17T17:24:07.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3610000000000003e-06 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.870614+0000", 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.205358854 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:15.871319+0000", 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070534999999999999 2024-08-17T17:24:07.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.006130+0000", 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13481073299999999 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.006211+0000", 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1600000000000005e-05 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:24:07.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.006302+0000", 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0536000000000002e-05 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.011306+0000", 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050044080000000001 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.038747+0000", 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027440688000000001 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.038798+0000", 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0939999999999997e-05 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.038850+0000", 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2009000000000003e-05 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:16.040491+0000", 2024-08-17T17:24:07.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016406389999999999 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:243104 9.15 9:aeacf3aa:::benchmark_data_smithi142_157452_object110983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:21:12.681580+0000", 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 175.14513753700001, 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.052856735, 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.982 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 243104 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681580+0000", 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681580+0000", 2024-08-17T17:24:07.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681579+0000", 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681585+0000", 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9730000000000002e-06 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681595+0000", 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0842000000000001e-05 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681601+0000", 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7169999999999996e-06 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:07.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:07.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681638+0000", 2024-08-17T17:24:07.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6916999999999999e-05 2024-08-17T17:24:07.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.681643+0000", 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3149999999999999e-06 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.703315+0000", 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0216720589999999 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.703423+0000", 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107265 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.703908+0000", 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00048535199999999998 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.710043+0000", 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061352330000000004 2024-08-17T17:24:07.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734357+0000", 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024313905 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734387+0000", 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0119000000000001e-05 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734414+0000", 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7027999999999999e-05 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734436+0000", 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2180000000000001e-05 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:243199 9.15 9:aa356b62:::benchmark_data_smithi142_157452_object111078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:21:12.770386+0000", 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 175.056331207, 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96416222799999995, 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 243199 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770386+0000", 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770386+0000", 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770385+0000", 2024-08-17T17:24:07.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770390+0000", 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8940000000000001e-06 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770400+0000", 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8840000000000006e-06 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:07.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770404+0000", 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1359999999999999e-06 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770448+0000", 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3875999999999998e-05 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.770456+0000", 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9379999999999992e-06 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.706168+0000", 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93571260899999997 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.706233+0000", 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4325000000000005e-05 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.706496+0000", 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026326700000000003 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.714135+0000", 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076384720000000003 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734474+0000", 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020339079999999999 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734499+0000", 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5442000000000001e-05 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734530+0000", 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1131999999999999e-05 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734548+0000", 2024-08-17T17:24:07.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8125999999999999e-05 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:243207 9.15 9:a86d7dcf:::benchmark_data_smithi142_157452_object111086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:21:12.777271+0000", 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 175.049446254, 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.957536418, 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:07.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 243207 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777271+0000", 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777271+0000", 2024-08-17T17:24:07.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777269+0000", 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777277+0000", 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9980000000000003e-06 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777301+0000", 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3116999999999999e-05 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777307+0000", 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2160000000000003e-06 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:08.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777370+0000", 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2891000000000005e-05 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.777385+0000", 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5712000000000001e-05 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.707896+0000", 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93051052899999998 2024-08-17T17:24:08.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.707960+0000", 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3641999999999998e-05 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.708192+0000", 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000232876 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.717704+0000", 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0095115930000000005 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734623+0000", 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016918533999999999 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734724+0000", 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101214 2024-08-17T17:24:08.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734768+0000", 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4623000000000003e-05 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734807+0000", 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9047000000000001e-05 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:08.005 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:243239 9.15 9:ab10e77e:::benchmark_data_smithi142_157452_object111118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:21:12.808064+0000", 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 175.018652938, 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92688289999999995, 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:24:08.006 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 243239 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808064+0000", 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808064+0000", 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808063+0000", 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808068+0000", 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9860000000000002e-06 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808079+0000", 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0655999999999999e-05 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808083+0000", 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2729999999999996e-06 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808131+0000", 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7364999999999998e-05 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:12.808140+0000", 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2839999999999995e-06 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.710076+0000", 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90193653500000004 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.710149+0000", 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2735000000000001e-05 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:24:08.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.710473+0000", 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032392 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.717711+0000", 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072377860000000004 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734671+0000", 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016960277999999999 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734859+0000", 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000188018 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734907+0000", 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7788999999999999e-05 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:21:13.734947+0000", 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0326e-05 2024-08-17T17:24:08.012 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:08.013 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:08.013 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:08.013 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:08.013 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:08.013 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:24:08.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:08.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:08.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:07 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47197/372654 objects degraded (12.665%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:08.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:07 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47197/372654 objects degraded (12.665%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:08.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:08.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:08.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:08.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:08.510 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:08.554 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:08.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:08.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:08.789 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:08.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:08 smithi152 ceph-mon[93456]: pgmap v790: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 144 B/s rd, 3.8 MiB/s wr, 281 op/s; 47197/372654 objects degraded (12.665%); 30899/372654 objects misplaced (8.292%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:24:08.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:08.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:24:08.928 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:08.966 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:09.068 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:09.110 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:09.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:08 smithi142 ceph-mon[103614]: pgmap v790: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 144 B/s rd, 3.8 MiB/s wr, 281 op/s; 47197/372654 objects degraded (12.665%); 30899/372654 objects misplaced (8.292%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:24:09.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:08 smithi142 ceph-mon[106729]: pgmap v790: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 144 B/s rd, 3.8 MiB/s wr, 281 op/s; 47197/372654 objects degraded (12.665%); 30899/372654 objects misplaced (8.292%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:24:09.178 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:09.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:09.259 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:09.275 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:09.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:09.331 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:09.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:09.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:09.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:09.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:09.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:09.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:09.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:09.836 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:09.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:09.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:10.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:10.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:10.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:10.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:10.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:10.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:10.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:10.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:10.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:24:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:24:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:24:10.653 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:10.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:10.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:10.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:10.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:10 smithi152 ceph-mon[93456]: pgmap v791: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 126 B/s rd, 2.6 MiB/s wr, 475 op/s; 47347/366798 objects degraded (12.908%); 30580/366798 objects misplaced (8.337%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:24:10.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:10.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:11.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:11.102 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:10 smithi142 ceph-mon[103614]: pgmap v791: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 126 B/s rd, 2.6 MiB/s wr, 475 op/s; 47347/366798 objects degraded (12.908%); 30580/366798 objects misplaced (8.337%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:24:11.102 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:10 smithi142 ceph-mon[106729]: pgmap v791: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 126 B/s rd, 2.6 MiB/s wr, 475 op/s; 47347/366798 objects degraded (12.908%); 30580/366798 objects misplaced (8.337%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:24:11.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.110 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:11.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:11.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:11.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:11.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:11.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:11.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:11.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:11.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:11.617 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:131424 10.9 e244/222 10:9755f0d9:::benchmark_data_smithi142_192453_object5676:head v 244'8165, mlcod=244'8163)", 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:11.614901+0000", 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00055160500000000004, 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00057931400000000004, 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:24:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614901+0000", 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614901+0000", 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614900+0000", 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614907+0000", 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5809999999999999e-06 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614920+0000", 2024-08-17T17:24:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3162e-05 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614926+0000", 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.139e-06 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.614978+0000", 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1904999999999999e-05 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:24:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:11.615001+0000", 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2733000000000001e-05 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 1 2024-08-17T17:24:11.622 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:24:11.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:11.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:11.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:11.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:11.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:11 smithi152 ceph-mon[93456]: from='client.46214 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:11.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:11.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:12.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:12.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:12.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-mon[103614]: from='client.46214 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:12.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:24:11] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-08-17T17:24:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-mon[106729]: from='client.46214 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:12.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:12.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:12.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:12.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:12.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:12.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:12.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:24:12.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:12.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:12.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:12.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:12.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.796 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:12.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:12.798 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:24:12.801 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:24:12.861 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:12.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:12.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:13.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:12 smithi152 ceph-mon[93456]: pgmap v792: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 2.1 MiB/s wr, 404 op/s; 47347/366174 objects degraded (12.930%); 30580/366174 objects misplaced (8.351%); 2.6 MiB/s, 40 objects/s recovering 2024-08-17T17:24:13.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:13.034 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:12 smithi142 ceph-mon[103614]: pgmap v792: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 2.1 MiB/s wr, 404 op/s; 47347/366174 objects degraded (12.930%); 30580/366174 objects misplaced (8.351%); 2.6 MiB/s, 40 objects/s recovering 2024-08-17T17:24:13.035 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:13.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:12 smithi142 ceph-mon[106729]: pgmap v792: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 102 B/s rd, 2.1 MiB/s wr, 404 op/s; 47347/366174 objects degraded (12.930%); 30580/366174 objects misplaced (8.351%); 2.6 MiB/s, 40 objects/s recovering 2024-08-17T17:24:13.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:13.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:13.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:13.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:13.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:13.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:13.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:13.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:13.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:13.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:13.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:13.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:13.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:13.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:13.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:13.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:13.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:14.024 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:13 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47468/360288 objects degraded (13.175%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:14.024 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:13 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47468/360288 objects degraded (13.175%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:14.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:13 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47468/360288 objects degraded (13.175%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:14.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:14.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:14.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:14.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:14.335 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:14.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:14.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:14.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:14.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:14.649 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:24:14.650 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:14.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:14.694 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:24:14.694 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:14.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:24:14.736 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:14.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:14.788 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:24:14.788 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:14.815 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:14.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:14.822 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:24:14.822 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:14.858 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:24:14.859 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:14.901 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:24:14.902 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:14.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:14.941 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:24:14.941 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:14.952 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:14.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:14 smithi142 ceph-mon[103614]: pgmap v793: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 568 KiB/s wr, 505 op/s; 47468/360288 objects degraded (13.175%); 30012/360288 objects misplaced (8.330%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:24:14.954 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3913664229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:14.955 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:14 smithi142 ceph-mon[106729]: pgmap v793: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 568 KiB/s wr, 505 op/s; 47468/360288 objects degraded (13.175%); 30012/360288 objects misplaced (8.330%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:24:14.955 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3913664229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:15.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:14 smithi152 ceph-mon[93456]: pgmap v793: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 568 KiB/s wr, 505 op/s; 47468/360288 objects degraded (13.175%); 30012/360288 objects misplaced (8.330%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:24:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3913664229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:15.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:15.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:15.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:15.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:15.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:15.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:15.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:15.524 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:15.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:15.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:15.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:15.801 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:24:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:15.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:15.842 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:15.842 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:15.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:15.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/483647101' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:15 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:16.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:16.081 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/483647101' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:16.081 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:15 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:16.081 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/483647101' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:16.081 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:15 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:16.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:16.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:16.130 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":791,"stamp":"2024-08-17T17:24:09.517429+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8011651045,"num_objects":122266,"num_object_clones":0,"num_object_copies":366798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47347,"num_objects_misplaced":30580,"num_objects_unfound":0,"num_objects_dirty":122266,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":129530,"num_write_kb":8050128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":28707536,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123782,"ondisk_log_size":123782,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23306720,"kb_used_data":18663208,"kb_used_omap":29,"kb_used_meta":4643426,"kb_avail":632909344,"statfs":{"total":671965249536,"available":648099168256,"internally_reserved":0,"allocated":19111124992,"data_stored":19102724608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30272,"internal_metadata":4754868672},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,9,6,4,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-208142312,"num_objects":-3175,"num_object_clones":0,"num_object_copies":-9525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":39522,"num_objects_misplaced":30580,"num_objects_unfound":0,"num_objects_dirty":-3175,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3837,"num_write_kb":21121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":26804790,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.065929"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":298,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.719298+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:24:02.719298+0000","last_peered":"2024-08-17T17:24:02.719298+0000","last_clean":"2024-08-17T17:24:02.719298+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:24:02.719298+0000","last_undegraded":"2024-08-17T17:24:02.719298+0000","last_fullsized":"2024-08-17T17:24:02.719298+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":201,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453433+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:24:01.453433+0000","last_peered":"2024-08-17T17:24:01.453433+0000","last_clean":"2024-08-17T17:24:01.453433+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:24:01.453433+0000","last_undegraded":"2024-08-17T17:24:01.453433+0000","last_fullsized":"2024-08-17T17:24:01.453433+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":181,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.452905+0000","last_change":"2024-08-17T17:21:31.903762+0000","last_active":"2024-08-17T17:24:01.452905+0000","last_peered":"2024-08-17T17:24:01.452905+0000","last_clean":"2024-08-17T17:24:01.452905+0000","last_became_active":"2024-08-17T17:21:31.902518+0000","last_became_peered":"2024-08-17T17:21:31.902518+0000","last_unstale":"2024-08-17T17:24:01.452905+0000","last_undegraded":"2024-08-17T17:24:01.452905+0000","last_fullsized":"2024-08-17T17:24:01.452905+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:13.790840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453210+0000","last_change":"2024-08-17T17:23:58.437765+0000","last_active":"2024-08-17T17:24:01.453210+0000","last_peered":"2024-08-17T17:24:01.453210+0000","last_clean":"2024-08-17T17:24:01.453210+0000","last_became_active":"2024-08-17T17:23:58.437424+0000","last_became_peered":"2024-08-17T17:23:58.437424+0000","last_unstale":"2024-08-17T17:24:01.453210+0000","last_undegraded":"2024-08-17T17:24:01.453210+0000","last_fullsized":"2024-08-17T17:24:01.453210+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:13.505002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.718606+0000","last_change":"2024-08-17T17:23:58.435501+0000","last_active":"2024-08-17T17:24:02.718606+0000","last_peered":"2024-08-17T17:24:02.718606+0000","last_clean":"2024-08-17T17:24:02.718606+0000","last_became_active":"2024-08-17T17:23:58.435183+0000","last_became_peered":"2024-08-17T17:23:58.435183+0000","last_unstale":"2024-08-17T17:24:02.718606+0000","last_undegraded":"2024-08-17T17:24:02.718606+0000","last_fullsized":"2024-08-17T17:24:02.718606+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:28.022038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453068+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:24:01.453068+0000","last_peered":"2024-08-17T17:24:01.453068+0000","last_clean":"2024-08-17T17:24:01.453068+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:24:01.453068+0000","last_undegraded":"2024-08-17T17:24:01.453068+0000","last_fullsized":"2024-08-17T17:24:01.453068+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":274,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.718955+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:24:02.718955+0000","last_peered":"2024-08-17T17:24:02.718955+0000","last_clean":"2024-08-17T17:24:02.718955+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:24:02.718955+0000","last_undegraded":"2024-08-17T17:24:02.718955+0000","last_fullsized":"2024-08-17T17:24:02.718955+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":244,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.455312+0000","last_change":"2024-08-17T17:23:58.452876+0000","last_active":"2024-08-17T17:24:01.455312+0000","last_peered":"2024-08-17T17:24:01.455312+0000","last_clean":"2024-08-17T17:24:01.455312+0000","last_became_active":"2024-08-17T17:23:58.439881+0000","last_became_peered":"2024-08-17T17:23:58.439881+0000","last_unstale":"2024-08-17T17:24:01.455312+0000","last_undegraded":"2024-08-17T17:24:01.455312+0000","last_fullsized":"2024-08-17T17:24:01.455312+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:49.106328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"244'8007","reported_seq":8063,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:07.321752+0000","last_change":"2024-08-17T17:23:59.914129+0000","last_active":"2024-08-17T17:24:07.321752+0000","last_peered":"2024-08-17T17:24:07.321752+0000","last_clean":"2024-08-17T17:23:57.406704+0000","last_became_active":"2024-08-17T17:23:59.866030+0000","last_became_peered":"2024-08-17T17:23:59.866030+0000","last_unstale":"2024-08-17T17:24:07.321752+0000","last_undegraded":"2024-08-17T17:23:59.516292+0000","last_fullsized":"2024-08-17T17:23:59.430718+0000","mapping_epoch":240,"log_start":"227'863","ondisk_log_start":"227'863","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7144,"log_dups_size":0,"ondisk_log_size":7144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:07.743208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":499843072,"num_objects":7627,"num_object_clones":0,"num_object_copies":22881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7805,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8007,"num_write_kb":500288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":7615},{"shards":"4,5","objects":-178}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":214,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.454243+0000","last_change":"2024-08-17T17:23:58.447195+0000","last_active":"2024-08-17T17:24:01.454243+0000","last_peered":"2024-08-17T17:24:01.454243+0000","last_clean":"2024-08-17T17:24:01.454243+0000","last_became_active":"2024-08-17T17:23:58.438918+0000","last_became_peered":"2024-08-17T17:23:58.438918+0000","last_unstale":"2024-08-17T17:24:01.454243+0000","last_undegraded":"2024-08-17T17:24:01.454243+0000","last_fullsized":"2024-08-17T17:24:01.454243+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:26.476424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":87,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.452843+0000","last_change":"2024-08-17T17:21:31.876798+0000","last_active":"2024-08-17T17:24:01.452843+0000","last_peered":"2024-08-17T17:24:01.452843+0000","last_clean":"2024-08-17T17:24:01.452843+0000","last_became_active":"2024-08-17T17:21:31.876370+0000","last_became_peered":"2024-08-17T17:21:31.876370+0000","last_unstale":"2024-08-17T17:24:01.452843+0000","last_undegraded":"2024-08-17T17:24:01.452843+0000","last_fullsized":"2024-08-17T17:24:01.452843+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:33:16.352719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"244'8051","reported_seq":8081,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:07.636506+0000","last_change":"2024-08-17T17:22:18.544826+0000","last_active":"2024-08-17T17:24:07.636506+0000","last_peered":"2024-08-17T17:24:07.636506+0000","last_clean":"2024-08-17T17:24:07.636506+0000","last_became_active":"2024-08-17T17:22:18.544566+0000","last_became_peered":"2024-08-17T17:22:18.544566+0000","last_unstale":"2024-08-17T17:24:07.636506+0000","last_undegraded":"2024-08-17T17:24:07.636506+0000","last_fullsized":"2024-08-17T17:24:07.636506+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8051,"log_dups_size":0,"ondisk_log_size":8051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:08.783581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":500105216,"num_objects":7631,"num_object_clones":0,"num_object_copies":22893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8051,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":201,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453992+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:24:01.453992+0000","last_peered":"2024-08-17T17:24:01.453992+0000","last_clean":"2024-08-17T17:24:01.453992+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:24:01.453992+0000","last_undegraded":"2024-08-17T17:24:01.453992+0000","last_fullsized":"2024-08-17T17:24:01.453992+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":215,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.589841+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:24:01.589841+0000","last_peered":"2024-08-17T17:24:01.589841+0000","last_clean":"2024-08-17T17:24:01.589841+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:24:01.589841+0000","last_undegraded":"2024-08-17T17:24:01.589841+0000","last_fullsized":"2024-08-17T17:24:01.589841+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"244'8099","reported_seq":8164,"reported_epoch":244,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:07.267366+0000","last_change":"2024-08-17T17:23:59.866912+0000","last_active":"2024-08-17T17:24:07.267366+0000","last_peered":"2024-08-17T17:24:07.267366+0000","last_clean":"2024-08-17T17:23:57.415609+0000","last_became_active":"2024-08-17T17:23:59.858662+0000","last_became_peered":"2024-08-17T17:23:59.858662+0000","last_unstale":"2024-08-17T17:24:07.267366+0000","last_undegraded":"2024-08-17T17:24:07.267366+0000","last_fullsized":"2024-08-17T17:24:07.267366+0000","mapping_epoch":240,"log_start":"227'933","ondisk_log_start":"227'933","created":222,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7166,"log_dups_size":0,"ondisk_log_size":7166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:07.370985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":503906304,"num_objects":7689,"num_object_clones":0,"num_object_copies":23067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7689,"num_objects_unfound":0,"num_objects_dirty":7689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8099,"num_write_kb":505216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,0],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":7689}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":200,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.454143+0000","last_change":"2024-08-17T17:21:31.904014+0000","last_active":"2024-08-17T17:24:01.454143+0000","last_peered":"2024-08-17T17:24:01.454143+0000","last_clean":"2024-08-17T17:24:01.454143+0000","last_became_active":"2024-08-17T17:21:31.903128+0000","last_became_peered":"2024-08-17T17:21:31.903128+0000","last_unstale":"2024-08-17T17:24:01.454143+0000","last_undegraded":"2024-08-17T17:24:01.454143+0000","last_fullsized":"2024-08-17T17:24:01.454143+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:56:59.504309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.455472+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:24:01.455472+0000","last_peered":"2024-08-17T17:24:01.455472+0000","last_clean":"2024-08-17T17:24:01.455472+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:24:01.455472+0000","last_undegraded":"2024-08-17T17:24:01.455472+0000","last_fullsized":"2024-08-17T17:24:01.455472+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"244'8293","reported_seq":8350,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:08.832226+0000","last_change":"2024-08-17T17:23:59.905221+0000","last_active":"2024-08-17T17:24:08.832226+0000","last_peered":"2024-08-17T17:24:08.832226+0000","last_clean":"2024-08-17T17:23:57.376065+0000","last_became_active":"2024-08-17T17:23:59.892377+0000","last_became_peered":"2024-08-17T17:23:59.892377+0000","last_unstale":"2024-08-17T17:24:08.832226+0000","last_undegraded":"2024-08-17T17:23:59.483921+0000","last_fullsized":"2024-08-17T17:23:59.427321+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8293,"log_dups_size":0,"ondisk_log_size":8293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":509804544,"num_objects":7779,"num_object_clones":0,"num_object_copies":23337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8016,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8293,"num_write_kb":514304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":7759},{"shards":"2,3","objects":-237}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":235,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.719406+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:24:02.719406+0000","last_peered":"2024-08-17T17:24:02.719406+0000","last_clean":"2024-08-17T17:24:02.719406+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:24:02.719406+0000","last_undegraded":"2024-08-17T17:24:02.719406+0000","last_fullsized":"2024-08-17T17:24:02.719406+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":79,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711217+0000","last_change":"2024-08-17T17:23:58.446251+0000","last_active":"2024-08-17T17:24:02.711217+0000","last_peered":"2024-08-17T17:24:02.711217+0000","last_clean":"2024-08-17T17:24:02.711217+0000","last_became_active":"2024-08-17T17:23:58.445949+0000","last_became_peered":"2024-08-17T17:23:58.445949+0000","last_unstale":"2024-08-17T17:24:02.711217+0000","last_undegraded":"2024-08-17T17:24:02.711217+0000","last_fullsized":"2024-08-17T17:24:02.711217+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:12.605051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"244'7997","reported_seq":8053,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:05.931851+0000","last_change":"2024-08-17T17:23:59.866840+0000","last_active":"2024-08-17T17:24:05.931851+0000","last_peered":"2024-08-17T17:24:05.931851+0000","last_clean":"2024-08-17T17:23:57.419490+0000","last_became_active":"2024-08-17T17:23:59.850671+0000","last_became_peered":"2024-08-17T17:23:59.850671+0000","last_unstale":"2024-08-17T17:24:05.931851+0000","last_undegraded":"2024-08-17T17:23:59.477480+0000","last_fullsized":"2024-08-17T17:23:59.431606+0000","mapping_epoch":240,"log_start":"228'1803","ondisk_log_start":"228'1803","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504168448,"num_objects":7693,"num_object_clones":0,"num_object_copies":23079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7825,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7997,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":7673},{"shards":"1,4","objects":-132}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":314,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.727266+0000","last_change":"2024-08-17T17:23:58.452037+0000","last_active":"2024-08-17T17:24:02.727266+0000","last_peered":"2024-08-17T17:24:02.727266+0000","last_clean":"2024-08-17T17:24:02.727266+0000","last_became_active":"2024-08-17T17:23:58.446461+0000","last_became_peered":"2024-08-17T17:23:58.446461+0000","last_unstale":"2024-08-17T17:24:02.727266+0000","last_undegraded":"2024-08-17T17:24:02.727266+0000","last_fullsized":"2024-08-17T17:24:02.727266+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:30.857645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590197+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:24:01.590197+0000","last_peered":"2024-08-17T17:24:01.590197+0000","last_clean":"2024-08-17T17:24:01.590197+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:24:01.590197+0000","last_undegraded":"2024-08-17T17:24:01.590197+0000","last_fullsized":"2024-08-17T17:24:01.590197+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"244'7977","reported_seq":8032,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:08.899654+0000","last_change":"2024-08-17T17:22:18.541297+0000","last_active":"2024-08-17T17:24:08.899654+0000","last_peered":"2024-08-17T17:24:08.899654+0000","last_clean":"2024-08-17T17:24:08.899654+0000","last_became_active":"2024-08-17T17:22:18.541002+0000","last_became_peered":"2024-08-17T17:22:18.541002+0000","last_unstale":"2024-08-17T17:24:08.899654+0000","last_undegraded":"2024-08-17T17:24:08.899654+0000","last_fullsized":"2024-08-17T17:24:08.899654+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:07.861642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":490143744,"num_objects":7479,"num_object_clones":0,"num_object_copies":22437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7977,"num_write_kb":494592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.455240+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:24:01.455240+0000","last_peered":"2024-08-17T17:24:01.455240+0000","last_clean":"2024-08-17T17:24:01.455240+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:24:01.455240+0000","last_undegraded":"2024-08-17T17:24:01.455240+0000","last_fullsized":"2024-08-17T17:24:01.455240+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":215,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590272+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:24:01.590272+0000","last_peered":"2024-08-17T17:24:01.590272+0000","last_clean":"2024-08-17T17:24:01.590272+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:24:01.590272+0000","last_undegraded":"2024-08-17T17:24:01.590272+0000","last_fullsized":"2024-08-17T17:24:01.590272+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"244'7944","reported_seq":7974,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:07.646150+0000","last_change":"2024-08-17T17:22:18.545009+0000","last_active":"2024-08-17T17:24:07.646150+0000","last_peered":"2024-08-17T17:24:07.646150+0000","last_clean":"2024-08-17T17:24:07.646150+0000","last_became_active":"2024-08-17T17:22:18.544525+0000","last_became_peered":"2024-08-17T17:22:18.544525+0000","last_unstale":"2024-08-17T17:24:07.646150+0000","last_undegraded":"2024-08-17T17:24:07.646150+0000","last_fullsized":"2024-08-17T17:24:07.646150+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7944,"log_dups_size":0,"ondisk_log_size":7944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:35.215727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":492044288,"num_objects":7508,"num_object_clones":0,"num_object_copies":22524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7944,"num_write_kb":494464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":199,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.589944+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:24:01.589944+0000","last_peered":"2024-08-17T17:24:01.589944+0000","last_clean":"2024-08-17T17:24:01.589944+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:24:01.589944+0000","last_undegraded":"2024-08-17T17:24:01.589944+0000","last_fullsized":"2024-08-17T17:24:01.589944+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590576+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:24:01.590576+0000","last_peered":"2024-08-17T17:24:01.590576+0000","last_clean":"2024-08-17T17:24:01.590576+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:24:01.590576+0000","last_undegraded":"2024-08-17T17:24:01.590576+0000","last_fullsized":"2024-08-17T17:24:01.590576+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"244'8156","reported_seq":8274,"reported_epoch":244,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:24:07.267399+0000","last_change":"2024-08-17T17:23:59.914309+0000","last_active":"2024-08-17T17:24:07.267399+0000","last_peered":"2024-08-17T17:24:07.267399+0000","last_clean":"2024-08-17T17:23:57.403207+0000","last_became_active":"2024-08-17T17:23:59.662728+0000","last_became_peered":"2024-08-17T17:23:59.662728+0000","last_unstale":"2024-08-17T17:24:07.267399+0000","last_undegraded":"2024-08-17T17:24:07.267399+0000","last_fullsized":"2024-08-17T17:24:07.267399+0000","mapping_epoch":240,"log_start":"227'1002","ondisk_log_start":"227'1002","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7154,"log_dups_size":0,"ondisk_log_size":7154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:42.692007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":509476864,"num_objects":7774,"num_object_clones":0,"num_object_copies":23322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7738,"num_objects_unfound":0,"num_objects_dirty":7774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8156,"num_write_kb":509760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":7774}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":276,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.717812+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:24:02.717812+0000","last_peered":"2024-08-17T17:24:02.717812+0000","last_clean":"2024-08-17T17:24:02.717812+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:24:02.717812+0000","last_undegraded":"2024-08-17T17:24:02.717812+0000","last_fullsized":"2024-08-17T17:24:02.717812+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":231,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.719025+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:24:02.719025+0000","last_peered":"2024-08-17T17:24:02.719025+0000","last_clean":"2024-08-17T17:24:02.719025+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:24:02.719025+0000","last_undegraded":"2024-08-17T17:24:02.719025+0000","last_fullsized":"2024-08-17T17:24:02.719025+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"244'8078","reported_seq":8118,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:07.611220+0000","last_change":"2024-08-17T17:23:59.929968+0000","last_active":"2024-08-17T17:24:07.611220+0000","last_peered":"2024-08-17T17:24:07.611220+0000","last_clean":"2024-08-17T17:23:57.402090+0000","last_became_active":"2024-08-17T17:23:59.911485+0000","last_became_peered":"2024-08-17T17:23:59.911485+0000","last_unstale":"2024-08-17T17:24:07.611220+0000","last_undegraded":"2024-08-17T17:23:59.498287+0000","last_fullsized":"2024-08-17T17:23:59.430373+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8078,"log_dups_size":0,"ondisk_log_size":8078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:03.059493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":500826112,"num_objects":7642,"num_object_clones":0,"num_object_copies":22926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7839,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8078,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":7621},{"shards":"1,6","objects":-197}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":565,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.458988+0000","last_change":"2024-08-17T17:24:01.458048+0000","last_active":"2024-08-17T17:24:01.458988+0000","last_peered":"2024-08-17T17:24:01.458988+0000","last_clean":"2024-08-17T17:24:01.458988+0000","last_became_active":"2024-08-17T17:24:01.457007+0000","last_became_peered":"2024-08-17T17:24:01.457007+0000","last_unstale":"2024-08-17T17:24:01.458988+0000","last_undegraded":"2024-08-17T17:24:01.458988+0000","last_fullsized":"2024-08-17T17:24:01.458988+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:12.738984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":150,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.712591+0000","last_change":"2024-08-17T17:21:31.878312+0000","last_active":"2024-08-17T17:24:02.712591+0000","last_peered":"2024-08-17T17:24:02.712591+0000","last_clean":"2024-08-17T17:24:02.712591+0000","last_became_active":"2024-08-17T17:21:31.877932+0000","last_became_peered":"2024-08-17T17:21:31.877932+0000","last_unstale":"2024-08-17T17:24:02.712591+0000","last_undegraded":"2024-08-17T17:24:02.712591+0000","last_fullsized":"2024-08-17T17:24:02.712591+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:17.773813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":273,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.718294+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:24:02.718294+0000","last_peered":"2024-08-17T17:24:02.718294+0000","last_clean":"2024-08-17T17:24:02.718294+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:24:02.718294+0000","last_undegraded":"2024-08-17T17:24:02.718294+0000","last_fullsized":"2024-08-17T17:24:02.718294+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.455846+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:24:01.455846+0000","last_peered":"2024-08-17T17:24:01.455846+0000","last_clean":"2024-08-17T17:24:01.455846+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:24:01.455846+0000","last_undegraded":"2024-08-17T17:24:01.455846+0000","last_fullsized":"2024-08-17T17:24:01.455846+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"244'8082","reported_seq":8596,"reported_epoch":244,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:08.939716+0000","last_change":"2024-08-17T17:24:04.917842+0000","last_active":"2024-08-17T17:24:08.939716+0000","last_peered":"2024-08-17T17:24:08.939716+0000","last_clean":"2024-08-17T17:23:57.421745+0000","last_became_active":"2024-08-17T17:23:59.700627+0000","last_became_peered":"2024-08-17T17:23:59.700627+0000","last_unstale":"2024-08-17T17:24:08.939716+0000","last_undegraded":"2024-08-17T17:24:04.913309+0000","last_fullsized":"2024-08-17T17:23:59.426978+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8082,"log_dups_size":0,"ondisk_log_size":8082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":492830720,"num_objects":7520,"num_object_clones":0,"num_object_copies":22560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":62,"num_objects_misplaced":7501,"num_objects_unfound":0,"num_objects_dirty":7520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8082,"num_write_kb":499264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":222,"num_bytes_recovered":14548992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2,7","objects":7501},{"shards":"2,7","objects":-43}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":238,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.452993+0000","last_change":"2024-08-17T17:23:58.442290+0000","last_active":"2024-08-17T17:24:01.452993+0000","last_peered":"2024-08-17T17:24:01.452993+0000","last_clean":"2024-08-17T17:24:01.452993+0000","last_became_active":"2024-08-17T17:23:58.441979+0000","last_became_peered":"2024-08-17T17:23:58.441979+0000","last_unstale":"2024-08-17T17:24:01.452993+0000","last_undegraded":"2024-08-17T17:24:01.452993+0000","last_fullsized":"2024-08-17T17:24:01.452993+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:02:14.599058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":170,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.589754+0000","last_change":"2024-08-17T17:23:58.435132+0000","last_active":"2024-08-17T17:24:01.589754+0000","last_peered":"2024-08-17T17:24:01.589754+0000","last_clean":"2024-08-17T17:24:01.589754+0000","last_became_active":"2024-08-17T17:23:58.434623+0000","last_became_peered":"2024-08-17T17:23:58.434623+0000","last_unstale":"2024-08-17T17:24:01.589754+0000","last_undegraded":"2024-08-17T17:24:01.589754+0000","last_fullsized":"2024-08-17T17:24:01.589754+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:54:06.724484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711105+0000","last_change":"2024-08-17T17:23:58.444515+0000","last_active":"2024-08-17T17:24:02.711105+0000","last_peered":"2024-08-17T17:24:02.711105+0000","last_clean":"2024-08-17T17:24:02.711105+0000","last_became_active":"2024-08-17T17:23:58.444124+0000","last_became_peered":"2024-08-17T17:23:58.444124+0000","last_unstale":"2024-08-17T17:24:02.711105+0000","last_undegraded":"2024-08-17T17:24:02.711105+0000","last_fullsized":"2024-08-17T17:24:02.711105+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:07.046290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"244'8074","reported_seq":8129,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:08.900502+0000","last_change":"2024-08-17T17:22:18.540626+0000","last_active":"2024-08-17T17:24:08.900502+0000","last_peered":"2024-08-17T17:24:08.900502+0000","last_clean":"2024-08-17T17:24:08.900502+0000","last_became_active":"2024-08-17T17:22:18.540354+0000","last_became_peered":"2024-08-17T17:22:18.540354+0000","last_unstale":"2024-08-17T17:24:08.900502+0000","last_undegraded":"2024-08-17T17:24:08.900502+0000","last_fullsized":"2024-08-17T17:24:08.900502+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8074,"log_dups_size":0,"ondisk_log_size":8074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:24:36.914893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":497287168,"num_objects":7588,"num_object_clones":0,"num_object_copies":22764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8074,"num_write_kb":501184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":342,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.708715+0000","last_change":"2024-08-17T17:23:58.446239+0000","last_active":"2024-08-17T17:24:02.708715+0000","last_peered":"2024-08-17T17:24:02.708715+0000","last_clean":"2024-08-17T17:24:02.708715+0000","last_became_active":"2024-08-17T17:23:58.445295+0000","last_became_peered":"2024-08-17T17:23:58.445295+0000","last_unstale":"2024-08-17T17:24:02.708715+0000","last_undegraded":"2024-08-17T17:24:02.708715+0000","last_fullsized":"2024-08-17T17:24:02.708715+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:47.985099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":316,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.710857+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:24:02.710857+0000","last_peered":"2024-08-17T17:24:02.710857+0000","last_clean":"2024-08-17T17:24:02.710857+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:24:02.710857+0000","last_undegraded":"2024-08-17T17:24:02.710857+0000","last_fullsized":"2024-08-17T17:24:02.710857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":359,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.710997+0000","last_change":"2024-08-17T17:23:58.443697+0000","last_active":"2024-08-17T17:24:02.710997+0000","last_peered":"2024-08-17T17:24:02.710997+0000","last_clean":"2024-08-17T17:24:02.710997+0000","last_became_active":"2024-08-17T17:23:58.443331+0000","last_became_peered":"2024-08-17T17:23:58.443331+0000","last_unstale":"2024-08-17T17:24:02.710997+0000","last_undegraded":"2024-08-17T17:24:02.710997+0000","last_fullsized":"2024-08-17T17:24:02.710997+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:16.757972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"244'8076","reported_seq":8116,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:07.619867+0000","last_change":"2024-08-17T17:23:59.885174+0000","last_active":"2024-08-17T17:24:07.619867+0000","last_peered":"2024-08-17T17:24:07.619867+0000","last_clean":"2024-08-17T17:23:57.418972+0000","last_became_active":"2024-08-17T17:23:59.864453+0000","last_became_peered":"2024-08-17T17:23:59.864453+0000","last_unstale":"2024-08-17T17:24:07.619867+0000","last_undegraded":"2024-08-17T17:23:59.504346+0000","last_fullsized":"2024-08-17T17:23:59.429730+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8076,"log_dups_size":0,"ondisk_log_size":8076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:21.044601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":498073600,"num_objects":7600,"num_object_clones":0,"num_object_copies":22800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7820,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8076,"num_write_kb":501632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":7582},{"shards":"5,6","objects":-220}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":157,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711413+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:24:02.711413+0000","last_peered":"2024-08-17T17:24:02.711413+0000","last_clean":"2024-08-17T17:24:02.711413+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:24:02.711413+0000","last_undegraded":"2024-08-17T17:24:02.711413+0000","last_fullsized":"2024-08-17T17:24:02.711413+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":209,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590270+0000","last_change":"2024-08-17T17:23:58.443591+0000","last_active":"2024-08-17T17:24:01.590270+0000","last_peered":"2024-08-17T17:24:01.590270+0000","last_clean":"2024-08-17T17:24:01.590270+0000","last_became_active":"2024-08-17T17:23:58.443299+0000","last_became_peered":"2024-08-17T17:23:58.443299+0000","last_unstale":"2024-08-17T17:24:01.590270+0000","last_undegraded":"2024-08-17T17:24:01.590270+0000","last_fullsized":"2024-08-17T17:24:01.590270+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:05.734769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":327,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711272+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:24:02.711272+0000","last_peered":"2024-08-17T17:24:02.711272+0000","last_clean":"2024-08-17T17:24:02.711272+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:24:02.711272+0000","last_undegraded":"2024-08-17T17:24:02.711272+0000","last_fullsized":"2024-08-17T17:24:02.711272+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"244'8126","reported_seq":8449,"reported_epoch":244,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:08.020933+0000","last_change":"2024-08-17T17:24:03.931563+0000","last_active":"2024-08-17T17:24:08.020933+0000","last_peered":"2024-08-17T17:24:08.020933+0000","last_clean":"2024-08-17T17:23:57.419041+0000","last_became_active":"2024-08-17T17:23:59.872933+0000","last_became_peered":"2024-08-17T17:23:59.872933+0000","last_unstale":"2024-08-17T17:24:08.020933+0000","last_undegraded":"2024-08-17T17:24:03.917566+0000","last_fullsized":"2024-08-17T17:23:59.429332+0000","mapping_epoch":240,"log_start":"226'200","ondisk_log_start":"226'200","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7926,"log_dups_size":0,"ondisk_log_size":7926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:58:11.160428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":502530048,"num_objects":7668,"num_object_clones":0,"num_object_copies":23004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":88,"num_objects_misplaced":7652,"num_objects_unfound":0,"num_objects_dirty":7668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8126,"num_write_kb":505408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":7652},{"shards":"4,7","objects":-72}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":252,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.721991+0000","last_change":"2024-08-17T17:21:31.870010+0000","last_active":"2024-08-17T17:24:02.721991+0000","last_peered":"2024-08-17T17:24:02.721991+0000","last_clean":"2024-08-17T17:24:02.721991+0000","last_became_active":"2024-08-17T17:21:31.869696+0000","last_became_peered":"2024-08-17T17:21:31.869696+0000","last_unstale":"2024-08-17T17:24:02.721991+0000","last_undegraded":"2024-08-17T17:24:02.721991+0000","last_fullsized":"2024-08-17T17:24:02.721991+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:57.550019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":361,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711431+0000","last_change":"2024-08-17T17:23:58.445133+0000","last_active":"2024-08-17T17:24:02.711431+0000","last_peered":"2024-08-17T17:24:02.711431+0000","last_clean":"2024-08-17T17:24:02.711431+0000","last_became_active":"2024-08-17T17:23:58.444763+0000","last_became_peered":"2024-08-17T17:23:58.444763+0000","last_unstale":"2024-08-17T17:24:02.711431+0000","last_undegraded":"2024-08-17T17:24:02.711431+0000","last_fullsized":"2024-08-17T17:24:02.711431+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:57:55.854706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":316,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711804+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:24:02.711804+0000","last_peered":"2024-08-17T17:24:02.711804+0000","last_clean":"2024-08-17T17:24:02.711804+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:24:02.711804+0000","last_undegraded":"2024-08-17T17:24:02.711804+0000","last_fullsized":"2024-08-17T17:24:02.711804+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"244'8118","reported_seq":8157,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:07.635574+0000","last_change":"2024-08-17T17:23:59.886752+0000","last_active":"2024-08-17T17:24:07.635574+0000","last_peered":"2024-08-17T17:24:07.635574+0000","last_clean":"2024-08-17T17:23:57.429915+0000","last_became_active":"2024-08-17T17:23:59.862382+0000","last_became_peered":"2024-08-17T17:23:59.862382+0000","last_unstale":"2024-08-17T17:24:07.635574+0000","last_undegraded":"2024-08-17T17:23:59.513905+0000","last_fullsized":"2024-08-17T17:23:59.429949+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:51.796863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":505348120,"num_objects":7712,"num_object_clones":0,"num_object_copies":23136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7892,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7712,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8118,"num_write_kb":506497,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":7689},{"shards":"4,6","objects":-180}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":326,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711350+0000","last_change":"2024-08-17T17:21:31.870419+0000","last_active":"2024-08-17T17:24:02.711350+0000","last_peered":"2024-08-17T17:24:02.711350+0000","last_clean":"2024-08-17T17:24:02.711350+0000","last_became_active":"2024-08-17T17:21:31.870030+0000","last_became_peered":"2024-08-17T17:21:31.870030+0000","last_unstale":"2024-08-17T17:24:02.711350+0000","last_undegraded":"2024-08-17T17:24:02.711350+0000","last_fullsized":"2024-08-17T17:24:02.711350+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:27.814200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":221,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.454139+0000","last_change":"2024-08-17T17:23:58.436497+0000","last_active":"2024-08-17T17:24:01.454139+0000","last_peered":"2024-08-17T17:24:01.454139+0000","last_clean":"2024-08-17T17:24:01.454139+0000","last_became_active":"2024-08-17T17:23:58.436191+0000","last_became_peered":"2024-08-17T17:23:58.436191+0000","last_unstale":"2024-08-17T17:24:01.454139+0000","last_undegraded":"2024-08-17T17:24:01.454139+0000","last_fullsized":"2024-08-17T17:24:01.454139+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:04.034339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":280,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453298+0000","last_change":"2024-08-17T17:23:58.442240+0000","last_active":"2024-08-17T17:24:01.453298+0000","last_peered":"2024-08-17T17:24:01.453298+0000","last_clean":"2024-08-17T17:24:01.453298+0000","last_became_active":"2024-08-17T17:23:58.441933+0000","last_became_peered":"2024-08-17T17:23:58.441933+0000","last_unstale":"2024-08-17T17:24:01.453298+0000","last_undegraded":"2024-08-17T17:24:01.453298+0000","last_fullsized":"2024-08-17T17:24:01.453298+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:14.838482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"244'8043","reported_seq":8073,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:07.321872+0000","last_change":"2024-08-17T17:22:18.541323+0000","last_active":"2024-08-17T17:24:07.321872+0000","last_peered":"2024-08-17T17:24:07.321872+0000","last_clean":"2024-08-17T17:24:07.321872+0000","last_became_active":"2024-08-17T17:22:18.541054+0000","last_became_peered":"2024-08-17T17:22:18.541054+0000","last_unstale":"2024-08-17T17:24:07.321872+0000","last_undegraded":"2024-08-17T17:24:07.321872+0000","last_fullsized":"2024-08-17T17:24:07.321872+0000","mapping_epoch":222,"log_start":"227'876","ondisk_log_start":"227'876","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7167,"log_dups_size":0,"ondisk_log_size":7167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:46.052040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":499449856,"num_objects":7621,"num_object_clones":0,"num_object_copies":22863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8043,"num_write_kb":501248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":314,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.719093+0000","last_change":"2024-08-17T17:23:58.448497+0000","last_active":"2024-08-17T17:24:02.719093+0000","last_peered":"2024-08-17T17:24:02.719093+0000","last_clean":"2024-08-17T17:24:02.719093+0000","last_became_active":"2024-08-17T17:23:58.437697+0000","last_became_peered":"2024-08-17T17:23:58.437697+0000","last_unstale":"2024-08-17T17:24:02.719093+0000","last_undegraded":"2024-08-17T17:24:02.719093+0000","last_fullsized":"2024-08-17T17:24:02.719093+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:17:58.366912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":315,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711459+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:24:02.711459+0000","last_peered":"2024-08-17T17:24:02.711459+0000","last_clean":"2024-08-17T17:24:02.711459+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:24:02.711459+0000","last_undegraded":"2024-08-17T17:24:02.711459+0000","last_fullsized":"2024-08-17T17:24:02.711459+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":307,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.717672+0000","last_change":"2024-08-17T17:23:58.443464+0000","last_active":"2024-08-17T17:24:02.717672+0000","last_peered":"2024-08-17T17:24:02.717672+0000","last_clean":"2024-08-17T17:24:02.717672+0000","last_became_active":"2024-08-17T17:23:58.443153+0000","last_became_peered":"2024-08-17T17:23:58.443153+0000","last_unstale":"2024-08-17T17:24:02.717672+0000","last_undegraded":"2024-08-17T17:24:02.717672+0000","last_fullsized":"2024-08-17T17:24:02.717672+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:21.368034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"244'8131","reported_seq":8161,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:07.527966+0000","last_change":"2024-08-17T17:22:18.545136+0000","last_active":"2024-08-17T17:24:07.527966+0000","last_peered":"2024-08-17T17:24:07.527966+0000","last_clean":"2024-08-17T17:24:07.527966+0000","last_became_active":"2024-08-17T17:22:18.544176+0000","last_became_peered":"2024-08-17T17:22:18.544176+0000","last_unstale":"2024-08-17T17:24:07.527966+0000","last_undegraded":"2024-08-17T17:24:07.527966+0000","last_fullsized":"2024-08-17T17:24:07.527966+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8131,"log_dups_size":0,"ondisk_log_size":8131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:08.787886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":505217024,"num_objects":7709,"num_object_clones":0,"num_object_copies":23127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8131,"num_write_kb":506880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":46,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711196+0000","last_change":"2024-08-17T17:23:58.450138+0000","last_active":"2024-08-17T17:24:02.711196+0000","last_peered":"2024-08-17T17:24:02.711196+0000","last_clean":"2024-08-17T17:24:02.711196+0000","last_became_active":"2024-08-17T17:23:58.443925+0000","last_became_peered":"2024-08-17T17:23:58.443925+0000","last_unstale":"2024-08-17T17:24:02.711196+0000","last_undegraded":"2024-08-17T17:24:02.711196+0000","last_fullsized":"2024-08-17T17:24:02.711196+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:13.125136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":211,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.456153+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:24:01.456153+0000","last_peered":"2024-08-17T17:24:01.456153+0000","last_clean":"2024-08-17T17:24:01.456153+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:24:01.456153+0000","last_undegraded":"2024-08-17T17:24:01.456153+0000","last_fullsized":"2024-08-17T17:24:01.456153+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.589912+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:24:01.589912+0000","last_peered":"2024-08-17T17:24:01.589912+0000","last_clean":"2024-08-17T17:24:01.589912+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:24:01.589912+0000","last_undegraded":"2024-08-17T17:24:01.589912+0000","last_fullsized":"2024-08-17T17:24:01.589912+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590185+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:24:01.590185+0000","last_peered":"2024-08-17T17:24:01.590185+0000","last_clean":"2024-08-17T17:24:01.590185+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:24:01.590185+0000","last_undegraded":"2024-08-17T17:24:01.590185+0000","last_fullsized":"2024-08-17T17:24:01.590185+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590356+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:24:01.590356+0000","last_peered":"2024-08-17T17:24:01.590356+0000","last_clean":"2024-08-17T17:24:01.590356+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:24:01.590356+0000","last_undegraded":"2024-08-17T17:24:01.590356+0000","last_fullsized":"2024-08-17T17:24:01.590356+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":315,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711519+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:24:02.711519+0000","last_peered":"2024-08-17T17:24:02.711519+0000","last_clean":"2024-08-17T17:24:02.711519+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:24:02.711519+0000","last_undegraded":"2024-08-17T17:24:02.711519+0000","last_fullsized":"2024-08-17T17:24:02.711519+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":202,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453892+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:24:01.453892+0000","last_peered":"2024-08-17T17:24:01.453892+0000","last_clean":"2024-08-17T17:24:01.453892+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:24:01.453892+0000","last_undegraded":"2024-08-17T17:24:01.453892+0000","last_fullsized":"2024-08-17T17:24:01.453892+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":142,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453849+0000","last_change":"2024-08-17T17:21:31.909108+0000","last_active":"2024-08-17T17:24:01.453849+0000","last_peered":"2024-08-17T17:24:01.453849+0000","last_clean":"2024-08-17T17:24:01.453849+0000","last_became_active":"2024-08-17T17:21:31.908882+0000","last_became_peered":"2024-08-17T17:21:31.908882+0000","last_unstale":"2024-08-17T17:24:01.453849+0000","last_undegraded":"2024-08-17T17:24:01.453849+0000","last_fullsized":"2024-08-17T17:24:01.453849+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:24.197352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":262,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.717910+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:24:02.717910+0000","last_peered":"2024-08-17T17:24:02.717910+0000","last_clean":"2024-08-17T17:24:02.717910+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:24:02.717910+0000","last_undegraded":"2024-08-17T17:24:02.717910+0000","last_fullsized":"2024-08-17T17:24:02.717910+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":294,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.719180+0000","last_change":"2024-08-17T17:23:58.438038+0000","last_active":"2024-08-17T17:24:02.719180+0000","last_peered":"2024-08-17T17:24:02.719180+0000","last_clean":"2024-08-17T17:24:02.719180+0000","last_became_active":"2024-08-17T17:23:58.436695+0000","last_became_peered":"2024-08-17T17:23:58.436695+0000","last_unstale":"2024-08-17T17:24:02.719180+0000","last_undegraded":"2024-08-17T17:24:02.719180+0000","last_fullsized":"2024-08-17T17:24:02.719180+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:36.340028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":274,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.717588+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:24:02.717588+0000","last_peered":"2024-08-17T17:24:02.717588+0000","last_clean":"2024-08-17T17:24:02.717588+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:24:02.717588+0000","last_undegraded":"2024-08-17T17:24:02.717588+0000","last_fullsized":"2024-08-17T17:24:02.717588+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":215,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.456094+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:24:01.456094+0000","last_peered":"2024-08-17T17:24:01.456094+0000","last_clean":"2024-08-17T17:24:01.456094+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:24:01.456094+0000","last_undegraded":"2024-08-17T17:24:01.456094+0000","last_fullsized":"2024-08-17T17:24:01.456094+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.717739+0000","last_change":"2024-08-17T17:23:58.443575+0000","last_active":"2024-08-17T17:24:02.717739+0000","last_peered":"2024-08-17T17:24:02.717739+0000","last_clean":"2024-08-17T17:24:02.717739+0000","last_became_active":"2024-08-17T17:23:58.443274+0000","last_became_peered":"2024-08-17T17:23:58.443274+0000","last_unstale":"2024-08-17T17:24:02.717739+0000","last_undegraded":"2024-08-17T17:24:02.717739+0000","last_fullsized":"2024-08-17T17:24:02.717739+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:39:47.789767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":170,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.453921+0000","last_change":"2024-08-17T17:23:58.438143+0000","last_active":"2024-08-17T17:24:01.453921+0000","last_peered":"2024-08-17T17:24:01.453921+0000","last_clean":"2024-08-17T17:24:01.453921+0000","last_became_active":"2024-08-17T17:23:58.437820+0000","last_became_peered":"2024-08-17T17:23:58.437820+0000","last_unstale":"2024-08-17T17:24:01.453921+0000","last_undegraded":"2024-08-17T17:24:01.453921+0000","last_fullsized":"2024-08-17T17:24:01.453921+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.712862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":314,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.711070+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:24:02.711070+0000","last_peered":"2024-08-17T17:24:02.711070+0000","last_clean":"2024-08-17T17:24:02.711070+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:24:02.711070+0000","last_undegraded":"2024-08-17T17:24:02.711070+0000","last_fullsized":"2024-08-17T17:24:02.711070+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":274,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.721090+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:24:02.721090+0000","last_peered":"2024-08-17T17:24:02.721090+0000","last_clean":"2024-08-17T17:24:02.721090+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:24:02.721090+0000","last_undegraded":"2024-08-17T17:24:02.721090+0000","last_fullsized":"2024-08-17T17:24:02.721090+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":217,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590116+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:24:01.590116+0000","last_peered":"2024-08-17T17:24:01.590116+0000","last_clean":"2024-08-17T17:24:01.590116+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:24:01.590116+0000","last_undegraded":"2024-08-17T17:24:01.590116+0000","last_fullsized":"2024-08-17T17:24:01.590116+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":338,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.713564+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:24:02.713564+0000","last_peered":"2024-08-17T17:24:02.713564+0000","last_clean":"2024-08-17T17:24:02.713564+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:24:02.713564+0000","last_undegraded":"2024-08-17T17:24:02.713564+0000","last_fullsized":"2024-08-17T17:24:02.713564+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":27,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590230+0000","last_change":"2024-08-17T17:23:58.443509+0000","last_active":"2024-08-17T17:24:01.590230+0000","last_peered":"2024-08-17T17:24:01.590230+0000","last_clean":"2024-08-17T17:24:01.590230+0000","last_became_active":"2024-08-17T17:23:58.443084+0000","last_became_peered":"2024-08-17T17:23:58.443084+0000","last_unstale":"2024-08-17T17:24:01.590230+0000","last_undegraded":"2024-08-17T17:24:01.590230+0000","last_fullsized":"2024-08-17T17:24:01.590230+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:19.073156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590153+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:24:01.590153+0000","last_peered":"2024-08-17T17:24:01.590153+0000","last_clean":"2024-08-17T17:24:01.590153+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:24:01.590153+0000","last_undegraded":"2024-08-17T17:24:01.590153+0000","last_fullsized":"2024-08-17T17:24:01.590153+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":98,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590276+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:24:01.590276+0000","last_peered":"2024-08-17T17:24:01.590276+0000","last_clean":"2024-08-17T17:24:01.590276+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:24:01.590276+0000","last_undegraded":"2024-08-17T17:24:01.590276+0000","last_fullsized":"2024-08-17T17:24:01.590276+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.589993+0000","last_change":"2024-08-17T17:23:58.437455+0000","last_active":"2024-08-17T17:24:01.589993+0000","last_peered":"2024-08-17T17:24:01.589993+0000","last_clean":"2024-08-17T17:24:01.589993+0000","last_became_active":"2024-08-17T17:23:58.437150+0000","last_became_peered":"2024-08-17T17:23:58.437150+0000","last_unstale":"2024-08-17T17:24:01.589993+0000","last_undegraded":"2024-08-17T17:24:01.589993+0000","last_fullsized":"2024-08-17T17:24:01.589993+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:36.013242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":98,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.590147+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:24:01.590147+0000","last_peered":"2024-08-17T17:24:01.590147+0000","last_clean":"2024-08-17T17:24:01.590147+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:24:01.590147+0000","last_undegraded":"2024-08-17T17:24:01.590147+0000","last_fullsized":"2024-08-17T17:24:01.590147+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":312,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.718147+0000","last_change":"2024-08-17T17:23:58.441390+0000","last_active":"2024-08-17T17:24:02.718147+0000","last_peered":"2024-08-17T17:24:02.718147+0000","last_clean":"2024-08-17T17:24:02.718147+0000","last_became_active":"2024-08-17T17:23:58.441009+0000","last_became_peered":"2024-08-17T17:23:58.441009+0000","last_unstale":"2024-08-17T17:24:02.718147+0000","last_undegraded":"2024-08-17T17:24:02.718147+0000","last_fullsized":"2024-08-17T17:24:02.718147+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:43.174228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":291,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.718756+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:24:02.718756+0000","last_peered":"2024-08-17T17:24:02.718756+0000","last_clean":"2024-08-17T17:24:02.718756+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:24:02.718756+0000","last_undegraded":"2024-08-17T17:24:02.718756+0000","last_fullsized":"2024-08-17T17:24:02.718756+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-17T17:24:01.589660+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:24:01.589660+0000","last_peered":"2024-08-17T17:24:01.589660+0000","last_clean":"2024-08-17T17:24:01.589660+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:24:01.589660+0000","last_undegraded":"2024-08-17T17:24:01.589660+0000","last_fullsized":"2024-08-17T17:24:01.589660+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":38,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-17T17:24:02.712452+0000","last_change":"2024-08-17T17:23:58.446348+0000","last_active":"2024-08-17T17:24:02.712452+0000","last_peered":"2024-08-17T17:24:02.712452+0000","last_clean":"2024-08-17T17:24:02.712452+0000","last_became_active":"2024-08-17T17:23:58.437255+0000","last_became_peered":"2024-08-17T17:23:58.437255+0000","last_unstale":"2024-08-17T17:24:02.712452+0000","last_undegraded":"2024-08-17T17:24:02.712452+0000","last_fullsized":"2024-08-17T17:24:02.712452+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:56.171321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":8011055128,"num_objects":122240,"num_object_clones":0,"num_object_copies":366720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47347,"num_objects_misplaced":30580,"num_objects_unfound":0,"num_objects_dirty":122240,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129252,"num_write_kb":8047681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":402,"num_bytes_recovered":26345472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24180961280,"data_stored":24180949064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123575,"ondisk_log_size":123575,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":592,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201626,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2861872,"kb_used_data":1964712,"kb_used_omap":1,"kb_used_meta":897150,"kb_avail":90883280,"statfs":{"total":95995035648,"available":93064478720,"internally_reserved":0,"allocated":2011865088,"data_stored":2010651963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918681944},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1,2,2,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463262,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4333936,"kb_used_data":3436072,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":89411216,"statfs":{"total":95995035648,"available":91557085184,"internally_reserved":0,"allocated":3518537728,"data_stored":3517325054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":199,"seq":854698491936,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2855700,"kb_used_data":2468684,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":90889452,"statfs":{"total":95995035648,"available":93070798848,"internally_reserved":0,"allocated":2527932416,"data_stored":2526742052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986536,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4828072,"kb_used_data":3939616,"kb_used_omap":1,"kb_used_meta":888446,"kb_avail":88917080,"statfs":{"total":95995035648,"available":91051089920,"internally_reserved":0,"allocated":4034166784,"data_stored":4032960988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909769048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225356,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2971924,"kb_used_data":2448332,"kb_used_omap":4,"kb_used_meta":523579,"kb_avail":90773228,"statfs":{"total":95995035648,"available":92951785472,"internally_reserved":0,"allocated":2507091968,"data_stored":2505901856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":536145344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075681,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3311256,"kb_used_data":2927696,"kb_used_omap":6,"kb_used_meta":383545,"kb_avail":90433896,"statfs":{"total":95995035648,"available":92604309504,"internally_reserved":0,"allocated":2997960704,"data_stored":2996767267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":392750523},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,2,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533573,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2143960,"kb_used_data":1478096,"kb_used_omap":5,"kb_used_meta":665850,"kb_avail":91601192,"statfs":{"total":95995035648,"available":93799620608,"internally_reserved":0,"allocated":1513570304,"data_stored":1512375428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":681830640},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":382252089550,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":5135204352,"data_stored":5135204352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1507852288,"data_stored":1507852288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2989228032,"data_stored":2989228032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2502692864,"data_stored":2502688792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4018081792,"data_stored":4018077720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2521956352,"data_stored":2521956352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3500347392,"data_stored":3500343320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2005598208,"data_stored":2005598208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:24:16.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pgp_num 16 2024-08-17T17:24:16.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:16.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:16.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:16.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:16.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:16.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:16.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:16.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:24:16.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:16.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:16.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:16.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:16.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:16.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:16.950 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:16.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:17.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:17.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:16 smithi152 ceph-mon[93456]: pgmap v794: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 427 KiB/s wr, 507 op/s; 47521/358197 objects degraded (13.267%); 29853/358197 objects misplaced (8.334%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:24:17.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:16 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:17.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:16 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:17.076 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:16 smithi152 ceph-mon[93456]: osdmap e245: 8 total, 8 up, 7 in 2024-08-17T17:24:17.084 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[103614]: pgmap v794: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 427 KiB/s wr, 507 op/s; 47521/358197 objects degraded (13.267%); 29853/358197 objects misplaced (8.334%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:24:17.084 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:17.084 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:17.084 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[103614]: osdmap e245: 8 total, 8 up, 7 in 2024-08-17T17:24:17.084 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[106729]: pgmap v794: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 427 KiB/s wr, 507 op/s; 47521/358197 objects degraded (13.267%); 29853/358197 objects misplaced (8.334%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:24:17.085 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:17.085 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:17.085 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:16 smithi142 ceph-mon[106729]: osdmap e245: 8 total, 8 up, 7 in 2024-08-17T17:24:17.093 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:17.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:17.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:17.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:17.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:17.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:17.503 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:17.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:17.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:17.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:17.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:17.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:17.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:24:17.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:17.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:17.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:17.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:17.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:18.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:18.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:18.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:18.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:18.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:18.360 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:18.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:18.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:18.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:18.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:18.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:18.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:18.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.775 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:18.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:18.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:18.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:24:18.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.912 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:18.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:18.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:18 smithi152 ceph-mon[93456]: pgmap v796: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 47521/355611 objects degraded (13.363%); 29503/355611 objects misplaced (8.296%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:24:19.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:19.047 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:18 smithi142 ceph-mon[106729]: pgmap v796: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 47521/355611 objects degraded (13.363%); 29503/355611 objects misplaced (8.296%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:24:19.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:18 smithi142 ceph-mon[103614]: pgmap v796: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 47521/355611 objects degraded (13.363%); 29503/355611 objects misplaced (8.296%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:24:19.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:19.126 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:19.126 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:19.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:19.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:19.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:19.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:19.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:19.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:19.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:19.605 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:19.707 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:19.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.746 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:19.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:19.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:19.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:19.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:19.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:20.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.018 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:20.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:20.070 INFO:teuthology.orchestra.run.smithi142.stderr:set pool 10 pgp_num to 16 2024-08-17T17:24:20.071 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2482451663' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:24:20.071 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:19 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:24:20.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2482451663' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:24:20.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:19 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:24:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2482451663' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:24:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:19 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:24:20.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:20.152 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:20.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:20.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:20.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:20.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:20.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:20.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:20.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:24:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:24:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:24:20.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:20.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:20.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:20.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:20.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:20.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:20.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:24:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:20.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:21.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:20 smithi152 ceph-mon[93456]: pgmap v797: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 47667/350070 objects degraded (13.616%); 29175/350070 objects misplaced (8.334%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47667/350070 objects degraded (13.616%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:20 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:24:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:20 smithi152 ceph-mon[93456]: osdmap e246: 8 total, 8 up, 7 in 2024-08-17T17:24:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:21.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[106729]: pgmap v797: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 47667/350070 objects degraded (13.616%); 29175/350070 objects misplaced (8.334%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47667/350070 objects degraded (13.616%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:24:21.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[106729]: osdmap e246: 8 total, 8 up, 7 in 2024-08-17T17:24:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[103614]: pgmap v797: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 47667/350070 objects degraded (13.616%); 29175/350070 objects misplaced (8.334%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47667/350070 objects degraded (13.616%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:24:21.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:20 smithi142 ceph-mon[103614]: osdmap e246: 8 total, 8 up, 7 in 2024-08-17T17:24:21.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:21.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:21.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:21.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:21.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:21.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:21.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:21.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:21.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:21.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.649 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:21.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:21.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:21.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:21.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:21.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:22.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:22.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:22.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:24:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:24:21] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-08-17T17:24:22.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:22.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:22.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:22.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:22.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:22.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:22.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:22.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:22.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:22.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:22.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:22.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:22.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:22.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:23.015 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:23.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:22 smithi152 ceph-mon[93456]: pgmap v799: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 47667/349587 objects degraded (13.635%); 29175/349587 objects misplaced (8.346%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:22 smithi142 ceph-mon[103614]: pgmap v799: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 47667/349587 objects degraded (13.635%); 29175/349587 objects misplaced (8.346%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:22 smithi142 ceph-mon[106729]: pgmap v799: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 47667/349587 objects degraded (13.635%); 29175/349587 objects misplaced (8.346%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:23.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:23.177 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:23.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:23.279 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:23.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:23.369 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:23.398 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:23.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:23.451 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:23.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:23.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:23.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:23.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:23.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:23.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:23.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:23.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:23.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:23.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:24.035 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:24:24.036 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:24.067 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:24:24.067 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:24.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:24.093 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:24:24.094 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:24.123 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:24:24.123 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:24.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:24.150 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:24:24.150 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:24.178 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:24:24.178 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:24.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:24:24.208 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:24.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:24.239 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:24:24.239 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:24.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:24.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:24.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:24.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:24.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:24.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:24.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:24.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:24.791 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:24.837 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:24.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:24.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:24.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:25.069 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:24 smithi142 ceph-mon[103614]: pgmap v800: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 47791/344010 objects degraded (13.892%); 28651/344010 objects misplaced (8.329%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:24:25.069 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1925972116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:25.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:24 smithi142 ceph-mon[106729]: pgmap v800: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 47791/344010 objects degraded (13.892%); 28651/344010 objects misplaced (8.329%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:24:25.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1925972116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:25.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:25.110 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:25.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:25.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:24 smithi152 ceph-mon[93456]: pgmap v800: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 47791/344010 objects degraded (13.892%); 28651/344010 objects misplaced (8.329%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:24:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1925972116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:25.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:25.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:25.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:25.670 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:25.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:25.816 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:25.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:25.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:26.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:26.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:26.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:26.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47791/344010 objects degraded (13.892%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:26.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47791/344010 objects degraded (13.892%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:26.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:26.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:26.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:26.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47791/344010 objects degraded (13.892%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:26.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:26.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:26.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:26.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:26.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.650 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:26.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:26.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:26.791 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:26.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:26.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:26.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:27.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:27.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:26 smithi142 ceph-mon[106729]: pgmap v801: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 47877/341931 objects degraded (14.002%); 28461/341931 objects misplaced (8.324%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:26 smithi142 ceph-mon[103614]: pgmap v801: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 47877/341931 objects degraded (14.002%); 28461/341931 objects misplaced (8.324%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:27.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:27.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:27.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:27.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:27.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:26 smithi152 ceph-mon[93456]: pgmap v801: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 47877/341931 objects degraded (14.002%); 28461/341931 objects misplaced (8.324%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:27.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:27.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:27.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:27.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:27.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:27.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:27.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:27.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:27.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:27.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:27.758 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:24:27.758 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:24:27.759 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-08-17T17:24:27.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 0 2024-08-17T17:24:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:27.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:27.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:27.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.913 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:27.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:28.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:28.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:28.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:28.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:28.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:28.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:28.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:28.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:28.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:28.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:28.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:28.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:28.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:28.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:28.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:29.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:29.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:28 smithi142 ceph-mon[106729]: pgmap v802: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 47877/339333 objects degraded (14.109%); 28129/339333 objects misplaced (8.289%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:29.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:28 smithi142 ceph-mon[103614]: pgmap v802: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 47877/339333 objects degraded (14.109%); 28129/339333 objects misplaced (8.289%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:29.126 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:29.127 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:28 smithi152 ceph-mon[93456]: pgmap v802: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 47877/339333 objects degraded (14.109%); 28129/339333 objects misplaced (8.289%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:29.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:29.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:29.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:29.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:29.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:29.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:29.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:29.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:29.642 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:29.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:29.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:29.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:29.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:29.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/198469556' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:29.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:29.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:30.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:30.110 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/198469556' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:30.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/198469556' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:30.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:30.127 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:24:30.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:30.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:30.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:30.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:30.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:30.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:30.547 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:24:30.547 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:24:30.548 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:24:30.548 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:24:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:24:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:24:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:24:30.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:30.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:30.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:30.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:30.920 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:30.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:30.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:31.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:31.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:31.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[106729]: pgmap v803: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 48071/333150 objects degraded (14.429%); 27759/333150 objects misplaced (8.332%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:31.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[106729]: osdmap e247: 8 total, 8 up, 7 in 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48071/333150 objects degraded (14.429%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[103614]: pgmap v803: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 48071/333150 objects degraded (14.429%); 27759/333150 objects misplaced (8.332%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[103614]: osdmap e247: 8 total, 8 up, 7 in 2024-08-17T17:24:31.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48071/333150 objects degraded (14.429%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:31.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:31.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:30 smithi152 ceph-mon[93456]: pgmap v803: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 48071/333150 objects degraded (14.429%); 27759/333150 objects misplaced (8.332%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:31.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:30 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:31.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:31.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:30 smithi152 ceph-mon[93456]: osdmap e247: 8 total, 8 up, 7 in 2024-08-17T17:24:31.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48071/333150 objects degraded (14.429%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.234 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:31.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:31.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:24:31.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:31.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:31.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:31.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:31.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:31.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:31.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:31.812 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:31.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:31.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:32.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:32.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:32.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:24:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:24:31] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.51.0" 2024-08-17T17:24:32.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:32.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:32.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:32.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:32.380 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:32.405 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:32.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:32.459 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:32.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:32.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:32.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:32.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:32.675 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:32.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:32.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:32.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:32.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:32.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:32.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:33.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:33.092 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:32 smithi142 ceph-mon[103614]: pgmap v805: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 48071/332622 objects degraded (14.452%); 27759/332622 objects misplaced (8.346%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:33.093 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:32 smithi142 ceph-mon[106729]: pgmap v805: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 48071/332622 objects degraded (14.452%); 27759/332622 objects misplaced (8.346%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:33.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:33.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:33.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:33.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:32 smithi152 ceph-mon[93456]: pgmap v805: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 48071/332622 objects degraded (14.452%); 27759/332622 objects misplaced (8.346%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:24:33.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:33.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:33.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:33.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:33.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:33.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:33.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:33.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:33.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:33.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:33.942 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:34.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:34.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:34.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:34.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:34.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:24:34.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:34.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:34.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:34.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:34.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:34.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:34.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:34.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:34.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:34 smithi142 ceph-mon[106729]: pgmap v806: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 48132/327174 objects degraded (14.711%); 27265/327174 objects misplaced (8.333%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:34 smithi142 ceph-mon[103614]: pgmap v806: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 48132/327174 objects degraded (14.711%); 27265/327174 objects misplaced (8.333%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:34.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:34.950 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:35.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:35.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:35.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:35.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:35.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:35.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:35.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:34 smithi152 ceph-mon[93456]: pgmap v806: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 48132/327174 objects degraded (14.711%); 27265/327174 objects misplaced (8.333%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:35.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:35.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:35.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:35.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:35.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:35.653 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:35.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:35.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:35.883 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:35.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:35.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:36.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:36.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:36.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:36.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48132/327174 objects degraded (14.711%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/829659891' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:24:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:24:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:24:35.785+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e247 definitely_dead 0 2024-08-17T17:24:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48132/327174 objects degraded (14.711%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/829659891' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:24:36.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:35 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:24:36.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:36.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:36.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:36.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:36.292 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48132/327174 objects degraded (14.711%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/829659891' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:24:36.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:35 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:24:36.393 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:36.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:36.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:36.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:36.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:36.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:36.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:36.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:36.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:36.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:36.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:37.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:37.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:36 smithi142 ceph-mon[103614]: pgmap v807: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 48197/325089 objects degraded (14.826%); 27095/325089 objects misplaced (8.335%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:36 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:24:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:36 smithi142 ceph-mon[103614]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:24:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:36 smithi142 ceph-mon[106729]: pgmap v807: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 48197/325089 objects degraded (14.826%); 27095/325089 objects misplaced (8.335%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:36 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:24:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:36 smithi142 ceph-mon[106729]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:24:37.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:37.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:37.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:37.223 INFO:teuthology.orchestra.run.smithi142.stderr:marked in osd.0. 2024-08-17T17:24:37.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:36 smithi152 ceph-mon[93456]: pgmap v807: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 48197/325089 objects degraded (14.826%); 27095/325089 objects misplaced (8.335%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:24:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:36 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:24:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:36 smithi152 ceph-mon[93456]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:24:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:37.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:37.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:37.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:37.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:37.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:37.592 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:37.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:37.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:37.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:37.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:37.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:37.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:38.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:38.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:38.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:38.238 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:37 smithi142 ceph-mon[106729]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:24:38.238 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:37 smithi142 ceph-mon[106729]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:24:38.238 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:37 smithi142 ceph-mon[103614]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:24:38.238 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:37 smithi142 ceph-mon[103614]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:24:38.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:38.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:38.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:37 smithi152 ceph-mon[93456]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:24:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:37 smithi152 ceph-mon[93456]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:24:38.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:38.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:38.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:38.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:38.619 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-08-17T17:24:38.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:38.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:38.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:38.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:38.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:38.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:39.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:39.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:39.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:39.105 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:38 smithi142 ceph-mon[103614]: pgmap v810: 89 pgs: 1 remapped+peering, 3 peering, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 33725/323007 objects degraded (10.441%); 13403/323007 objects misplaced (4.149%); 1.5 MiB/s, 25 objects/s recovering 2024-08-17T17:24:39.106 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:38 smithi142 ceph-mon[106729]: pgmap v810: 89 pgs: 1 remapped+peering, 3 peering, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 33725/323007 objects degraded (10.441%); 13403/323007 objects misplaced (4.149%); 1.5 MiB/s, 25 objects/s recovering 2024-08-17T17:24:39.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:39.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:39.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:39.293 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:24:39.293 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:39.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:38 smithi152 ceph-mon[93456]: pgmap v810: 89 pgs: 1 remapped+peering, 3 peering, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 33725/323007 objects degraded (10.441%); 13403/323007 objects misplaced (4.149%); 1.5 MiB/s, 25 objects/s recovering 2024-08-17T17:24:39.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:39.336 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:24:39.336 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:39.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:39.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:39.375 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:24:39.375 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:39.409 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:24:39.409 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:39.444 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:24:39.445 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:39.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:39.480 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:24:39.480 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.485 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:39.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:39.513 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:24:39.513 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:39.545 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:24:39.545 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:39.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:39.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:39.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:39.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.749 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:39.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:39.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:39.856 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:39.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:39.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:40.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.021 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:40.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:40.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:40.159 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3598875428' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:40.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:40.296 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:40.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:40.323 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3598875428' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:40.323 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3598875428' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:40.398 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:40.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:40.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:40.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:40.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:24:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:24:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:24:40.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:40.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:40.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:40.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:40.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:40.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:41.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:41.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:41.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:41.248 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:40 smithi142 ceph-mon[103614]: pgmap v812: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 peering, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 7239/318495 objects degraded (2.273%); 2.1 MiB/s, 41 objects/s recovering 2024-08-17T17:24:41.248 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7239/318495 objects degraded (2.273%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:41.248 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:40 smithi142 ceph-mon[106729]: pgmap v812: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 peering, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 7239/318495 objects degraded (2.273%); 2.1 MiB/s, 41 objects/s recovering 2024-08-17T17:24:41.248 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7239/318495 objects degraded (2.273%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:41.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:40 smithi152 ceph-mon[93456]: pgmap v812: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 peering, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 7239/318495 objects degraded (2.273%); 2.1 MiB/s, 41 objects/s recovering 2024-08-17T17:24:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7239/318495 objects degraded (2.273%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:41.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:41.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:41.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:41.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:41.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:41.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:41.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:41.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:41.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:41.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:24:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:24:41] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-08-17T17:24:41.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:41.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:41.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:42.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:42.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:42.171 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:42.172 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:42.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:42.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:42.347 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:42.347 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:42.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:42.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:42.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:24:42.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:42.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:42.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:42.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:42.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:42.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:42.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:42.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:42.912 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.994 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:42 smithi142 ceph-mon[103614]: pgmap v813: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 8443/317910 objects degraded (2.656%); 992 KiB/s, 25 objects/s recovering 2024-08-17T17:24:42.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:42 smithi142 ceph-mon[106729]: pgmap v813: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 8443/317910 objects degraded (2.656%); 992 KiB/s, 25 objects/s recovering 2024-08-17T17:24:43.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:43.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:24:43.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:43.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:43.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:43.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:43.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:43.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:43.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:43.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:42 smithi152 ceph-mon[93456]: pgmap v813: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 8443/317910 objects degraded (2.656%); 992 KiB/s, 25 objects/s recovering 2024-08-17T17:24:43.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:43.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:43.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:43.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:43.619 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:24:43.620 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:24:43.620 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-17T17:24:43.620 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:24:43.620 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:24:43.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-17T17:24:43.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:43.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:43.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:43.802 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:43.816 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:43.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:43.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:43.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:43.981 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:44.043 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:44.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:44.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:44.230 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:44.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:44.368 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:44.470 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:44.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:44.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:44.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:44.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:44.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:44.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.781 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:44.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:44.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:44.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:44.936 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:24:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:45.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:45.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:44 smithi152 ceph-mon[93456]: pgmap v814: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 8443/309384 objects degraded (2.729%); 776 KiB/s, 47 objects/s recovering 2024-08-17T17:24:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3420167763' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:44 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:45.073 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:44 smithi142 ceph-mon[106729]: pgmap v814: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 8443/309384 objects degraded (2.729%); 776 KiB/s, 47 objects/s recovering 2024-08-17T17:24:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3420167763' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:45.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:44 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:44 smithi142 ceph-mon[103614]: pgmap v814: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 8443/309384 objects degraded (2.729%); 776 KiB/s, 47 objects/s recovering 2024-08-17T17:24:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3420167763' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:45.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:44 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:45.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:45.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:45.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:45.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:45.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:45.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:45.480 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:45.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:45.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:45.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:45.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:46.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:46.086 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:24:46.087 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:24:46.087 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:24:46.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[106729]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:24:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8443/309384 objects degraded (2.729%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[103614]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:24:46.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8443/309384 objects degraded (2.729%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:46.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:46.262 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:46.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:46.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:45 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:45 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:45 smithi152 ceph-mon[93456]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:24:46.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8443/309384 objects degraded (2.729%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:46.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:46.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:46.542 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:46.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:46.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:46.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:46.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:46.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.874 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:46.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:46.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:47.019 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:46 smithi152 ceph-mon[93456]: pgmap v816: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 8443/306543 objects degraded (2.754%); 744 KiB/s, 45 objects/s recovering 2024-08-17T17:24:47.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:46 smithi142 ceph-mon[103614]: pgmap v816: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 8443/306543 objects degraded (2.754%); 744 KiB/s, 45 objects/s recovering 2024-08-17T17:24:47.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:46 smithi142 ceph-mon[106729]: pgmap v816: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 8443/306543 objects degraded (2.754%); 744 KiB/s, 45 objects/s recovering 2024-08-17T17:24:47.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:47.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:47.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:47.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:47.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:47.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:47.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:47.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:47.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:47.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:47.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:47.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:47.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:47.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:47.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:47.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:47.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:48.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:48.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:48.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:48.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:48.240 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:48.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:48.252 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:48.252 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:48.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:48.378 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:48.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:48.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:48.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:48.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:48.638 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:48.639 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:48.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:48.668 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:24:48.668 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:48.701 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:24:48.701 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:48.732 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:24:48.733 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:48.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:48.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:48.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:48.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:48.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:49.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:49.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:49.077 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:48 smithi142 ceph-mon[106729]: pgmap v817: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 8443/302961 objects degraded (2.787%); 483 KiB/s, 31 objects/s recovering 2024-08-17T17:24:49.078 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1385886248' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:49.078 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:48 smithi142 ceph-mon[103614]: pgmap v817: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 8443/302961 objects degraded (2.787%); 483 KiB/s, 31 objects/s recovering 2024-08-17T17:24:49.078 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1385886248' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:49.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:24:49.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:49.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:49.305 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:49.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:48 smithi152 ceph-mon[93456]: pgmap v817: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 8443/302961 objects degraded (2.787%); 483 KiB/s, 31 objects/s recovering 2024-08-17T17:24:49.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1385886248' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:49.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:49.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:49.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:49.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:24:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.510 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:49.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:49.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:49.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:49.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:49.783 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:49.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:49.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:50.005 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:50.024 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:50.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:50.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:50.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:24:50.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:24:50.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:50.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:24:50.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:50.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:50.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:50.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:50.503 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:50.529 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:24:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:24:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:24:50.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:50.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:50.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:50.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:50.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:50.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:50.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:50.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:51.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:51.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:51.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:50 smithi142 ceph-mon[103614]: pgmap v818: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 8443/294774 objects degraded (2.864%); 0 B/s, 40 objects/s recovering 2024-08-17T17:24:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8443/294774 objects degraded (2.864%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:51.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:50 smithi142 ceph-mon[106729]: pgmap v818: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 8443/294774 objects degraded (2.864%); 0 B/s, 40 objects/s recovering 2024-08-17T17:24:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8443/294774 objects degraded (2.864%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:51.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:51.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:50 smithi152 ceph-mon[93456]: pgmap v818: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 8443/294774 objects degraded (2.864%); 0 B/s, 40 objects/s recovering 2024-08-17T17:24:51.246 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8443/294774 objects degraded (2.864%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:51.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:51.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:51.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:51.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:51.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:51.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:51.669 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:51.771 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:51.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:51.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:51.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:51.835 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:24:51.835 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:24:51.835 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-08-17T17:24:51.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:24:51.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:51.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:24:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:24:51] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-08-17T17:24:52.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:52.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:52.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:52.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:52.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:52.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:52.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:52.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:52.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:52.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:52.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:52.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:52.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:52.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:53.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:53.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:53.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:53.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:52 smithi142 ceph-mon[103614]: pgmap v819: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 8443/294018 objects degraded (2.872%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:53.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:52 smithi142 ceph-mon[106729]: pgmap v819: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 8443/294018 objects degraded (2.872%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:53.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:53.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:53.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:53.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:52 smithi152 ceph-mon[93456]: pgmap v819: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 8443/294018 objects degraded (2.872%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:53.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:53.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:53.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:53.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:53.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:53.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:53.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:53.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:53.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:53.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:53.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:53.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:54.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:54.089 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:24:54.089 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:24:54.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:157872 10.2 10:42d32889:::benchmark_data_smithi142_192453_object32124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:54.072108+0000", 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.011284517000000001, 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011317516999999999, 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:24:54.090 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 157872 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072108+0000", 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072108+0000", 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:24:54.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072111+0000", 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.689e-06 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072119+0000", 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0949999999999996e-06 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:24:54.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072139+0000", 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9448000000000001e-05 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072148+0000", 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2369999999999998e-06 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072237+0000", 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8851999999999996e-05 2024-08-17T17:24:54.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072403+0000", 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000165835 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.072549+0000", 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000146539 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.094 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.076728+0000", 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041792189999999996 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.080612+0000", 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038836980000000001 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.095 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:157877 10.b e251/248 10:d36af817:::benchmark_data_smithi142_192453_object32129:head v 251'9850, mlcod=241'7841)", 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:54.081951+0000", 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.001442016, 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0015488769999999999, 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.081951+0000", 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:24:54.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.081951+0000", 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.081952+0000", 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8700000000000004e-07 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.081959+0000", 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4789999999999999e-06 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.081973+0000", 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4112e-05 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:24:54.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.081980+0000", 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9929999999999998e-06 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.082161+0000", 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000180378 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:54.082218+0000", 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7247000000000003e-05 2024-08-17T17:24:54.099 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:24:54.100 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:24:54.100 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:24:54.100 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:24:54.100 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:24:54.100 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 2 2024-08-17T17:24:54.100 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:24:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:54.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:54.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:54.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:54.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:54.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:54.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:54.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:54.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:54.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:54.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:54.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:54.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:54.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:54.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:54.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:55.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:55.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:55.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:55.203 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:55.242 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:54 smithi142 ceph-mon[106729]: pgmap v820: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 8443/286194 objects degraded (2.950%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:55.245 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:54 smithi142 ceph-mon[103614]: pgmap v820: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 8443/286194 objects degraded (2.950%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:55.305 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:55.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:54 smithi152 ceph-mon[93456]: pgmap v820: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 8443/286194 objects degraded (2.950%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.350 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:55.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:55.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:55.469 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:55.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:55.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:55.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:55.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:55.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:55.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:55.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:55.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:55.975 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:24:56.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:56.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:24:56.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:56.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2288671505' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8443/286194 objects degraded (2.950%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:56.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:56.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:56.263 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:56.279 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2288671505' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:56.279 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8443/286194 objects degraded (2.950%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:56.280 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2288671505' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:56.280 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8443/286194 objects degraded (2.950%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:24:56.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:56.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:56.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:56.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:56.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:56.609 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:56.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:56.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:56.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:56.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:56.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:57.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: pgmap v821: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8443/283497 objects degraded (2.978%); 0 B/s, 37 objects/s recovering 2024-08-17T17:24:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2288671505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: pgmap v821: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8443/283497 objects degraded (2.978%); 0 B/s, 37 objects/s recovering 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2288671505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:24:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:57.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:57.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:57.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:57.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:57.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:57.293 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:57.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:57.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: pgmap v821: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8443/283497 objects degraded (2.978%); 0 B/s, 37 objects/s recovering 2024-08-17T17:24:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2288671505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:24:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:24:57.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:57.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:57.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:57.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:57.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:57.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:57.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:57.799 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:57.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:57.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:57.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:58.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:58.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:58.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:58.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:58.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:58.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:58.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:58.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:58.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:58.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:58.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:58.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:58.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:58.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:58.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:58.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:58.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:59.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:59.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:24:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:59.085 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:59.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:59.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:59.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:59.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:24:58 smithi152 ceph-mon[93456]: pgmap v823: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8443/279846 objects degraded (3.017%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:59.350 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:59.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:24:58 smithi142 ceph-mon[106729]: pgmap v823: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8443/279846 objects degraded (3.017%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:59.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:24:58 smithi142 ceph-mon[103614]: pgmap v823: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8443/279846 objects degraded (3.017%); 0 B/s, 39 objects/s recovering 2024-08-17T17:24:59.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:59.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:59.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:59.617 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:59.653 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:59.755 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:59.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:59.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:24:59.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:59.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:24:59.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:00.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:00.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:00.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:00.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:00.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:00.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:00.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:00.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:00.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:00.505 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:00.558 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:25:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:25:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:00.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:00.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:00.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:00.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:00.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:00.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:00.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:01.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:01.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:01.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:01.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:01.297 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:00 smithi152 ceph-mon[93456]: pgmap v824: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 8443/271803 objects degraded (3.106%); 0 B/s, 47 objects/s recovering 2024-08-17T17:25:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8443/271803 objects degraded (3.106%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:01.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:01.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:01.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:00 smithi142 ceph-mon[106729]: pgmap v824: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 8443/271803 objects degraded (3.106%); 0 B/s, 47 objects/s recovering 2024-08-17T17:25:01.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8443/271803 objects degraded (3.106%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:00 smithi142 ceph-mon[103614]: pgmap v824: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 8443/271803 objects degraded (3.106%); 0 B/s, 47 objects/s recovering 2024-08-17T17:25:01.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8443/271803 objects degraded (3.106%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:01.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:01.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:01.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:01.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:01.615 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:01.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:01.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:01.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:01.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.897 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:01.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:01.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:02.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:25:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:25:01] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-08-17T17:25:02.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:02.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:02.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:02.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:02.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:02.402 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.437 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:02.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:02.538 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:02.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:02.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:02.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:02.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:02.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:02.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:02.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:03.058 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:25:03.059 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:03.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:03.099 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:25:03.099 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:03.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:03.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:03.131 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:25:03.131 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:03.161 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:25:03.161 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:03.192 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:25:03.192 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:03.231 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:02 smithi142 ceph-mon[103614]: pgmap v825: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 8443/271008 objects degraded (3.115%); 0 B/s, 47 objects/s recovering 2024-08-17T17:25:03.231 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:25:03.231 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:03.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:03.234 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:02 smithi142 ceph-mon[106729]: pgmap v825: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 8443/271008 objects degraded (3.115%); 0 B/s, 47 objects/s recovering 2024-08-17T17:25:03.280 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:25:03.281 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:03.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:03.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:03.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:02 smithi152 ceph-mon[93456]: pgmap v825: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 8443/271008 objects degraded (3.115%); 0 B/s, 47 objects/s recovering 2024-08-17T17:25:03.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:25:03.323 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:03.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.439 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:03.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:03.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:03.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:03.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:03.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:03.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:03.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:03.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:03.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:04.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:04.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:04.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:04.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4010522882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4010522882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:04.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:04.236 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:04.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4010522882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:04.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:04.356 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:25:04.357 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:25:04.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:04.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:04.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:04.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:04.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:04.748 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:04.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:04.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:04.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:05.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.192 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:05.306 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:05 smithi142 ceph-mon[106729]: pgmap v826: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 8443/262785 objects degraded (3.213%); 0 B/s, 53 objects/s recovering 2024-08-17T17:25:05.306 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:05 smithi142 ceph-mon[103614]: pgmap v826: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 8443/262785 objects degraded (3.213%); 0 B/s, 53 objects/s recovering 2024-08-17T17:25:05.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:05.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:05 smithi152 ceph-mon[93456]: pgmap v826: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 8443/262785 objects degraded (3.213%); 0 B/s, 53 objects/s recovering 2024-08-17T17:25:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:05.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:05.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:05.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:05.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:05.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:05.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:05.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:05.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.941 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:05.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:06.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:06.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:06.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:06.129 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":826,"stamp":"2024-08-17T17:25:03.539786+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5739452389,"num_objects":87595,"num_object_clones":0,"num_object_copies":262785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8443,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87595,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":164201,"num_write_kb":8050128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2618,"num_bytes_recovered":93785376,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138153,"ondisk_log_size":138153,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":277,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22849840,"kb_used_data":17494960,"kb_used_omap":35,"kb_used_meta":5354588,"kb_avail":727111376,"statfs":{"total":767960285184,"available":744562049024,"internally_reserved":0,"allocated":17914839040,"data_stored":17904758431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36357,"internal_metadata":5483098619},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,5,10,14,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-511377408,"num_objects":-7803,"num_object_clones":0,"num_object_copies":-23409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7803,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":531,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003223"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":315,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991556+0000","last_change":"2024-08-17T17:12:40.332792+0000","last_active":"2024-08-17T17:24:55.991556+0000","last_peered":"2024-08-17T17:24:55.991556+0000","last_clean":"2024-08-17T17:24:55.991556+0000","last_became_active":"2024-08-17T17:12:40.331214+0000","last_became_peered":"2024-08-17T17:12:40.331214+0000","last_unstale":"2024-08-17T17:24:55.991556+0000","last_undegraded":"2024-08-17T17:24:55.991556+0000","last_fullsized":"2024-08-17T17:24:55.991556+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:56.880219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":209,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927457+0000","last_change":"2024-08-17T17:10:55.314746+0000","last_active":"2024-08-17T17:24:44.927457+0000","last_peered":"2024-08-17T17:24:44.927457+0000","last_clean":"2024-08-17T17:24:44.927457+0000","last_became_active":"2024-08-17T17:10:55.314400+0000","last_became_peered":"2024-08-17T17:10:55.314400+0000","last_unstale":"2024-08-17T17:24:44.927457+0000","last_undegraded":"2024-08-17T17:24:44.927457+0000","last_fullsized":"2024-08-17T17:24:44.927457+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:42.716493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927338+0000","last_change":"2024-08-17T17:21:31.903762+0000","last_active":"2024-08-17T17:24:44.927338+0000","last_peered":"2024-08-17T17:24:44.927338+0000","last_clean":"2024-08-17T17:24:44.927338+0000","last_became_active":"2024-08-17T17:21:31.902518+0000","last_became_peered":"2024-08-17T17:21:31.902518+0000","last_unstale":"2024-08-17T17:24:44.927338+0000","last_undegraded":"2024-08-17T17:24:44.927338+0000","last_fullsized":"2024-08-17T17:24:44.927338+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:13.790840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":188,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927222+0000","last_change":"2024-08-17T17:24:37.018179+0000","last_active":"2024-08-17T17:24:44.927222+0000","last_peered":"2024-08-17T17:24:44.927222+0000","last_clean":"2024-08-17T17:24:44.927222+0000","last_became_active":"2024-08-17T17:24:37.017881+0000","last_became_peered":"2024-08-17T17:24:37.017881+0000","last_unstale":"2024-08-17T17:24:44.927222+0000","last_undegraded":"2024-08-17T17:24:44.927222+0000","last_fullsized":"2024-08-17T17:24:44.927222+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:31.948414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":197,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991648+0000","last_change":"2024-08-17T17:24:37.077280+0000","last_active":"2024-08-17T17:24:55.991648+0000","last_peered":"2024-08-17T17:24:55.991648+0000","last_clean":"2024-08-17T17:24:55.991648+0000","last_became_active":"2024-08-17T17:24:37.076982+0000","last_became_peered":"2024-08-17T17:24:37.076982+0000","last_unstale":"2024-08-17T17:24:55.991648+0000","last_undegraded":"2024-08-17T17:24:55.991648+0000","last_fullsized":"2024-08-17T17:24:55.991648+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:19:51.551049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":190,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.929736+0000","last_change":"2024-08-17T16:57:28.388403+0000","last_active":"2024-08-17T17:24:44.929736+0000","last_peered":"2024-08-17T17:24:44.929736+0000","last_clean":"2024-08-17T17:24:44.929736+0000","last_became_active":"2024-08-17T16:57:28.388149+0000","last_became_peered":"2024-08-17T16:57:28.388149+0000","last_unstale":"2024-08-17T17:24:44.929736+0000","last_undegraded":"2024-08-17T17:24:44.929736+0000","last_fullsized":"2024-08-17T17:24:44.929736+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:39.819451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":291,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991798+0000","last_change":"2024-08-17T17:12:40.351876+0000","last_active":"2024-08-17T17:24:55.991798+0000","last_peered":"2024-08-17T17:24:55.991798+0000","last_clean":"2024-08-17T17:24:55.991798+0000","last_became_active":"2024-08-17T17:12:40.351012+0000","last_became_peered":"2024-08-17T17:12:40.351012+0000","last_unstale":"2024-08-17T17:24:55.991798+0000","last_undegraded":"2024-08-17T17:24:55.991798+0000","last_fullsized":"2024-08-17T17:24:55.991798+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:46.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":272,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.935357+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:24:44.935357+0000","last_peered":"2024-08-17T17:24:44.935357+0000","last_clean":"2024-08-17T17:24:44.935357+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:24:44.935357+0000","last_undegraded":"2024-08-17T17:24:44.935357+0000","last_fullsized":"2024-08-17T17:24:44.935357+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"252'10196","reported_seq":10270,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:02.358534+0000","last_change":"2024-08-17T17:24:37.092742+0000","last_active":"2024-08-17T17:25:02.358534+0000","last_peered":"2024-08-17T17:25:02.358534+0000","last_clean":"2024-08-17T17:23:57.406704+0000","last_became_active":"2024-08-17T17:24:37.077409+0000","last_became_peered":"2024-08-17T17:24:37.077409+0000","last_unstale":"2024-08-17T17:25:02.358534+0000","last_undegraded":"2024-08-17T17:24:36.881233+0000","last_fullsized":"2024-08-17T17:24:36.866632+0000","mapping_epoch":248,"log_start":"229'2963","ondisk_log_start":"229'2963","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7233,"log_dups_size":0,"ondisk_log_size":7233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:04.922682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356384768,"num_objects":5438,"num_object_clones":0,"num_object_copies":16314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1158,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10196,"num_write_kb":500288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"1,4,5","objects":12},{"shards":"4,5","objects":5426}],"blocked_by":[],"up_primary":0,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.928039+0000","last_change":"2024-08-17T17:24:37.001809+0000","last_active":"2024-08-17T17:24:44.928039+0000","last_peered":"2024-08-17T17:24:44.928039+0000","last_clean":"2024-08-17T17:24:44.928039+0000","last_became_active":"2024-08-17T17:24:36.967452+0000","last_became_peered":"2024-08-17T17:24:36.967452+0000","last_unstale":"2024-08-17T17:24:44.928039+0000","last_undegraded":"2024-08-17T17:24:44.928039+0000","last_fullsized":"2024-08-17T17:24:44.928039+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:52.169993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":95,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.928019+0000","last_change":"2024-08-17T17:21:31.876798+0000","last_active":"2024-08-17T17:24:44.928019+0000","last_peered":"2024-08-17T17:24:44.928019+0000","last_clean":"2024-08-17T17:24:44.928019+0000","last_became_active":"2024-08-17T17:21:31.876370+0000","last_became_peered":"2024-08-17T17:21:31.876370+0000","last_unstale":"2024-08-17T17:24:44.928019+0000","last_undegraded":"2024-08-17T17:24:44.928019+0000","last_fullsized":"2024-08-17T17:24:44.928019+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:33:16.352719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"252'10261","reported_seq":10299,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:25:02.731111+0000","last_change":"2024-08-17T17:22:18.544826+0000","last_active":"2024-08-17T17:25:02.731111+0000","last_peered":"2024-08-17T17:25:02.731111+0000","last_clean":"2024-08-17T17:25:02.731111+0000","last_became_active":"2024-08-17T17:22:18.544566+0000","last_became_peered":"2024-08-17T17:22:18.544566+0000","last_unstale":"2024-08-17T17:25:02.731111+0000","last_undegraded":"2024-08-17T17:25:02.731111+0000","last_fullsized":"2024-08-17T17:25:02.731111+0000","mapping_epoch":222,"log_start":"228'1400","ondisk_log_start":"228'1400","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8861,"log_dups_size":0,"ondisk_log_size":8861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:08.783581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355270656,"num_objects":5421,"num_object_clones":0,"num_object_copies":16263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10261,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":209,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927845+0000","last_change":"2024-08-17T17:07:51.480332+0000","last_active":"2024-08-17T17:24:44.927845+0000","last_peered":"2024-08-17T17:24:44.927845+0000","last_clean":"2024-08-17T17:24:44.927845+0000","last_became_active":"2024-08-17T17:07:51.479827+0000","last_became_peered":"2024-08-17T17:07:51.479827+0000","last_unstale":"2024-08-17T17:24:44.927845+0000","last_undegraded":"2024-08-17T17:24:44.927845+0000","last_fullsized":"2024-08-17T17:24:44.927845+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:36.919617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931105+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:24:44.931105+0000","last_peered":"2024-08-17T17:24:44.931105+0000","last_clean":"2024-08-17T17:24:44.931105+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:24:44.931105+0000","last_undegraded":"2024-08-17T17:24:44.931105+0000","last_fullsized":"2024-08-17T17:24:44.931105+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"252'10323","reported_seq":10408,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:25:02.364718+0000","last_change":"2024-08-17T17:24:37.880316+0000","last_active":"2024-08-17T17:25:02.364718+0000","last_peered":"2024-08-17T17:25:02.364718+0000","last_clean":"2024-08-17T17:25:02.364718+0000","last_became_active":"2024-08-17T17:24:37.867996+0000","last_became_peered":"2024-08-17T17:24:37.867996+0000","last_unstale":"2024-08-17T17:25:02.364718+0000","last_undegraded":"2024-08-17T17:25:02.364718+0000","last_fullsized":"2024-08-17T17:25:02.364718+0000","mapping_epoch":249,"log_start":"229'3133","ondisk_log_start":"229'3133","created":222,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7190,"log_dups_size":0,"ondisk_log_size":7190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:54.127182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10323,"num_write_kb":505216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":208,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927537+0000","last_change":"2024-08-17T17:21:31.904014+0000","last_active":"2024-08-17T17:24:44.927537+0000","last_peered":"2024-08-17T17:24:44.927537+0000","last_clean":"2024-08-17T17:24:44.927537+0000","last_became_active":"2024-08-17T17:21:31.903128+0000","last_became_peered":"2024-08-17T17:21:31.903128+0000","last_unstale":"2024-08-17T17:24:44.927537+0000","last_undegraded":"2024-08-17T17:24:44.927537+0000","last_fullsized":"2024-08-17T17:24:44.927537+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:56:59.504309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.935209+0000","last_change":"2024-08-17T17:07:51.484588+0000","last_active":"2024-08-17T17:24:44.935209+0000","last_peered":"2024-08-17T17:24:44.935209+0000","last_clean":"2024-08-17T17:24:44.935209+0000","last_became_active":"2024-08-17T17:07:51.483921+0000","last_became_peered":"2024-08-17T17:07:51.483921+0000","last_unstale":"2024-08-17T17:24:44.935209+0000","last_undegraded":"2024-08-17T17:24:44.935209+0000","last_fullsized":"2024-08-17T17:24:44.935209+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:17.711917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"252'10310","reported_seq":10391,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:59.250906+0000","last_change":"2024-08-17T17:24:37.189967+0000","last_active":"2024-08-17T17:24:59.250906+0000","last_peered":"2024-08-17T17:24:59.250906+0000","last_clean":"2024-08-17T17:23:57.376065+0000","last_became_active":"2024-08-17T17:24:37.169711+0000","last_became_peered":"2024-08-17T17:24:37.169711+0000","last_unstale":"2024-08-17T17:24:59.250906+0000","last_undegraded":"2024-08-17T17:24:37.020760+0000","last_fullsized":"2024-08-17T17:24:36.862553+0000","mapping_epoch":248,"log_start":"227'300","ondisk_log_start":"227'300","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377618432,"num_objects":5762,"num_object_clones":0,"num_object_copies":17286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10310,"num_write_kb":514304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":5742},{"shards":"2,3,7","objects":20}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":251,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991687+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:24:55.991687+0000","last_peered":"2024-08-17T17:24:55.991687+0000","last_clean":"2024-08-17T17:24:55.991687+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:24:55.991687+0000","last_undegraded":"2024-08-17T17:24:55.991687+0000","last_fullsized":"2024-08-17T17:24:55.991687+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":18,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991351+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:24:55.991351+0000","last_peered":"2024-08-17T17:24:55.991351+0000","last_clean":"2024-08-17T17:24:55.991351+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:24:55.991351+0000","last_undegraded":"2024-08-17T17:24:55.991351+0000","last_fullsized":"2024-08-17T17:24:55.991351+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"252'10213","reported_seq":10299,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:00.965115+0000","last_change":"2024-08-17T17:24:37.117181+0000","last_active":"2024-08-17T17:25:00.965115+0000","last_peered":"2024-08-17T17:25:00.965115+0000","last_clean":"2024-08-17T17:23:57.419490+0000","last_became_active":"2024-08-17T17:24:37.101719+0000","last_became_peered":"2024-08-17T17:24:37.101719+0000","last_unstale":"2024-08-17T17:25:00.965115+0000","last_undegraded":"2024-08-17T17:24:36.932397+0000","last_fullsized":"2024-08-17T17:24:36.865200+0000","mapping_epoch":248,"log_start":"228'2903","ondisk_log_start":"228'2903","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7310,"log_dups_size":0,"ondisk_log_size":7310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358940672,"num_objects":5477,"num_object_clones":0,"num_object_copies":16431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10213,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,3,4","objects":20},{"shards":"1,4","objects":5457}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":341,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991245+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:24:55.991245+0000","last_peered":"2024-08-17T17:24:55.991245+0000","last_clean":"2024-08-17T17:24:55.991245+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:24:55.991245+0000","last_undegraded":"2024-08-17T17:24:55.991245+0000","last_fullsized":"2024-08-17T17:24:55.991245+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931429+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:24:44.931429+0000","last_peered":"2024-08-17T17:24:44.931429+0000","last_clean":"2024-08-17T17:24:44.931429+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:24:44.931429+0000","last_undegraded":"2024-08-17T17:24:44.931429+0000","last_fullsized":"2024-08-17T17:24:44.931429+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"252'9940","reported_seq":10013,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:59.288414+0000","last_change":"2024-08-17T17:22:18.541297+0000","last_active":"2024-08-17T17:24:59.288414+0000","last_peered":"2024-08-17T17:24:59.288414+0000","last_clean":"2024-08-17T17:24:59.288414+0000","last_became_active":"2024-08-17T17:22:18.541002+0000","last_became_peered":"2024-08-17T17:22:18.541002+0000","last_unstale":"2024-08-17T17:24:59.288414+0000","last_undegraded":"2024-08-17T17:24:59.288414+0000","last_fullsized":"2024-08-17T17:24:59.288414+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":9940,"log_dups_size":0,"ondisk_log_size":9940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:49.446221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9940,"num_write_kb":494592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.935434+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:24:44.935434+0000","last_peered":"2024-08-17T17:24:44.935434+0000","last_clean":"2024-08-17T17:24:44.935434+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:24:44.935434+0000","last_undegraded":"2024-08-17T17:24:44.935434+0000","last_fullsized":"2024-08-17T17:24:44.935434+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931503+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:24:44.931503+0000","last_peered":"2024-08-17T17:24:44.931503+0000","last_clean":"2024-08-17T17:24:44.931503+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:24:44.931503+0000","last_undegraded":"2024-08-17T17:24:44.931503+0000","last_fullsized":"2024-08-17T17:24:44.931503+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"252'10112","reported_seq":10150,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:25:02.731220+0000","last_change":"2024-08-17T17:22:18.545009+0000","last_active":"2024-08-17T17:25:02.731220+0000","last_peered":"2024-08-17T17:25:02.731220+0000","last_clean":"2024-08-17T17:25:02.731220+0000","last_became_active":"2024-08-17T17:22:18.544525+0000","last_became_peered":"2024-08-17T17:22:18.544525+0000","last_unstale":"2024-08-17T17:25:02.731220+0000","last_undegraded":"2024-08-17T17:25:02.731220+0000","last_fullsized":"2024-08-17T17:25:02.731220+0000","mapping_epoch":222,"log_start":"228'1200","ondisk_log_start":"228'1200","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8912,"log_dups_size":0,"ondisk_log_size":8912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:35.215727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349962240,"num_objects":5340,"num_object_clones":0,"num_object_copies":16020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10112,"num_write_kb":494464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931250+0000","last_change":"2024-08-17T17:10:55.309763+0000","last_active":"2024-08-17T17:24:44.931250+0000","last_peered":"2024-08-17T17:24:44.931250+0000","last_clean":"2024-08-17T17:24:44.931250+0000","last_became_active":"2024-08-17T17:10:55.309426+0000","last_became_peered":"2024-08-17T17:10:55.309426+0000","last_unstale":"2024-08-17T17:24:44.931250+0000","last_undegraded":"2024-08-17T17:24:44.931250+0000","last_fullsized":"2024-08-17T17:24:44.931250+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:32.333446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931653+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:24:44.931653+0000","last_peered":"2024-08-17T17:24:44.931653+0000","last_clean":"2024-08-17T17:24:44.931653+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:24:44.931653+0000","last_undegraded":"2024-08-17T17:24:44.931653+0000","last_fullsized":"2024-08-17T17:24:44.931653+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"252'10393","reported_seq":10533,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:25:02.342941+0000","last_change":"2024-08-17T17:24:37.039258+0000","last_active":"2024-08-17T17:25:02.342941+0000","last_peered":"2024-08-17T17:25:02.342941+0000","last_clean":"2024-08-17T17:25:02.342941+0000","last_became_active":"2024-08-17T17:24:36.990742+0000","last_became_peered":"2024-08-17T17:24:36.990742+0000","last_unstale":"2024-08-17T17:25:02.342941+0000","last_undegraded":"2024-08-17T17:25:02.342941+0000","last_fullsized":"2024-08-17T17:25:02.342941+0000","mapping_epoch":248,"log_start":"229'3202","ondisk_log_start":"229'3202","created":222,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7191,"log_dups_size":0,"ondisk_log_size":7191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:32.770446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362872832,"num_objects":5537,"num_object_clones":0,"num_object_copies":16611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10393,"num_write_kb":509760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":292,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.990642+0000","last_change":"2024-08-17T17:12:40.349625+0000","last_active":"2024-08-17T17:24:55.990642+0000","last_peered":"2024-08-17T17:24:55.990642+0000","last_clean":"2024-08-17T17:24:55.990642+0000","last_became_active":"2024-08-17T17:12:40.343510+0000","last_became_peered":"2024-08-17T17:12:40.343510+0000","last_unstale":"2024-08-17T17:24:55.990642+0000","last_undegraded":"2024-08-17T17:24:55.990642+0000","last_fullsized":"2024-08-17T17:24:55.990642+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:12.858586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":247,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991907+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:24:55.991907+0000","last_peered":"2024-08-17T17:24:55.991907+0000","last_clean":"2024-08-17T17:24:55.991907+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:24:55.991907+0000","last_undegraded":"2024-08-17T17:24:55.991907+0000","last_fullsized":"2024-08-17T17:24:55.991907+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"252'10298","reported_seq":10365,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:02.754483+0000","last_change":"2024-08-17T17:24:37.108494+0000","last_active":"2024-08-17T17:25:02.754483+0000","last_peered":"2024-08-17T17:25:02.754483+0000","last_clean":"2024-08-17T17:23:57.402090+0000","last_became_active":"2024-08-17T17:24:37.043537+0000","last_became_peered":"2024-08-17T17:24:37.043537+0000","last_unstale":"2024-08-17T17:25:02.754483+0000","last_undegraded":"2024-08-17T17:24:37.108248+0000","last_fullsized":"2024-08-17T17:24:36.865430+0000","mapping_epoch":248,"log_start":"228'1400","ondisk_log_start":"228'1400","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:38.028073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355336192,"num_objects":5422,"num_object_clones":0,"num_object_copies":16266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10298,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":19},{"shards":"1,6","objects":5403}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":583,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:44.928242+0000","last_change":"2024-08-17T17:24:37.870478+0000","last_active":"2024-08-17T17:24:44.928242+0000","last_peered":"2024-08-17T17:24:44.928242+0000","last_clean":"2024-08-17T17:24:29.860281+0000","last_became_active":"2024-08-17T17:24:37.870074+0000","last_became_peered":"2024-08-17T17:24:37.870074+0000","last_unstale":"2024-08-17T17:24:44.928242+0000","last_undegraded":"2024-08-17T17:24:37.866338+0000","last_fullsized":"2024-08-17T17:24:37.864791+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:55.373526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,6],"avail_no_missing":["7","1","6"],"object_location_counts":[{"shards":"1,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":167,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.984477+0000","last_change":"2024-08-17T17:21:31.878312+0000","last_active":"2024-08-17T17:24:55.984477+0000","last_peered":"2024-08-17T17:24:55.984477+0000","last_clean":"2024-08-17T17:24:55.984477+0000","last_became_active":"2024-08-17T17:21:31.877932+0000","last_became_peered":"2024-08-17T17:21:31.877932+0000","last_unstale":"2024-08-17T17:24:55.984477+0000","last_undegraded":"2024-08-17T17:24:55.984477+0000","last_fullsized":"2024-08-17T17:24:55.984477+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:17.773813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":289,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991067+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:24:55.991067+0000","last_peered":"2024-08-17T17:24:55.991067+0000","last_clean":"2024-08-17T17:24:55.991067+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:24:55.991067+0000","last_undegraded":"2024-08-17T17:24:55.991067+0000","last_fullsized":"2024-08-17T17:24:55.991067+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":189,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.935574+0000","last_change":"2024-08-17T16:57:30.217742+0000","last_active":"2024-08-17T17:24:44.935574+0000","last_peered":"2024-08-17T17:24:44.935574+0000","last_clean":"2024-08-17T17:24:44.935574+0000","last_became_active":"2024-08-17T16:57:30.217304+0000","last_became_peered":"2024-08-17T16:57:30.217304+0000","last_unstale":"2024-08-17T17:24:44.935574+0000","last_undegraded":"2024-08-17T17:24:44.935574+0000","last_fullsized":"2024-08-17T17:24:44.935574+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:13.774300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"252'10063","reported_seq":11696,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:24:59.248061+0000","last_change":"2024-08-17T17:24:37.159088+0000","last_active":"2024-08-17T17:24:59.248061+0000","last_peered":"2024-08-17T17:24:59.248061+0000","last_clean":"2024-08-17T17:23:57.421745+0000","last_became_active":"2024-08-17T17:24:37.146272+0000","last_became_peered":"2024-08-17T17:24:37.146272+0000","last_unstale":"2024-08-17T17:24:59.248061+0000","last_undegraded":"2024-08-17T17:24:37.026615+0000","last_fullsized":"2024-08-17T17:24:36.862086+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":10063,"log_dups_size":0,"ondisk_log_size":10063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363003904,"num_objects":5539,"num_object_clones":0,"num_object_copies":16617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1235,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10063,"num_write_kb":499264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":771,"num_bytes_recovered":50462720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,5,7","objects":18},{"shards":"2,7","objects":5521}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":256,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.928143+0000","last_change":"2024-08-17T17:24:36.968412+0000","last_active":"2024-08-17T17:24:44.928143+0000","last_peered":"2024-08-17T17:24:44.928143+0000","last_clean":"2024-08-17T17:24:44.928143+0000","last_became_active":"2024-08-17T17:24:36.967865+0000","last_became_peered":"2024-08-17T17:24:36.967865+0000","last_unstale":"2024-08-17T17:24:44.928143+0000","last_undegraded":"2024-08-17T17:24:44.928143+0000","last_fullsized":"2024-08-17T17:24:44.928143+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:38.927278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":188,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.930971+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:24:44.930971+0000","last_peered":"2024-08-17T17:24:44.930971+0000","last_clean":"2024-08-17T17:24:44.930971+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:24:44.930971+0000","last_undegraded":"2024-08-17T17:24:44.930971+0000","last_fullsized":"2024-08-17T17:24:44.930971+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":18,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991146+0000","last_change":"2024-08-17T17:24:37.073148+0000","last_active":"2024-08-17T17:24:55.991146+0000","last_peered":"2024-08-17T17:24:55.991146+0000","last_clean":"2024-08-17T17:24:55.991146+0000","last_became_active":"2024-08-17T17:24:37.072372+0000","last_became_peered":"2024-08-17T17:24:37.072372+0000","last_unstale":"2024-08-17T17:24:55.991146+0000","last_undegraded":"2024-08-17T17:24:55.991146+0000","last_fullsized":"2024-08-17T17:24:55.991146+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.659683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"252'10049","reported_seq":10122,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:59.298784+0000","last_change":"2024-08-17T17:22:18.540626+0000","last_active":"2024-08-17T17:24:59.298784+0000","last_peered":"2024-08-17T17:24:59.298784+0000","last_clean":"2024-08-17T17:24:59.298784+0000","last_became_active":"2024-08-17T17:22:18.540354+0000","last_became_peered":"2024-08-17T17:22:18.540354+0000","last_unstale":"2024-08-17T17:24:59.298784+0000","last_undegraded":"2024-08-17T17:24:59.298784+0000","last_fullsized":"2024-08-17T17:24:59.298784+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":10049,"log_dups_size":0,"ondisk_log_size":10049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:39:22.173642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367853568,"num_objects":5613,"num_object_clones":0,"num_object_copies":16839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10049,"num_write_kb":501184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":365,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.986660+0000","last_change":"2024-08-17T17:24:36.969933+0000","last_active":"2024-08-17T17:24:55.986660+0000","last_peered":"2024-08-17T17:24:55.986660+0000","last_clean":"2024-08-17T17:24:55.986660+0000","last_became_active":"2024-08-17T17:24:36.969657+0000","last_became_peered":"2024-08-17T17:24:36.969657+0000","last_unstale":"2024-08-17T17:24:55.986660+0000","last_undegraded":"2024-08-17T17:24:55.986660+0000","last_fullsized":"2024-08-17T17:24:55.986660+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:10.367450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":332,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.993081+0000","last_change":"2024-08-17T17:10:55.336484+0000","last_active":"2024-08-17T17:24:55.993081+0000","last_peered":"2024-08-17T17:24:55.993081+0000","last_clean":"2024-08-17T17:24:55.993081+0000","last_became_active":"2024-08-17T17:10:55.336148+0000","last_became_peered":"2024-08-17T17:10:55.336148+0000","last_unstale":"2024-08-17T17:24:55.993081+0000","last_undegraded":"2024-08-17T17:24:55.993081+0000","last_fullsized":"2024-08-17T17:24:55.993081+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:34.690587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":382,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.995854+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:24:55.995854+0000","last_peered":"2024-08-17T17:24:55.995854+0000","last_clean":"2024-08-17T17:24:55.995854+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:24:55.995854+0000","last_undegraded":"2024-08-17T17:24:55.995854+0000","last_fullsized":"2024-08-17T17:24:55.995854+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"252'10325","reported_seq":12784,"reported_epoch":252,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:25:02.765069+0000","last_change":"2024-08-17T17:24:37.175383+0000","last_active":"2024-08-17T17:25:02.765069+0000","last_peered":"2024-08-17T17:25:02.765069+0000","last_clean":"2024-08-17T17:23:57.418972+0000","last_became_active":"2024-08-17T17:24:37.093081+0000","last_became_peered":"2024-08-17T17:24:37.093081+0000","last_unstale":"2024-08-17T17:25:02.765069+0000","last_undegraded":"2024-08-17T17:25:02.765069+0000","last_fullsized":"2024-08-17T17:24:36.864542+0000","mapping_epoch":248,"log_start":"228'1500","ondisk_log_start":"228'1500","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8825,"log_dups_size":0,"ondisk_log_size":8825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:36:35.481999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350683136,"num_objects":5351,"num_object_clones":0,"num_object_copies":16053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10325,"num_write_kb":501632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1201,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":5351}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":174,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.988471+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:24:55.988471+0000","last_peered":"2024-08-17T17:24:55.988471+0000","last_clean":"2024-08-17T17:24:55.988471+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:24:55.988471+0000","last_undegraded":"2024-08-17T17:24:55.988471+0000","last_fullsized":"2024-08-17T17:24:55.988471+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":227,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931303+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:24:44.931303+0000","last_peered":"2024-08-17T17:24:44.931303+0000","last_clean":"2024-08-17T17:24:44.931303+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:24:44.931303+0000","last_undegraded":"2024-08-17T17:24:44.931303+0000","last_fullsized":"2024-08-17T17:24:44.931303+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":344,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.988729+0000","last_change":"2024-08-17T17:12:40.341797+0000","last_active":"2024-08-17T17:24:55.988729+0000","last_peered":"2024-08-17T17:24:55.988729+0000","last_clean":"2024-08-17T17:24:55.988729+0000","last_became_active":"2024-08-17T17:12:40.341525+0000","last_became_peered":"2024-08-17T17:12:40.341525+0000","last_unstale":"2024-08-17T17:24:55.988729+0000","last_undegraded":"2024-08-17T17:24:55.988729+0000","last_fullsized":"2024-08-17T17:24:55.988729+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:36.529991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"252'10343","reported_seq":11533,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:03.136251+0000","last_change":"2024-08-17T17:24:37.105563+0000","last_active":"2024-08-17T17:25:03.136251+0000","last_peered":"2024-08-17T17:25:03.136251+0000","last_clean":"2024-08-17T17:23:57.419041+0000","last_became_active":"2024-08-17T17:24:37.086172+0000","last_became_peered":"2024-08-17T17:24:37.086172+0000","last_unstale":"2024-08-17T17:25:03.136251+0000","last_undegraded":"2024-08-17T17:24:36.874535+0000","last_fullsized":"2024-08-17T17:24:36.862985+0000","mapping_epoch":248,"log_start":"228'1700","ondisk_log_start":"228'1700","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8643,"log_dups_size":0,"ondisk_log_size":8643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:22.975746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357236736,"num_objects":5451,"num_object_clones":0,"num_object_copies":16353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1223,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10343,"num_write_kb":505408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":567,"num_bytes_recovered":37093376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"3,4,7","objects":16},{"shards":"4,7","objects":5435}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":269,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.992154+0000","last_change":"2024-08-17T17:21:31.870010+0000","last_active":"2024-08-17T17:24:55.992154+0000","last_peered":"2024-08-17T17:24:55.992154+0000","last_clean":"2024-08-17T17:24:55.992154+0000","last_became_active":"2024-08-17T17:21:31.869696+0000","last_became_peered":"2024-08-17T17:21:31.869696+0000","last_unstale":"2024-08-17T17:24:55.992154+0000","last_undegraded":"2024-08-17T17:24:55.992154+0000","last_fullsized":"2024-08-17T17:24:55.992154+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:57.550019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":385,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.993305+0000","last_change":"2024-08-17T17:24:36.967179+0000","last_active":"2024-08-17T17:24:55.993305+0000","last_peered":"2024-08-17T17:24:55.993305+0000","last_clean":"2024-08-17T17:24:55.993305+0000","last_became_active":"2024-08-17T17:24:36.966858+0000","last_became_peered":"2024-08-17T17:24:36.966858+0000","last_unstale":"2024-08-17T17:24:55.993305+0000","last_undegraded":"2024-08-17T17:24:55.993305+0000","last_fullsized":"2024-08-17T17:24:55.993305+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:06.114944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":333,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.990080+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:24:55.990080+0000","last_peered":"2024-08-17T17:24:55.990080+0000","last_clean":"2024-08-17T17:24:55.990080+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:24:55.990080+0000","last_undegraded":"2024-08-17T17:24:55.990080+0000","last_fullsized":"2024-08-17T17:24:55.990080+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"252'10428","reported_seq":10484,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:25:02.703158+0000","last_change":"2024-08-17T17:24:37.088906+0000","last_active":"2024-08-17T17:25:02.703158+0000","last_peered":"2024-08-17T17:25:02.703158+0000","last_clean":"2024-08-17T17:23:57.429915+0000","last_became_active":"2024-08-17T17:24:37.076922+0000","last_became_peered":"2024-08-17T17:24:37.076922+0000","last_unstale":"2024-08-17T17:25:02.703158+0000","last_undegraded":"2024-08-17T17:24:36.881086+0000","last_fullsized":"2024-08-17T17:24:36.865574+0000","mapping_epoch":248,"log_start":"228'1600","ondisk_log_start":"228'1600","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8828,"log_dups_size":0,"ondisk_log_size":8828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:14.779357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353959960,"num_objects":5402,"num_object_clones":0,"num_object_copies":16206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1272,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5402,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10428,"num_write_kb":506497,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"3,4,6","objects":23},{"shards":"4,6","objects":5379}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":343,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.988816+0000","last_change":"2024-08-17T17:21:31.870419+0000","last_active":"2024-08-17T17:24:55.988816+0000","last_peered":"2024-08-17T17:24:55.988816+0000","last_clean":"2024-08-17T17:24:55.988816+0000","last_became_active":"2024-08-17T17:21:31.870030+0000","last_became_peered":"2024-08-17T17:21:31.870030+0000","last_unstale":"2024-08-17T17:24:55.988816+0000","last_undegraded":"2024-08-17T17:24:55.988816+0000","last_fullsized":"2024-08-17T17:24:55.988816+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:27.814200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927600+0000","last_change":"2024-08-17T17:24:36.967766+0000","last_active":"2024-08-17T17:24:44.927600+0000","last_peered":"2024-08-17T17:24:44.927600+0000","last_clean":"2024-08-17T17:24:44.927600+0000","last_became_active":"2024-08-17T17:24:36.967444+0000","last_became_peered":"2024-08-17T17:24:36.967444+0000","last_unstale":"2024-08-17T17:24:44.927600+0000","last_undegraded":"2024-08-17T17:24:44.927600+0000","last_fullsized":"2024-08-17T17:24:44.927600+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:13.517482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":298,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927307+0000","last_change":"2024-08-17T17:24:36.968214+0000","last_active":"2024-08-17T17:24:44.927307+0000","last_peered":"2024-08-17T17:24:44.927307+0000","last_clean":"2024-08-17T17:24:44.927307+0000","last_became_active":"2024-08-17T17:24:36.966783+0000","last_became_peered":"2024-08-17T17:24:36.966783+0000","last_unstale":"2024-08-17T17:24:44.927307+0000","last_undegraded":"2024-08-17T17:24:44.927307+0000","last_fullsized":"2024-08-17T17:24:44.927307+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:46.370596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"252'10240","reported_seq":10278,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:25:02.293063+0000","last_change":"2024-08-17T17:22:18.541323+0000","last_active":"2024-08-17T17:25:02.293063+0000","last_peered":"2024-08-17T17:25:02.293063+0000","last_clean":"2024-08-17T17:25:02.293063+0000","last_became_active":"2024-08-17T17:22:18.541054+0000","last_became_peered":"2024-08-17T17:22:18.541054+0000","last_unstale":"2024-08-17T17:25:02.293063+0000","last_undegraded":"2024-08-17T17:25:02.293063+0000","last_fullsized":"2024-08-17T17:25:02.293063+0000","mapping_epoch":222,"log_start":"229'3076","ondisk_log_start":"229'3076","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":7164,"log_dups_size":0,"ondisk_log_size":7164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:46.052040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355467264,"num_objects":5424,"num_object_clones":0,"num_object_copies":16272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10240,"num_write_kb":501248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":341,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.992233+0000","last_change":"2024-08-17T17:24:37.030734+0000","last_active":"2024-08-17T17:24:55.992233+0000","last_peered":"2024-08-17T17:24:55.992233+0000","last_clean":"2024-08-17T17:24:55.992233+0000","last_became_active":"2024-08-17T17:24:36.993991+0000","last_became_peered":"2024-08-17T17:24:36.993991+0000","last_unstale":"2024-08-17T17:24:55.992233+0000","last_undegraded":"2024-08-17T17:24:55.992233+0000","last_fullsized":"2024-08-17T17:24:55.992233+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:10:34.521505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":331,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.988471+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:24:55.988471+0000","last_peered":"2024-08-17T17:24:55.988471+0000","last_clean":"2024-08-17T17:24:55.988471+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:24:55.988471+0000","last_undegraded":"2024-08-17T17:24:55.988471+0000","last_fullsized":"2024-08-17T17:24:55.988471+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":331,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.988174+0000","last_change":"2024-08-17T17:24:37.013910+0000","last_active":"2024-08-17T17:24:55.988174+0000","last_peered":"2024-08-17T17:24:55.988174+0000","last_clean":"2024-08-17T17:24:55.988174+0000","last_became_active":"2024-08-17T17:24:37.013558+0000","last_became_peered":"2024-08-17T17:24:37.013558+0000","last_unstale":"2024-08-17T17:24:55.988174+0000","last_undegraded":"2024-08-17T17:24:55.988174+0000","last_fullsized":"2024-08-17T17:24:55.988174+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:56.516442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"252'10429","reported_seq":10467,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:25:02.679750+0000","last_change":"2024-08-17T17:22:18.545136+0000","last_active":"2024-08-17T17:25:02.679750+0000","last_peered":"2024-08-17T17:25:02.679750+0000","last_clean":"2024-08-17T17:25:02.679750+0000","last_became_active":"2024-08-17T17:22:18.544176+0000","last_became_peered":"2024-08-17T17:22:18.544176+0000","last_unstale":"2024-08-17T17:25:02.679750+0000","last_undegraded":"2024-08-17T17:25:02.679750+0000","last_fullsized":"2024-08-17T17:25:02.679750+0000","mapping_epoch":222,"log_start":"228'1600","ondisk_log_start":"228'1600","created":222,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:17.526108+0000","last_clean_scrub_stamp":"2024-08-17T17:22:17.526108+0000","objects_scrubbed":0,"log_size":8829,"log_dups_size":0,"ondisk_log_size":8829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:08.787886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354615296,"num_objects":5411,"num_object_clones":0,"num_object_copies":16233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10429,"num_write_kb":506880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":22,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991047+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:24:55.991047+0000","last_peered":"2024-08-17T17:24:55.991047+0000","last_clean":"2024-08-17T17:24:55.991047+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:24:55.991047+0000","last_undegraded":"2024-08-17T17:24:55.991047+0000","last_fullsized":"2024-08-17T17:24:55.991047+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":219,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.935610+0000","last_change":"2024-08-17T17:10:55.309715+0000","last_active":"2024-08-17T17:24:44.935610+0000","last_peered":"2024-08-17T17:24:44.935610+0000","last_clean":"2024-08-17T17:24:44.935610+0000","last_became_active":"2024-08-17T17:10:55.309244+0000","last_became_peered":"2024-08-17T17:10:55.309244+0000","last_unstale":"2024-08-17T17:24:44.935610+0000","last_undegraded":"2024-08-17T17:24:44.935610+0000","last_fullsized":"2024-08-17T17:24:44.935610+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:36.488876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931176+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:24:44.931176+0000","last_peered":"2024-08-17T17:24:44.931176+0000","last_clean":"2024-08-17T17:24:44.931176+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:24:44.931176+0000","last_undegraded":"2024-08-17T17:24:44.931176+0000","last_fullsized":"2024-08-17T17:24:44.931176+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931229+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:24:44.931229+0000","last_peered":"2024-08-17T17:24:44.931229+0000","last_clean":"2024-08-17T17:24:44.931229+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:24:44.931229+0000","last_undegraded":"2024-08-17T17:24:44.931229+0000","last_fullsized":"2024-08-17T17:24:44.931229+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":212,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931513+0000","last_change":"2024-08-17T16:57:28.391678+0000","last_active":"2024-08-17T17:24:44.931513+0000","last_peered":"2024-08-17T17:24:44.931513+0000","last_clean":"2024-08-17T17:24:44.931513+0000","last_became_active":"2024-08-17T16:57:28.390961+0000","last_became_peered":"2024-08-17T16:57:28.390961+0000","last_unstale":"2024-08-17T17:24:44.931513+0000","last_undegraded":"2024-08-17T17:24:44.931513+0000","last_fullsized":"2024-08-17T17:24:44.931513+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":332,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.989936+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:24:55.989936+0000","last_peered":"2024-08-17T17:24:55.989936+0000","last_clean":"2024-08-17T17:24:55.989936+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:24:55.989936+0000","last_undegraded":"2024-08-17T17:24:55.989936+0000","last_fullsized":"2024-08-17T17:24:55.989936+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":210,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927417+0000","last_change":"2024-08-17T17:12:40.330351+0000","last_active":"2024-08-17T17:24:44.927417+0000","last_peered":"2024-08-17T17:24:44.927417+0000","last_clean":"2024-08-17T17:24:44.927417+0000","last_became_active":"2024-08-17T17:12:40.329542+0000","last_became_peered":"2024-08-17T17:12:40.329542+0000","last_unstale":"2024-08-17T17:24:44.927417+0000","last_undegraded":"2024-08-17T17:24:44.927417+0000","last_fullsized":"2024-08-17T17:24:44.927417+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:22.839051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":150,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927719+0000","last_change":"2024-08-17T17:21:31.909108+0000","last_active":"2024-08-17T17:24:44.927719+0000","last_peered":"2024-08-17T17:24:44.927719+0000","last_clean":"2024-08-17T17:24:44.927719+0000","last_became_active":"2024-08-17T17:21:31.908882+0000","last_became_peered":"2024-08-17T17:21:31.908882+0000","last_unstale":"2024-08-17T17:24:44.927719+0000","last_undegraded":"2024-08-17T17:24:44.927719+0000","last_fullsized":"2024-08-17T17:24:44.927719+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:24.197352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":279,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.990637+0000","last_change":"2024-08-17T17:12:40.330758+0000","last_active":"2024-08-17T17:24:55.990637+0000","last_peered":"2024-08-17T17:24:55.990637+0000","last_clean":"2024-08-17T17:24:55.990637+0000","last_became_active":"2024-08-17T17:12:40.330248+0000","last_became_peered":"2024-08-17T17:12:40.330248+0000","last_unstale":"2024-08-17T17:24:55.990637+0000","last_undegraded":"2024-08-17T17:24:55.990637+0000","last_fullsized":"2024-08-17T17:24:55.990637+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:49.362492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":317,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991996+0000","last_change":"2024-08-17T17:24:37.013440+0000","last_active":"2024-08-17T17:24:55.991996+0000","last_peered":"2024-08-17T17:24:55.991996+0000","last_clean":"2024-08-17T17:24:55.991996+0000","last_became_active":"2024-08-17T17:24:37.013049+0000","last_became_peered":"2024-08-17T17:24:37.013049+0000","last_unstale":"2024-08-17T17:24:55.991996+0000","last_undegraded":"2024-08-17T17:24:55.991996+0000","last_fullsized":"2024-08-17T17:24:55.991996+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:50.353483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":291,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.988077+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:24:55.988077+0000","last_peered":"2024-08-17T17:24:55.988077+0000","last_clean":"2024-08-17T17:24:55.988077+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:24:55.988077+0000","last_undegraded":"2024-08-17T17:24:55.988077+0000","last_fullsized":"2024-08-17T17:24:55.988077+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.935534+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:24:44.935534+0000","last_peered":"2024-08-17T17:24:44.935534+0000","last_clean":"2024-08-17T17:24:44.935534+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:24:44.935534+0000","last_undegraded":"2024-08-17T17:24:44.935534+0000","last_fullsized":"2024-08-17T17:24:44.935534+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":323,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.990801+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:24:55.990801+0000","last_peered":"2024-08-17T17:24:55.990801+0000","last_clean":"2024-08-17T17:24:55.990801+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:24:55.990801+0000","last_undegraded":"2024-08-17T17:24:55.990801+0000","last_fullsized":"2024-08-17T17:24:55.990801+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":188,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.927778+0000","last_change":"2024-08-17T17:24:37.074122+0000","last_active":"2024-08-17T17:24:44.927778+0000","last_peered":"2024-08-17T17:24:44.927778+0000","last_clean":"2024-08-17T17:24:44.927778+0000","last_became_active":"2024-08-17T17:24:37.073843+0000","last_became_peered":"2024-08-17T17:24:37.073843+0000","last_unstale":"2024-08-17T17:24:44.927778+0000","last_undegraded":"2024-08-17T17:24:44.927778+0000","last_fullsized":"2024-08-17T17:24:44.927778+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:43.139115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":331,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.987630+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:24:55.987630+0000","last_peered":"2024-08-17T17:24:55.987630+0000","last_clean":"2024-08-17T17:24:55.987630+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:24:55.987630+0000","last_undegraded":"2024-08-17T17:24:55.987630+0000","last_fullsized":"2024-08-17T17:24:55.987630+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":290,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991969+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:24:55.991969+0000","last_peered":"2024-08-17T17:24:55.991969+0000","last_clean":"2024-08-17T17:24:55.991969+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:24:55.991969+0000","last_undegraded":"2024-08-17T17:24:55.991969+0000","last_fullsized":"2024-08-17T17:24:55.991969+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":225,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931155+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:24:44.931155+0000","last_peered":"2024-08-17T17:24:44.931155+0000","last_clean":"2024-08-17T17:24:44.931155+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:24:44.931155+0000","last_undegraded":"2024-08-17T17:24:44.931155+0000","last_fullsized":"2024-08-17T17:24:44.931155+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":354,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.984019+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:24:55.984019+0000","last_peered":"2024-08-17T17:24:55.984019+0000","last_clean":"2024-08-17T17:24:55.984019+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:24:55.984019+0000","last_undegraded":"2024-08-17T17:24:55.984019+0000","last_fullsized":"2024-08-17T17:24:55.984019+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":18,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991374+0000","last_change":"2024-08-17T17:24:37.038593+0000","last_active":"2024-08-17T17:24:55.991374+0000","last_peered":"2024-08-17T17:24:55.991374+0000","last_clean":"2024-08-17T17:24:55.991374+0000","last_became_active":"2024-08-17T17:24:37.037490+0000","last_became_peered":"2024-08-17T17:24:37.037490+0000","last_unstale":"2024-08-17T17:24:55.991374+0000","last_undegraded":"2024-08-17T17:24:55.991374+0000","last_fullsized":"2024-08-17T17:24:55.991374+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:50.047710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":233,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931211+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:24:44.931211+0000","last_peered":"2024-08-17T17:24:44.931211+0000","last_clean":"2024-08-17T17:24:44.931211+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:24:44.931211+0000","last_undegraded":"2024-08-17T17:24:44.931211+0000","last_fullsized":"2024-08-17T17:24:44.931211+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":106,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931446+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:24:44.931446+0000","last_peered":"2024-08-17T17:24:44.931446+0000","last_clean":"2024-08-17T17:24:44.931446+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:24:44.931446+0000","last_undegraded":"2024-08-17T17:24:44.931446+0000","last_fullsized":"2024-08-17T17:24:44.931446+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":243,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931087+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:24:44.931087+0000","last_peered":"2024-08-17T17:24:44.931087+0000","last_clean":"2024-08-17T17:24:44.931087+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:24:44.931087+0000","last_undegraded":"2024-08-17T17:24:44.931087+0000","last_fullsized":"2024-08-17T17:24:44.931087+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":106,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.931471+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:24:44.931471+0000","last_peered":"2024-08-17T17:24:44.931471+0000","last_clean":"2024-08-17T17:24:44.931471+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:24:44.931471+0000","last_undegraded":"2024-08-17T17:24:44.931471+0000","last_fullsized":"2024-08-17T17:24:44.931471+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":336,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991412+0000","last_change":"2024-08-17T17:24:36.972596+0000","last_active":"2024-08-17T17:24:55.991412+0000","last_peered":"2024-08-17T17:24:55.991412+0000","last_clean":"2024-08-17T17:24:55.991412+0000","last_became_active":"2024-08-17T17:24:36.971541+0000","last_became_peered":"2024-08-17T17:24:36.971541+0000","last_unstale":"2024-08-17T17:24:55.991412+0000","last_undegraded":"2024-08-17T17:24:55.991412+0000","last_fullsized":"2024-08-17T17:24:55.991412+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:38.457003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":308,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991735+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:24:55.991735+0000","last_peered":"2024-08-17T17:24:55.991735+0000","last_clean":"2024-08-17T17:24:55.991735+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:24:55.991735+0000","last_undegraded":"2024-08-17T17:24:55.991735+0000","last_fullsized":"2024-08-17T17:24:55.991735+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-17T17:24:44.930926+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:24:44.930926+0000","last_peered":"2024-08-17T17:24:44.930926+0000","last_clean":"2024-08-17T17:24:44.930926+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:24:44.930926+0000","last_undegraded":"2024-08-17T17:24:44.930926+0000","last_fullsized":"2024-08-17T17:24:44.930926+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":24,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-17T17:24:55.991134+0000","last_change":"2024-08-17T17:24:37.032048+0000","last_active":"2024-08-17T17:24:55.991134+0000","last_peered":"2024-08-17T17:24:55.991134+0000","last_clean":"2024-08-17T17:24:55.991134+0000","last_became_active":"2024-08-17T17:24:36.972161+0000","last_became_peered":"2024-08-17T17:24:36.972161+0000","last_unstale":"2024-08-17T17:24:55.991134+0000","last_undegraded":"2024-08-17T17:24:55.991134+0000","last_fullsized":"2024-08-17T17:24:55.991134+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:55.363820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5738856472,"num_objects":87569,"num_object_clones":0,"num_object_copies":262707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87569,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163923,"num_write_kb":8047681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2590,"num_bytes_recovered":91422720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17813417984,"data_stored":17813405768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137946,"ondisk_log_size":137946,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1184,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19878,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201637,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301808,"kb_used_data":1396328,"kb_used_omap":1,"kb_used_meta":905470,"kb_avail":91443344,"statfs":{"total":95995035648,"available":93637984256,"internally_reserved":0,"allocated":1429839872,"data_stored":1428564949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927201624},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463273,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3313772,"kb_used_data":2420004,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90431380,"statfs":{"total":95995035648,"available":92601733120,"internally_reserved":0,"allocated":2478084096,"data_stored":2476813150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698491947,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2166932,"kb_used_data":1779916,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":91578220,"statfs":{"total":95995035648,"available":93776097280,"internally_reserved":0,"allocated":1822633984,"data_stored":1821381822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986547,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3703080,"kb_used_data":2810464,"kb_used_omap":1,"kb_used_meta":892606,"kb_avail":90042072,"statfs":{"total":95995035648,"available":92203081728,"internally_reserved":0,"allocated":2877915136,"data_stored":2876647542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914028888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225366,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2363412,"kb_used_data":1821644,"kb_used_omap":4,"kb_used_meta":541627,"kb_avail":91381740,"statfs":{"total":95995035648,"available":93574901760,"internally_reserved":0,"allocated":1865363456,"data_stored":1864111546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":554626496},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075691,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2542424,"kb_used_data":2158864,"kb_used_omap":6,"kb_used_meta":383545,"kb_avail":91202728,"statfs":{"total":95995035648,"available":93391593472,"internally_reserved":0,"allocated":2210676736,"data_stored":2209421501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":392750523},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533584,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1734676,"kb_used_data":1059084,"kb_used_omap":5,"kb_used_meta":675578,"kb_avail":92010476,"statfs":{"total":95995035648,"available":94218727424,"internally_reserved":0,"allocated":1084502016,"data_stored":1083249416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":691792112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089561,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4723736,"kb_used_data":4048656,"kb_used_omap":5,"kb_used_meta":675066,"kb_avail":89021416,"statfs":{"total":95995035648,"available":91157929984,"internally_reserved":0,"allocated":4145823744,"data_stored":4144568505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":691267643},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4123262976,"data_stored":4123262976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1077936128,"data_stored":1077936128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2201812992,"data_stored":2201812992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1844711424,"data_stored":1844707352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2855014400,"data_stored":2855010328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1814364160,"data_stored":1814364160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2473791488,"data_stored":2473787416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1422524416,"data_stored":1422524416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:25:06.133 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:25:06.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 26 2024-08-17T17:25:06.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:06.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:06.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:06.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:06 smithi152 ceph-mon[93456]: from='client.36462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:06 smithi152 ceph-mon[93456]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:25:06.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8443/262785 objects degraded (3.213%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:06.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:06 smithi142 ceph-mon[106729]: from='client.36462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:06.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:06 smithi142 ceph-mon[106729]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:25:06.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8443/262785 objects degraded (3.213%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:06.375 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:06 smithi142 ceph-mon[103614]: from='client.36462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:25:06.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:06 smithi142 ceph-mon[103614]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:25:06.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8443/262785 objects degraded (3.213%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:06.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:06.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:06.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:06.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:06.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:06.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:06.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:06.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:06.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:07.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:07.057 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:07.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:07.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:07.183 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:07.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:07 smithi152 ceph-mon[93456]: pgmap v828: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 7208/259647 objects degraded (2.776%); 17/259647 objects misplaced (0.007%); 6.7 KiB/s, 55 objects/s recovering 2024-08-17T17:25:07.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:07 smithi152 ceph-mon[93456]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:25:07.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:07 smithi142 ceph-mon[106729]: pgmap v828: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 7208/259647 objects degraded (2.776%); 17/259647 objects misplaced (0.007%); 6.7 KiB/s, 55 objects/s recovering 2024-08-17T17:25:07.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:07 smithi142 ceph-mon[106729]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:25:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:07 smithi142 ceph-mon[103614]: pgmap v828: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 7208/259647 objects degraded (2.776%); 17/259647 objects misplaced (0.007%); 6.7 KiB/s, 55 objects/s recovering 2024-08-17T17:25:07.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:07 smithi142 ceph-mon[103614]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:25:07.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:07.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:07.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:07.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:07.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.752 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:07.752 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:07.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:07.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:07.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:08.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:08.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:08.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:08.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:08.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:08.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:08.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:08.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:08.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:08.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:08.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:08.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:08.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:08.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:08.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:08.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:08.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:08.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:09.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:09.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:09.214 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:09.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:09.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:09.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:09 smithi152 ceph-mon[93456]: pgmap v830: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 7208/256404 objects degraded (2.811%); 17/256404 objects misplaced (0.007%); 8.0 KiB/s, 33 objects/s recovering 2024-08-17T17:25:09.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:09.363 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:09 smithi142 ceph-mon[103614]: pgmap v830: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 7208/256404 objects degraded (2.811%); 17/256404 objects misplaced (0.007%); 8.0 KiB/s, 33 objects/s recovering 2024-08-17T17:25:09.363 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:09 smithi142 ceph-mon[106729]: pgmap v830: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 7208/256404 objects degraded (2.811%); 17/256404 objects misplaced (0.007%); 8.0 KiB/s, 33 objects/s recovering 2024-08-17T17:25:09.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:09.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:09.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:09.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:09.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:09.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:09.824 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:09.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:25:09.835 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:09.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:09.880 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:09.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:10.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:10.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.173 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:10.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.274 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:10.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:10.416 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:10.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:25:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:25:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:10.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:10.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:10.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:10.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:10.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:10.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:11.019 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:11.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:11.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:11.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:11.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:11.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:11.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:11 smithi152 ceph-mon[93456]: pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 7208/249696 objects degraded (2.887%); 144 KiB/s, 86 objects/s recovering 2024-08-17T17:25:11.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7208/249696 objects degraded (2.887%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:11 smithi142 ceph-mon[103614]: pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 7208/249696 objects degraded (2.887%); 144 KiB/s, 86 objects/s recovering 2024-08-17T17:25:11.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7208/249696 objects degraded (2.887%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:11 smithi142 ceph-mon[106729]: pgmap v831: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 7208/249696 objects degraded (2.887%); 144 KiB/s, 86 objects/s recovering 2024-08-17T17:25:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7208/249696 objects degraded (2.887%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:11.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:11.421 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:11.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:11.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:11.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:11.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:11.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:11.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:11.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:25:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:25:11] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-08-17T17:25:11.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:11.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:12.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:12.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:12.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:12.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:12.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:12.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:12.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:12.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:12.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:12.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:12.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:12.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:12.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:12.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:12.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:12.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:12.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:12.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:13.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:13.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:13.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:13.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:13.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:13.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:13.297 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:13 smithi142 ceph-mon[106729]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7208/248964 objects degraded (2.895%); 144 KiB/s, 53 objects/s recovering 2024-08-17T17:25:13.297 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:13 smithi142 ceph-mon[103614]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7208/248964 objects degraded (2.895%); 144 KiB/s, 53 objects/s recovering 2024-08-17T17:25:13.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:13 smithi152 ceph-mon[93456]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7208/248964 objects degraded (2.895%); 144 KiB/s, 53 objects/s recovering 2024-08-17T17:25:13.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:13.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.461 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:13.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:13.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:13.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:13.761 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:13.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:13.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:14.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:14.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:14.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:14.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:14.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:14.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:14.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:14.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:14.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:14.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:14.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:14.640 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:14.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:14.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:14.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:14.810 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:14 smithi142 ceph-mon[106729]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 7208/240669 objects degraded (2.995%); 128 KiB/s, 50 objects/s recovering 2024-08-17T17:25:14.810 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:14.810 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:14 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:14.811 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:14 smithi142 ceph-mon[103614]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 7208/240669 objects degraded (2.995%); 128 KiB/s, 50 objects/s recovering 2024-08-17T17:25:14.811 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:14.811 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:14 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:14.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:14.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:14.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.035 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:14 smithi152 ceph-mon[93456]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 7208/240669 objects degraded (2.995%); 128 KiB/s, 50 objects/s recovering 2024-08-17T17:25:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:14 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:15.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:15.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:15.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:15.409 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:25:15.409 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:15.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:15.445 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:25:15.446 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:15.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:15.476 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:25:15.476 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:15.504 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:25:15.504 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:15.544 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:25:15.544 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:15.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:15.583 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:25:15.584 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:15.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:15.632 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:25:15.632 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:15.668 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:25:15.669 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:15.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:15.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:15.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:15.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:15.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:15.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:15.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:15.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/823788816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:15.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7208/240669 objects degraded (2.995%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:15.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:15.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:15.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:15.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/823788816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:15.972 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7208/240669 objects degraded (2.995%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:15.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:16.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/823788816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7208/240669 objects degraded (2.995%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:16.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:16.179 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:16.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:16.281 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:16.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:16.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:16.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:16.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:16.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:16.678 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:25:16.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:16.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:16.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:16.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:16.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:16.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:16.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:17.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:16 smithi152 ceph-mon[93456]: pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7208/237741 objects degraded (3.032%); 109 KiB/s, 64 objects/s recovering 2024-08-17T17:25:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3585046359' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:16 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:17.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:17.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:16 smithi142 ceph-mon[106729]: pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7208/237741 objects degraded (3.032%); 109 KiB/s, 64 objects/s recovering 2024-08-17T17:25:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3585046359' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:17.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:16 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:16 smithi142 ceph-mon[103614]: pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7208/237741 objects degraded (3.032%); 109 KiB/s, 64 objects/s recovering 2024-08-17T17:25:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3585046359' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:16 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:17.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:17.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:17.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:17.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:17.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:17.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:17.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:17.525 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:17.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:17.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:17.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:17.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:17.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:17.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:17.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:18.025 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:18.044 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:17 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:18.044 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:17 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:18.044 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:17 smithi142 ceph-mon[106729]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:25:18.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:17 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:18.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:17 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:18.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:17 smithi142 ceph-mon[103614]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:25:18.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:18.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:17 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:17 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:17 smithi152 ceph-mon[93456]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:25:18.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:18.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:18.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:18.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:18.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:18.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:18.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:18.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:18.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:18.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:18.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.745 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:18.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:18.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:18.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:18.938 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:18 smithi142 ceph-mon[106729]: pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 7208/234378 objects degraded (3.075%); 109 KiB/s, 64 objects/s recovering 2024-08-17T17:25:18.939 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:18 smithi142 ceph-mon[103614]: pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 7208/234378 objects degraded (3.075%); 109 KiB/s, 64 objects/s recovering 2024-08-17T17:25:18.976 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:19.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:18 smithi152 ceph-mon[93456]: pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 7208/234378 objects degraded (3.075%); 109 KiB/s, 64 objects/s recovering 2024-08-17T17:25:19.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:19.146 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:19.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.287 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:19.371 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:19.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:19.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:19.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:19.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:19.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:19.720 INFO:teuthology.orchestra.run.smithi142.stderr:set pool 10 pg_num to 26 2024-08-17T17:25:19.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:19.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:19.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:19.938 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:20.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:20.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:20.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3594504637' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:25:20.078 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3594504637' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:25:20.079 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3594504637' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:25:20.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:20.208 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:20.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:20.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:20.332 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:20.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:20.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:20.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:25:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:25:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:20.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:20.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:20.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[103614]: pgmap v837: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 7208/226464 objects degraded (3.183%); 0 B/s, 41 objects/s recovering 2024-08-17T17:25:20.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3594504637' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:25:20.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[103614]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:25:20.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7208/226464 objects degraded (3.183%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:20.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:20.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[106729]: pgmap v837: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 7208/226464 objects degraded (3.183%); 0 B/s, 41 objects/s recovering 2024-08-17T17:25:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3594504637' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:25:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[106729]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:25:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7208/226464 objects degraded (3.183%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:21.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:20 smithi152 ceph-mon[93456]: pgmap v837: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 7208/226464 objects degraded (3.183%); 0 B/s, 41 objects/s recovering 2024-08-17T17:25:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3594504637' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:25:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:20 smithi152 ceph-mon[93456]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:25:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7208/226464 objects degraded (3.183%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:21.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:21.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:21.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:21.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:21.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:21.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:25:21.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:21.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:21.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:21.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.644 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:21.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:21.736 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:25:21.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:21.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:21.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:21.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:21.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:22.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:25:22.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:22.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:25:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:25:21] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-08-17T17:25:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:25:22.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:22.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:22.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:22.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:22.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:22.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:22.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:22.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:22.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:22.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:22.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:22.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:22.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:22.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:22.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:22.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:22 smithi152 ceph-mon[93456]: pgmap v839: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7208/225660 objects degraded (3.194%); 0 B/s, 52 objects/s recovering 2024-08-17T17:25:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:25:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:22 smithi152 ceph-mon[93456]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:25:23.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:23.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:22 smithi142 ceph-mon[103614]: pgmap v839: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7208/225660 objects degraded (3.194%); 0 B/s, 52 objects/s recovering 2024-08-17T17:25:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:25:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:22 smithi142 ceph-mon[103614]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:25:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:22 smithi142 ceph-mon[106729]: pgmap v839: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7208/225660 objects degraded (3.194%); 0 B/s, 52 objects/s recovering 2024-08-17T17:25:23.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:25:23.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:22 smithi142 ceph-mon[106729]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:25:23.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:23.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:23.238 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:25:23.238 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:25:23.238 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:25:23.238 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:25:23.238 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.238 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73423 9.10 9:0e0af26c:::benchmark_data_smithi142_157452_object73422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62089705900001, 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4164945229999999, 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73423 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607311+0000", 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.240 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607312+0000", 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4619999999999999e-06 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607340+0000", 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7764999999999999e-05 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607353+0000", 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2471e-05 2024-08-17T17:25:23.241 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607364+0000", 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1392999999999999e-05 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607428+0000", 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3658000000000006e-05 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.242 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607439+0000", 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0777e-05 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013664+0000", 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4062256989999999 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013801+0000", 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000136594 2024-08-17T17:25:23.243 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.013929+0000", 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012783400000000001 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018572+0000", 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046436580000000002 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.244 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018584+0000", 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1574e-05 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018612+0000", 2024-08-17T17:25:23.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7915e-05 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022687+0000", 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.004075543 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022742+0000", 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4341e-05 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023806+0000", 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001063839 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73452 9.10 9:08329d80:::benchmark_data_smithi142_157452_object73451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62082829799999, 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4217852020000001, 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.248 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73452 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:25:23.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.15e-07 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607410+0000", 2024-08-17T17:25:23.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9464000000000001e-05 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607416+0000", 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2319999999999998e-06 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607420+0000", 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2649999999999998e-06 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607541+0000", 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012065699999999999 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607548+0000", 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1559999999999998e-06 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016317+0000", 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.408768746 2024-08-17T17:25:23.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016455+0000", 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013853699999999999 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.016639+0000", 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018399900000000001 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018680+0000", 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020403840000000001 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023861+0000", 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051812050000000004 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027763+0000", 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039021400000000001 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.028010+0000", 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024739199999999998 2024-08-17T17:25:23.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.028049+0000", 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9082000000000003e-05 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029165+0000", 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001115528 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.256 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73460 9.10 9:08db73f6:::benchmark_data_smithi142_157452_object73459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62076757199998, 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4227749750000001, 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.257 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73460 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:25:23.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607441+0000", 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6199999999999999e-07 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607459+0000", 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7660999999999999e-05 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607463+0000", 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7570000000000004e-06 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607468+0000", 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.34e-06 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607628+0000", 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016045799999999999 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607638+0000", 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8360000000000004e-06 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017660+0000", 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.410022291 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017755+0000", 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4372999999999999e-05 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.017851+0000", 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6256999999999997e-05 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020381+0000", 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025300259999999999 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027826+0000", 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0074454389999999999 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027858+0000", 2024-08-17T17:25:23.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1591000000000003e-05 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029231+0000", 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001372877 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.029290+0000", 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9687e-05 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030215+0000", 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00092502000000000003 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73489 9.10 9:0a567102:::benchmark_data_smithi142_157452_object73488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.620734959, 2024-08-17T17:25:23.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4276421909999999, 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73489 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607473+0000", 2024-08-17T17:25:23.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8599999999999999e-07 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607502+0000", 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8651000000000001e-05 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607511+0000", 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7800000000000006e-06 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607517+0000", 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8259999999999998e-06 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607995+0000", 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00047819600000000001 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608003+0000", 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4479999999999994e-06 2024-08-17T17:25:23.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018702+0000", 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4106989999999999 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018795+0000", 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2928000000000002e-05 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.018901+0000", 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105565 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.022627+0000", 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037258590000000002 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027930+0000", 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053030780000000001 2024-08-17T17:25:23.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033448+0000", 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005518406 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033579+0000", 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013044700000000001 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033635+0000", 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5965999999999999e-05 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035115+0000", 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001480755 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73517 9.10 9:09edc6df:::benchmark_data_smithi142_157452_object73516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.273 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62066449399998, 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.428877572, 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.274 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73517 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607544+0000", 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5499999999999998e-07 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607565+0000", 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0472e-05 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607569+0000", 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4059999999999998e-06 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607572+0000", 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3560000000000001e-06 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608033+0000", 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046030099999999998 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608037+0000", 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9820000000000002e-06 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019497+0000", 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.411460553 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019597+0000", 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9899e-05 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.019700+0000", 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000102571 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023871+0000", 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041716840000000002 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027970+0000", 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040992240000000003 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033519+0000", 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0055483060000000002 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035184+0000", 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0016652030000000001 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.035240+0000", 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5546e-05 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036421+0000", 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011816140000000001 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.281 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73531 9.10 9:0aa603eb:::benchmark_data_smithi142_157452_object73530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62062984400001, 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4341237490000001, 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73531 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607578+0000", 2024-08-17T17:25:23.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607579+0000", 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.03e-07 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607597+0000", 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8376000000000002e-05 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607601+0000", 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.963e-06 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607604+0000", 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8870000000000001e-06 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608070+0000", 2024-08-17T17:25:23.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046594300000000002 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608077+0000", 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2099999999999996e-06 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020422+0000", 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4123452620000001 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020558+0000", 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135914 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.020719+0000", 2024-08-17T17:25:23.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016086799999999999 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023882+0000", 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003162621 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036562+0000", 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012679953000000001 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040088+0000", 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035266020000000002 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040135+0000", 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6859e-05 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.040193+0000", 2024-08-17T17:25:23.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8128000000000003e-05 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041702+0000", 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0015088600000000001 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73543 9.10 9:0a994c9f:::benchmark_data_smithi142_157452_object73542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:25:23.291 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62058787699999, 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4357055299999999, 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73543 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607620+0000", 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607621+0000", 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3599999999999999e-07 2024-08-17T17:25:23.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607638+0000", 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.702e-05 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607644+0000", 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.037e-06 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607647+0000", 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0410000000000002e-06 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608107+0000", 2024-08-17T17:25:23.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045997900000000001 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608110+0000", 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3220000000000001e-06 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021537+0000", 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.413427129 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021655+0000", 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117779 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.021811+0000", 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015619200000000001 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026398+0000", 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045870349999999997 2024-08-17T17:25:23.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036616+0000", 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010218213 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041785+0000", 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051685489999999997 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041907+0000", 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012203500000000001 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041969+0000", 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2238999999999998e-05 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043326+0000", 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013566240000000001 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73564 9.10 9:0c98c0a8:::benchmark_data_smithi142_157452_object73563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.620556755, 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.441867732, 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.300 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73564 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607651+0000", 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607652+0000", 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1199999999999999e-07 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607715+0000", 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3386000000000006e-05 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607725+0000", 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8679999999999994e-06 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607730+0000", 2024-08-17T17:25:23.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.04e-06 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608144+0000", 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00041373799999999997 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608152+0000", 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4060000000000005e-06 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.023936+0000", 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.415784218 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.024030+0000", 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3970999999999999e-05 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.024123+0000", 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2508000000000003e-05 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027775+0000", 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003651774 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036678+0000", 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0089036219999999999 2024-08-17T17:25:23.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043422+0000", 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067443060000000003 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.048000+0000", 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045772770000000003 2024-08-17T17:25:23.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.048076+0000", 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6469000000000003e-05 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049519+0000", 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001442837 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.308 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73583 9.10 9:0b96fbe3:::benchmark_data_smithi142_157452_object73582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62047347800001, 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.442907784, 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.309 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73583 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:25:23.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607735+0000", 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8999999999999998e-07 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607756+0000", 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1569e-05 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607761+0000", 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5689999999999998e-06 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607764+0000", 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9529999999999999e-06 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608180+0000", 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000415847 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608183+0000", 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1930000000000002e-06 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025148+0000", 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4169648399999999 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025286+0000", 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138417 2024-08-17T17:25:23.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.025447+0000", 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000160946 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027868+0000", 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00242055 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.041848+0000", 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013980726000000001 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047049+0000", 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052002410000000004 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049578+0000", 2024-08-17T17:25:23.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025293820000000002 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.049642+0000", 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3837000000000006e-05 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050642+0000", 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001000424 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.316 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73678 9.10 9:08769dcd:::benchmark_data_smithi142_157452_object73677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62044000999998, 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.44418486, 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73678 2024-08-17T17:25:23.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607768+0000", 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.65e-07 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607782+0000", 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3886999999999999e-05 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607787+0000", 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6739999999999996e-06 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607790+0000", 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7970000000000001e-06 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608215+0000", 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00042531699999999998 2024-08-17T17:25:23.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608224+0000", 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9339999999999995e-06 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026442+0000", 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.418218049 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026597+0000", 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015451500000000001 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.026767+0000", 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170335 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.027979+0000", 2024-08-17T17:25:23.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012120810000000001 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050711+0000", 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.022731950000000001 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050763+0000", 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2281999999999999e-05 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050966+0000", 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020286300000000001 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051021+0000", 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5359000000000003e-05 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051953+0000", 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00093155199999999999 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.324 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73695 9.10 9:0bc067c9:::benchmark_data_smithi142_157452_object73694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62041476500002, 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4639758309999999, 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.325 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73695 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607793+0000", 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607794+0000", 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.96e-07 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607808+0000", 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4695000000000001e-05 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607813+0000", 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5129999999999998e-06 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607816+0000", 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7240000000000001e-06 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608255+0000", 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00043993199999999997 2024-08-17T17:25:23.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608260+0000", 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9010000000000002e-06 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030261+0000", 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4220003640000001 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030373+0000", 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011251000000000001 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.030485+0000", 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111542 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.033537+0000", 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030526939999999999 2024-08-17T17:25:23.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.050876+0000", 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017338675000000001 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.051992+0000", 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011160079999999999 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.070213+0000", 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018221029 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.070274+0000", 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0436999999999998e-05 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071769+0000", 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014955109999999999 2024-08-17T17:25:23.332 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73736 9.10 9:0f48bdcd:::benchmark_data_smithi142_157452_object73735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62038889500002, 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4653223989999999, 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.333 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73736 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:25:23.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607819+0000", 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4999999999999999e-07 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607834+0000", 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4226000000000001e-05 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607841+0000", 2024-08-17T17:25:23.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4939999999999997e-06 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607847+0000", 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3850000000000001e-06 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608291+0000", 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000444864 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608300+0000", 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6119999999999999e-06 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032028+0000", 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4237279540000001 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032170+0000", 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014252899999999999 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.032268+0000", 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7950999999999998e-05 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036506+0000", 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042373360000000004 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054725+0000", 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018219227000000001 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055000+0000", 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00027527500000000001 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071822+0000", 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016821922999999999 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.071885+0000", 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2649000000000003e-05 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073142+0000", 2024-08-17T17:25:23.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012567240000000001 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73777 9.10 9:08e4faeb:::benchmark_data_smithi142_157452_object73776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62035687000002, 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4667512089999999, 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.341 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73777 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607851+0000", 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4900000000000002e-07 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607866+0000", 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4863e-05 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607870+0000", 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0010000000000003e-06 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607876+0000", 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8320000000000002e-06 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608327+0000", 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045121099999999997 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608331+0000", 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9169999999999999e-06 2024-08-17T17:25:23.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036717+0000", 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4283860289999999 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.036865+0000", 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014740099999999999 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.037067+0000", 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020239499999999999 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043346+0000", 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062784049999999999 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054765+0000", 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011419155 2024-08-17T17:25:23.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073198+0000", 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018433105000000002 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073365+0000", 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016761400000000001 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073424+0000", 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.876e-05 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074602+0000", 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001178272 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.349 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73834 9.10 9:0efa507b:::benchmark_data_smithi142_157452_object73833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62031420800002, 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.467875166, 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73834 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.350 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607894+0000", 2024-08-17T17:25:23.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0999999999999999e-07 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607911+0000", 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7147999999999999e-05 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607916+0000", 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9400000000000001e-06 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607921+0000", 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1590000000000003e-06 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608364+0000", 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00044320200000000002 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608372+0000", 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8730000000000007e-06 2024-08-17T17:25:23.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038589+0000", 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.430217689 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038737+0000", 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014733899999999999 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.038912+0000", 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017523699999999999 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043352+0000", 2024-08-17T17:25:23.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044401780000000004 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.054813+0000", 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011461135000000001 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073240+0000", 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018426973999999999 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074657+0000", 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014169829999999999 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.074711+0000", 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4185999999999999e-05 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075769+0000", 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010577130000000001 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73885 9.10 9:08993fba:::benchmark_data_smithi142_157452_object73884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62028338099998, 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.468984944, 2024-08-17T17:25:23.358 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73885 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:25:23.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607925+0000", 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5499999999999999e-07 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607940+0000", 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4739999999999999e-05 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607944+0000", 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5630000000000002e-06 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607947+0000", 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7439999999999999e-06 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608397+0000", 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045033600000000002 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608401+0000", 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5870000000000001e-06 2024-08-17T17:25:23.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043476+0000", 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4350746729999999 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043632+0000", 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000156176 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.043817+0000", 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000184754 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047067+0000", 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032502579999999998 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055046+0000", 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079788489999999997 2024-08-17T17:25:23.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073280+0000", 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018234017000000002 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075815+0000", 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025350659999999999 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.075873+0000", 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8557000000000002e-05 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.076910+0000", 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010363690000000001 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:73890 9.10 9:0c2319f1:::benchmark_data_smithi142_157452_object73889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e164)", 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 426.62025751900001, 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5039437520000001, 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 73890 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.367 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607951+0000", 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4900000000000002e-07 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607965+0000", 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3899e-05 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607969+0000", 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5879999999999999e-06 2024-08-17T17:25:23.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.607972+0000", 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7520000000000001e-06 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608429+0000", 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045679500000000002 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:16.608437+0000", 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3999999999999992e-06 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045264+0000", 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.436826288 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045402+0000", 2024-08-17T17:25:23.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138221 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,4,7", 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.045554+0000", 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000152585 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.047073+0000", 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001518637 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.055084+0000", 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080111230000000002 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.073321+0000", 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018236588000000001 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.077006+0000", 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0036852550000000001 2024-08-17T17:25:23.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.077063+0000", 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6827999999999999e-05 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:18:18.111894+0000", 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034831543999999999 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144787 10.2 10:46e7094a:::benchmark_data_smithi142_192453_object19039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.798321+0000", 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 47.429886969000002, 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3466595450000001, 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:23.377 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144787 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.798321+0000", 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.798321+0000", 2024-08-17T17:25:23.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.798323+0000", 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8780000000000001e-06 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.798329+0000", 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9479999999999996e-06 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.798349+0000", 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0117000000000001e-05 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.798358+0000", 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2099999999999999e-06 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.803369+0000", 2024-08-17T17:25:23.381 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0050110789999999999 2024-08-17T17:25:23.381 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.381 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.381 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:23.381 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.803718+0000", 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00034897299999999998 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.804117+0000", 2024-08-17T17:25:23.382 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00039826299999999999 2024-08-17T17:25:23.383 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.383 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.811499+0000", 2024-08-17T17:25:23.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0073827550000000004 2024-08-17T17:25:23.383 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.815009+0000", 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035093120000000001 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.900833+0000", 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.085824011000000006 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.900884+0000", 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1279999999999997e-05 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.116838+0000", 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2159545599999999 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.116994+0000", 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015543499999999999 2024-08-17T17:25:23.385 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.144799+0000", 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027805051000000001 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.144981+0000", 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018167300000000001 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.386 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144789 10.2 10:47ee9fe0:::benchmark_data_smithi142_192453_object19041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.802694+0000", 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 47.425513664, 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3581519010000001, 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144789 2024-08-17T17:25:23.387 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.802694+0000", 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.802694+0000", 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.388 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.802696+0000", 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8789999999999999e-06 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.802702+0000", 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7930000000000003e-06 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.802717+0000", 2024-08-17T17:25:23.389 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.535e-05 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.802725+0000", 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.345e-06 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.807761+0000", 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0050360240000000001 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.390 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.807956+0000", 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019526999999999999 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.808143+0000", 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000186973 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.811627+0000", 2024-08-17T17:25:23.391 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003483917 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.815160+0000", 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035328619999999999 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.903117+0000", 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.087956812999999995 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.392 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.903128+0000", 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1742e-05 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.117395+0000", 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2142661480000001 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:23.393 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.117527+0000", 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132409 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.160698+0000", 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.043171039000000001 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.160846+0000", 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014833700000000001 2024-08-17T17:25:23.394 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144804 10.2 10:403cbbe1:::benchmark_data_smithi142_192453_object19056:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.829990+0000", 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 47.398218241000002, 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3462824609999999, 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:23.395 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144804 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.829990+0000", 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.829990+0000", 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.829993+0000", 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9869999999999999e-06 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.830000+0000", 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0600000000000002e-06 2024-08-17T17:25:23.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.398 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.398 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.398 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:23.398 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.830023+0000", 2024-08-17T17:25:23.398 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2702e-05 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.830032+0000", 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7580000000000004e-06 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.830149+0000", 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116941 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.399 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.830322+0000", 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017288399999999999 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.830514+0000", 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019207100000000001 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.837262+0000", 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067478110000000003 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.837464+0000", 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020197699999999999 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.903156+0000", 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.065691891000000002 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.903163+0000", 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1910000000000003e-06 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.117584+0000", 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2144206610000001 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.117704+0000", 2024-08-17T17:25:23.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012013 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.176166+0000", 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.058462457000000002 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.176272+0000", 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010594000000000001 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144810 10.2 10:43645d91:::benchmark_data_smithi142_192453_object19062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.838049+0000", 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 47.390158941000003, 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3387129579999999, 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144810 2024-08-17T17:25:23.404 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838049+0000", 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838049+0000", 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:23.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838051+0000", 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5269999999999999e-06 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838057+0000", 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.55e-06 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838076+0000", 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8493000000000001e-05 2024-08-17T17:25:23.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838085+0000", 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1309999999999995e-06 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838143+0000", 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7988000000000001e-05 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:23.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838332+0000", 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000189335 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.838517+0000", 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000184384 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.845121+0000", 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0066043009999999999 2024-08-17T17:25:23.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.854638+0000", 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0095169369999999996 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.903191+0000", 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.048552922999999998 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.903198+0000", 2024-08-17T17:25:23.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2030000000000003e-06 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.117744+0000", 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2145460450000001 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.117853+0000", 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108842 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.410 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.176676+0000", 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.058822799000000002 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.176762+0000", 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6500000000000002e-05 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:25:23.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:23.433 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:23.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:23.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:23.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:23.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:23.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:23.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:23.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:23 smithi142 ceph-mon[106729]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:25:23.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:23 smithi142 ceph-mon[103614]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:25:23.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:23.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:23.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:24.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:24.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:24.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:23 smithi152 ceph-mon[93456]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:25:24.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:24.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:24.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:24.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:24.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:24.419 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:24.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:24.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:24.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:24.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:24.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:25:24.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:24.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:24.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:24.966 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:24 smithi142 ceph-mon[106729]: 10.3 scrub starts 2024-08-17T17:25:24.966 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:24 smithi142 ceph-mon[106729]: pgmap v842: 99 pgs: 1 activating, 3 peering, 1 remapped+peering, 8 unknown, 2 active+recovering+undersized+remapped, 1 active, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 4827/205209 objects degraded (2.352%); 0 B/s, 59 objects/s recovering 2024-08-17T17:25:24.966 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:24 smithi142 ceph-mon[106729]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:25:24.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:24 smithi142 ceph-mon[103614]: 10.3 scrub starts 2024-08-17T17:25:24.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:24 smithi142 ceph-mon[103614]: pgmap v842: 99 pgs: 1 activating, 3 peering, 1 remapped+peering, 8 unknown, 2 active+recovering+undersized+remapped, 1 active, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 4827/205209 objects degraded (2.352%); 0 B/s, 59 objects/s recovering 2024-08-17T17:25:24.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:24 smithi142 ceph-mon[103614]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:25:24.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:25.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:24 smithi152 ceph-mon[93456]: 10.3 scrub starts 2024-08-17T17:25:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:24 smithi152 ceph-mon[93456]: pgmap v842: 99 pgs: 1 activating, 3 peering, 1 remapped+peering, 8 unknown, 2 active+recovering+undersized+remapped, 1 active, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 4827/205209 objects degraded (2.352%); 0 B/s, 59 objects/s recovering 2024-08-17T17:25:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:24 smithi152 ceph-mon[93456]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:25:25.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:25.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:25.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:25.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:25.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:25.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:25.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:25.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:25.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:25.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:25.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:25.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:25.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:25.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:25.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:26.014 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:25 smithi142 ceph-mon[106729]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:25:26.014 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4827/205209 objects degraded (2.352%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:26.014 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:25 smithi142 ceph-mon[103614]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:25:26.015 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4827/205209 objects degraded (2.352%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:25 smithi152 ceph-mon[93456]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:25:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4827/205209 objects degraded (2.352%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:26.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:26.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:26.219 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:26.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:26.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:26.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:26.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:26.525 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:25:26.526 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:25:26.526 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-17T17:25:26.526 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:25:26.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:25:26.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:26.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:26.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:26.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:26.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:26.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:26.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:26 smithi152 ceph-mon[93456]: pgmap v845: 99 pgs: 2 active+recovery_wait+degraded, 1 activating, 3 peering, 2 remapped+peering, 4 unknown, 1 active+recovering+undersized+remapped, 1 active, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5006/203439 objects degraded (2.461%); 0 B/s, 67 objects/s recovering 2024-08-17T17:25:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:26.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:26 smithi152 systemd[1]: Stopping Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:25:26.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:26.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:26.989 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:27.077 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:27.088 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:26 smithi142 ceph-mon[106729]: pgmap v845: 99 pgs: 2 active+recovery_wait+degraded, 1 activating, 3 peering, 2 remapped+peering, 4 unknown, 1 active+recovering+undersized+remapped, 1 active, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5006/203439 objects degraded (2.461%); 0 B/s, 67 objects/s recovering 2024-08-17T17:25:27.088 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:27.089 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:26 smithi142 ceph-mon[103614]: pgmap v845: 99 pgs: 2 active+recovery_wait+degraded, 1 activating, 3 peering, 2 remapped+peering, 4 unknown, 1 active+recovering+undersized+remapped, 1 active, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5006/203439 objects degraded (2.461%); 0 B/s, 67 objects/s recovering 2024-08-17T17:25:27.089 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:27.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:27.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:27.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:27.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:27.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[59204]: 2024-08-17T17:25:26.988+0000 7fabfd7c7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:25:27.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[59204]: 2024-08-17T17:25:26.988+0000 7fabfd7c7640 -1 osd.7 260 *** Got signal Terminated *** 2024-08-17T17:25:27.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:26 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[59204]: 2024-08-17T17:25:26.988+0000 7fabfd7c7640 -1 osd.7 260 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:25:27.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:27.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:27.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:27.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:27.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:27.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:27.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:27.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:27.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:27.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:27.897 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:27 smithi142 ceph-mon[106729]: 10.13 scrub starts 2024-08-17T17:25:27.897 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:27 smithi142 ceph-mon[106729]: osd.7 marked itself down and dead 2024-08-17T17:25:27.898 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:27 smithi142 ceph-mon[103614]: 10.13 scrub starts 2024-08-17T17:25:27.898 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:27 smithi142 ceph-mon[103614]: osd.7 marked itself down and dead 2024-08-17T17:25:27.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:27.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:28.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:28.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:28.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:27 smithi152 ceph-mon[93456]: 10.13 scrub starts 2024-08-17T17:25:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:27 smithi152 ceph-mon[93456]: osd.7 marked itself down and dead 2024-08-17T17:25:28.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:28.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:28.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:28.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:28.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:28.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:28.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:28.440 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:28.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:28.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:28.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:28.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:28.665 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:28.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:28.718 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:28 smithi152 podman[145665]: 2024-08-17 17:25:28.405194457 +0000 UTC m=+1.551632122 container died 0a45f0507de1ea28ecfe5db07bda616a936a11f4cd44f3e364358e7eb2ed7300 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4) 2024-08-17T17:25:28.775 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:25:28.789 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:28.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:28.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:28.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:29.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:29.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:28 smithi152 podman[145665]: 2024-08-17 17:25:28.920684112 +0000 UTC m=+2.067121785 container cleanup 0a45f0507de1ea28ecfe5db07bda616a936a11f4cd44f3e364358e7eb2ed7300 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:25:29.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:28 smithi152 bash[145665]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:25:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:28 smithi152 ceph-mon[93456]: pgmap v846: 99 pgs: 6 active+recovery_wait+degraded, 1 activating, 1 peering, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 6164/213906 objects degraded (2.882%); 176 KiB/s, 49 objects/s recovering 2024-08-17T17:25:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:28 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:25:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:28 smithi152 ceph-mon[93456]: osdmap e261: 8 total, 7 up, 8 in 2024-08-17T17:25:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/440374338' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:29.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:29.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[106729]: pgmap v846: 99 pgs: 6 active+recovery_wait+degraded, 1 activating, 1 peering, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 6164/213906 objects degraded (2.882%); 176 KiB/s, 49 objects/s recovering 2024-08-17T17:25:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:25:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[106729]: osdmap e261: 8 total, 7 up, 8 in 2024-08-17T17:25:29.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/440374338' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[103614]: pgmap v846: 99 pgs: 6 active+recovery_wait+degraded, 1 activating, 1 peering, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 6164/213906 objects degraded (2.882%); 176 KiB/s, 49 objects/s recovering 2024-08-17T17:25:29.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:25:29.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[103614]: osdmap e261: 8 total, 7 up, 8 in 2024-08-17T17:25:29.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/440374338' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:29.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:29.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:29.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:29.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:29.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:29.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:29.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:29.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:29.565 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:29 smithi152 podman[145751]: 2024-08-17 17:25:29.231114505 +0000 UTC m=+0.786845707 container remove 0a45f0507de1ea28ecfe5db07bda616a936a11f4cd44f3e364358e7eb2ed7300 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-08-17T17:25:29.569 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:29.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:29.692 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:29.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:29.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:29 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/440374338' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:29 smithi152 ceph-mon[93456]: osdmap e262: 8 total, 7 up, 8 in 2024-08-17T17:25:29.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:29 smithi152 podman[145878]: 2024-08-17 17:25:29.565471572 +0000 UTC m=+0.113310615 container create ed0caceef15a2e3738eaaa86cdfde8eb0bcc4596fe3560b610e358297b3bab7b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4) 2024-08-17T17:25:29.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:29 smithi152 podman[145878]: 2024-08-17 17:25:29.471473559 +0000 UTC m=+0.019312596 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:25:29.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:29.896 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:25:29.896 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:29.896 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:29 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:29.896 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/440374338' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:29.897 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:29 smithi142 ceph-mon[106729]: osdmap e262: 8 total, 7 up, 8 in 2024-08-17T17:25:29.897 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:29 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:29.897 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/440374338' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:29.897 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:29 smithi142 ceph-mon[103614]: osdmap e262: 8 total, 7 up, 8 in 2024-08-17T17:25:29.926 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:25:29.926 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:29.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:29.947 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:25:29.947 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:29.954 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:29.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:29.968 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:25:29.968 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:29.994 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:25:29.994 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:30.020 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:25:30.020 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:30.044 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:25:30.044 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:30.044 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:25:30.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:30.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:30.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:30.119 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:29 smithi152 podman[145878]: 2024-08-17 17:25:29.860337341 +0000 UTC m=+0.408176387 container init ed0caceef15a2e3738eaaa86cdfde8eb0bcc4596fe3560b610e358297b3bab7b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0) 2024-08-17T17:25:30.119 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:29 smithi152 podman[145878]: 2024-08-17 17:25:29.866779258 +0000 UTC m=+0.414618297 container start ed0caceef15a2e3738eaaa86cdfde8eb0bcc4596fe3560b610e358297b3bab7b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T17:25:30.119 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:29 smithi152 podman[145878]: 2024-08-17 17:25:29.968938703 +0000 UTC m=+0.516777748 container attach ed0caceef15a2e3738eaaa86cdfde8eb0bcc4596fe3560b610e358297b3bab7b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-08-17T17:25:30.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:30.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:30.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:30.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:30.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:30.491 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:25:30 smithi152 podman[145878]: 2024-08-17 17:25:30.119112383 +0000 UTC m=+0.666951427 container died ed0caceef15a2e3738eaaa86cdfde8eb0bcc4596fe3560b610e358297b3bab7b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T17:25:30.491 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:25:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:25:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:30.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:30.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:30.660 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:30.695 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:30.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:30.797 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:30.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:30.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:30.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:25:30.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:31.020 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:25:31.020 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:25:31.048 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[106729]: pgmap v849: 99 pgs: 1 stale+active+recovering+undersized+remapped, 9 active+undersized, 6 active+recovery_wait+degraded, 5 peering, 3 remapped+peering, 8 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 60 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 2.66k op/s; 10258/206394 objects degraded (4.970%); 116 KiB/s, 23 objects/s recovering 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[106729]: osdmap e263: 8 total, 7 up, 8 in 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/226083871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10258/206394 objects degraded (4.970%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[103614]: pgmap v849: 99 pgs: 1 stale+active+recovering+undersized+remapped, 9 active+undersized, 6 active+recovery_wait+degraded, 5 peering, 3 remapped+peering, 8 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 60 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 2.66k op/s; 10258/206394 objects degraded (4.970%); 116 KiB/s, 23 objects/s recovering 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[103614]: osdmap e263: 8 total, 7 up, 8 in 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/226083871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:31.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10258/206394 objects degraded (4.970%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:31.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:30 smithi152 ceph-mon[93456]: pgmap v849: 99 pgs: 1 stale+active+recovering+undersized+remapped, 9 active+undersized, 6 active+recovery_wait+degraded, 5 peering, 3 remapped+peering, 8 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 60 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 2.66k op/s; 10258/206394 objects degraded (4.970%); 116 KiB/s, 23 objects/s recovering 2024-08-17T17:25:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:30 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:25:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:30 smithi152 ceph-mon[93456]: osdmap e263: 8 total, 7 up, 8 in 2024-08-17T17:25:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/226083871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10258/206394 objects degraded (4.970%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:31.098 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:31.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:31.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:31.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:31.332 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:31.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:31.436 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:31.436 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:31.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:31.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:31.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:31.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:31.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:31.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:25:31.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:31.762 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:25:31.763 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:25:31.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:31.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:31.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:31.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:31.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:32.025 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:32.107 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:25:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:25:31] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-08-17T17:25:32.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:32.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:32.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:32.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:32.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:32.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:32.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:32.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:32.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:32.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:32.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:25:32.674 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:32.679 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:32.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:32.828 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:32 smithi142 ceph-mon[103614]: pgmap v851: 99 pgs: 1 stale+active+recovering+undersized+remapped, 12 active+undersized, 6 active+recovery_wait+degraded, 5 peering, 3 remapped+peering, 6 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 59 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 2.51k op/s; 10258/205845 objects degraded (4.983%); 112 KiB/s, 2 objects/s recovering 2024-08-17T17:25:32.828 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:32 smithi142 ceph-mon[106729]: pgmap v851: 99 pgs: 1 stale+active+recovering+undersized+remapped, 12 active+undersized, 6 active+recovery_wait+degraded, 5 peering, 3 remapped+peering, 6 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 59 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 2.51k op/s; 10258/205845 objects degraded (4.983%); 112 KiB/s, 2 objects/s recovering 2024-08-17T17:25:32.837 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:32.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:32.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:33.028 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:33.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:32 smithi152 ceph-mon[93456]: pgmap v851: 99 pgs: 1 stale+active+recovering+undersized+remapped, 12 active+undersized, 6 active+recovery_wait+degraded, 5 peering, 3 remapped+peering, 6 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 59 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 2.51k op/s; 10258/205845 objects degraded (4.983%); 112 KiB/s, 2 objects/s recovering 2024-08-17T17:25:33.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:33.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:33.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:33.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:33.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:33.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:33.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:33.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:33.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:33.599 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:33.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:33.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:33.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:33.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:33.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:33.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:33.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:34.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:34.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:33 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-08-17T17:25:34.125 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:33 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-08-17T17:25:34.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:34.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:34.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:34.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:33 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-08-17T17:25:34.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:34.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:34.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:34.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:34.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:34.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:34.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:34.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:34.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:34.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:34.928 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:34.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:34.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:35.003 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:34 smithi142 ceph-mon[103614]: pgmap v852: 99 pgs: 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 18296/200085 objects degraded (9.144%); 2/200085 objects misplaced (0.001%); 1.8 MiB/s, 261 objects/s recovering 2024-08-17T17:25:35.004 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:34 smithi142 ceph-mon[106729]: pgmap v852: 99 pgs: 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 18296/200085 objects degraded (9.144%); 2/200085 objects misplaced (0.001%); 1.8 MiB/s, 261 objects/s recovering 2024-08-17T17:25:35.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:35.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:35.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:35.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:35.225 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:35.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:35.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:34 smithi152 ceph-mon[93456]: pgmap v852: 99 pgs: 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 2 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 18296/200085 objects degraded (9.144%); 2/200085 objects misplaced (0.001%); 1.8 MiB/s, 261 objects/s recovering 2024-08-17T17:25:35.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:35.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:35.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:35.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:35.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:35.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:35.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:35.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:35.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:35.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:35.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:35.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:35.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:36.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:36.021 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:25:36.021 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:25:36.022 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:25:36.022 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.647065 2024-08-17T17:25:36.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 0.6470646730819167 2024-08-17T17:25:36.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:36.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18296/200085 objects degraded (9.144%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:36.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18296/200085 objects degraded (9.144%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:36.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:36.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:36.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:36.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:36.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18296/200085 objects degraded (9.144%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:25:36.416 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:36.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:36.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:36.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:36.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:36.830 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:36.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:36.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:36.880 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:25:36.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:37.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:37.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:36 smithi142 ceph-mon[103614]: pgmap v853: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 22203/197145 objects degraded (11.262%); 2/197145 objects misplaced (0.001%); 3.6 MiB/s, 257 objects/s recovering 2024-08-17T17:25:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1431525288' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:36 smithi142 ceph-mon[106729]: pgmap v853: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 22203/197145 objects degraded (11.262%); 2/197145 objects misplaced (0.001%); 3.6 MiB/s, 257 objects/s recovering 2024-08-17T17:25:37.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1431525288' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:37.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:37.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:37.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:36 smithi152 ceph-mon[93456]: pgmap v853: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 22203/197145 objects degraded (11.262%); 2/197145 objects misplaced (0.001%); 3.6 MiB/s, 257 objects/s recovering 2024-08-17T17:25:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1431525288' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:37.390 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:37.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:37.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:37.557 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:37.557 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:37.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:37.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:37.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:37.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:37.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:37.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:37.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:38.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:38.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:37 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1431525288' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:38.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:37 smithi142 ceph-mon[106729]: osdmap e264: 8 total, 7 up, 8 in 2024-08-17T17:25:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:37 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1431525288' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:37 smithi142 ceph-mon[103614]: osdmap e264: 8 total, 7 up, 8 in 2024-08-17T17:25:38.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:38.223 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:38.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:37 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:38.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1431525288' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:38.326 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:37 smithi152 ceph-mon[93456]: osdmap e264: 8 total, 7 up, 8 in 2024-08-17T17:25:38.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:38.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:38.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:38.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:38.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:38.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:38.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:38.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:38.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:38.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:38.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:38.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:38.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:39.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:39.040 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:39.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:39.141 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:39.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:39.224 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:38 smithi142 ceph-mon[103614]: pgmap v855: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 21626/192273 objects degraded (11.248%); 2/192273 objects misplaced (0.001%); 5.7 MiB/s, 284 objects/s recovering 2024-08-17T17:25:39.225 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:38 smithi142 ceph-mon[106729]: pgmap v855: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 21626/192273 objects degraded (11.248%); 2/192273 objects misplaced (0.001%); 5.7 MiB/s, 284 objects/s recovering 2024-08-17T17:25:39.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:39.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:39.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:39.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:38 smithi152 ceph-mon[93456]: pgmap v855: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 21626/192273 objects degraded (11.248%); 2/192273 objects misplaced (0.001%); 5.7 MiB/s, 284 objects/s recovering 2024-08-17T17:25:39.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:39.427 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:39.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:39.429 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:39.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:39.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:39.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:39.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:39.430 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:25:39.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:39.431 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:25:39.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:39.558 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:39.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:39.660 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:39.694 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:39.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:39.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:39.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:39.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:40.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:40.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:40.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3104200478' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:40.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3104200478' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:40.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:40.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3104200478' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:40.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:40.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:40.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:40.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:40.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:40.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:40.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:40.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:25:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:25:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:40.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:40.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:40.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:40.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:25:40.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:40.828 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:40.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:40.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:41.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:40 smithi142 ceph-mon[103614]: pgmap v856: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 21045/183945 objects degraded (11.441%); 2/183945 objects misplaced (0.001%); 7.2 MiB/s, 273 objects/s recovering 2024-08-17T17:25:41.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:40 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 21045/183945 objects degraded (11.441%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:41.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:40 smithi142 ceph-mon[106729]: pgmap v856: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 21045/183945 objects degraded (11.441%); 2/183945 objects misplaced (0.001%); 7.2 MiB/s, 273 objects/s recovering 2024-08-17T17:25:41.050 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:40 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 21045/183945 objects degraded (11.441%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:41.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:41.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:41.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:41.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:40 smithi152 ceph-mon[93456]: pgmap v856: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 21045/183945 objects degraded (11.441%); 2/183945 objects misplaced (0.001%); 7.2 MiB/s, 273 objects/s recovering 2024-08-17T17:25:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:40 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 21045/183945 objects degraded (11.441%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:41.324 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:25:41.324 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:25:41.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:41.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:41.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:41.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:41.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:41.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:41.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:41.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:41.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:25:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:25:41] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-08-17T17:25:41.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:41.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:42.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:42.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:42.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:42.173 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:42.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:42.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:25:42.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:42.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:42.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:42.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:42.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:42.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:42.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:42 smithi142 ceph-mon[103614]: pgmap v857: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 21045/183105 objects degraded (11.493%); 2/183105 objects misplaced (0.001%); 7.0 MiB/s, 267 objects/s recovering 2024-08-17T17:25:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:42.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:42.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:42 smithi142 ceph-mon[106729]: pgmap v857: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 21045/183105 objects degraded (11.493%); 2/183105 objects misplaced (0.001%); 7.0 MiB/s, 267 objects/s recovering 2024-08-17T17:25:42.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:42.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:25:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:42 smithi152 ceph-mon[93456]: pgmap v857: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 21045/183105 objects degraded (11.493%); 2/183105 objects misplaced (0.001%); 7.0 MiB/s, 267 objects/s recovering 2024-08-17T17:25:42.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:42.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:42.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:42.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:43.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:43.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:43.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:43.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:43.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:43.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:43.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:43.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:43.389 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:43.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:43.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:43.562 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:43.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:43.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:43.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:43.848 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:43.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:43.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:43.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:44.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:44.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:44.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:44.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:44.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:44.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:44.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:44.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:44.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:44.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:44.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:44.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:44.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:44.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:44.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:44 smithi142 ceph-mon[106729]: pgmap v858: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 20466/173598 objects degraded (11.789%); 2/173598 objects misplaced (0.001%); 7.7 MiB/s, 139 objects/s recovering 2024-08-17T17:25:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:44 smithi142 ceph-mon[103614]: pgmap v858: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 20466/173598 objects degraded (11.789%); 2/173598 objects misplaced (0.001%); 7.7 MiB/s, 139 objects/s recovering 2024-08-17T17:25:44.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:44.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:44.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:45.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:44 smithi152 ceph-mon[93456]: pgmap v858: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 20466/173598 objects degraded (11.789%); 2/173598 objects misplaced (0.001%); 7.7 MiB/s, 139 objects/s recovering 2024-08-17T17:25:45.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:45.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:45.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:45.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:45.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:45.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:45.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:45.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:45.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:45.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:45.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:45.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:45.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:45.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:45.839 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:45.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:45.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1649479770' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167} : dispatch 2024-08-17T17:25:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:45 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167} : dispatch 2024-08-17T17:25:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 20466/173598 objects degraded (11.789%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1649479770' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167} : dispatch 2024-08-17T17:25:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:45 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167} : dispatch 2024-08-17T17:25:45.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 20466/173598 objects degraded (11.789%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:45.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:45.997 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:46.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1649479770' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167} : dispatch 2024-08-17T17:25:46.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:45 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167} : dispatch 2024-08-17T17:25:46.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 20466/173598 objects degraded (11.789%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:46.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:46.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:46.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:46.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:46.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:46.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:46.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:46.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:46.595 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:46.617 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:46.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:46.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:46.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:46.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:46.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:46.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:47.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:47.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:46 smithi152 ceph-mon[93456]: pgmap v859: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 19905/170325 objects degraded (11.686%); 2/170325 objects misplaced (0.001%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:25:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:46 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167}]': finished 2024-08-17T17:25:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:46 smithi152 ceph-mon[93456]: osdmap e265: 8 total, 7 up, 8 in 2024-08-17T17:25:47.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:46 smithi142 ceph-mon[106729]: pgmap v859: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 19905/170325 objects degraded (11.686%); 2/170325 objects misplaced (0.001%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:25:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:46 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167}]': finished 2024-08-17T17:25:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:46 smithi142 ceph-mon[106729]: osdmap e265: 8 total, 7 up, 8 in 2024-08-17T17:25:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:46 smithi142 ceph-mon[103614]: pgmap v859: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 19905/170325 objects degraded (11.686%); 2/170325 objects misplaced (0.001%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:25:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:46 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.6470646730819167}]': finished 2024-08-17T17:25:47.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:46 smithi142 ceph-mon[103614]: osdmap e265: 8 total, 7 up, 8 in 2024-08-17T17:25:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:47.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:47.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:47.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:47.375 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:47.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:47.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:47.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:47.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:47.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:47.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:47.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:47.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:47.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:47.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:47.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:48.025 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:48.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:47 smithi152 ceph-mon[93456]: osdmap e266: 8 total, 7 up, 8 in 2024-08-17T17:25:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:47 smithi142 ceph-mon[106729]: osdmap e266: 8 total, 7 up, 8 in 2024-08-17T17:25:48.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:47 smithi142 ceph-mon[103614]: osdmap e266: 8 total, 7 up, 8 in 2024-08-17T17:25:48.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:48.196 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:48.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:48.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:48.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:48.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:48.496 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.5 primary-affinity to 0.647065 (8424062) 2024-08-17T17:25:48.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:48.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:48.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:48.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:48.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:48.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:48.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:48 smithi142 ceph-mon[106729]: pgmap v862: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 19442/166161 objects degraded (11.701%); 2/166161 objects misplaced (0.001%); 7.4 MiB/s, 117 objects/s recovering 2024-08-17T17:25:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:48 smithi142 ceph-mon[103614]: pgmap v862: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 19442/166161 objects degraded (11.701%); 2/166161 objects misplaced (0.001%); 7.4 MiB/s, 117 objects/s recovering 2024-08-17T17:25:48.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:49.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:49.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:48 smithi152 ceph-mon[93456]: pgmap v862: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 19442/166161 objects degraded (11.701%); 2/166161 objects misplaced (0.001%); 7.4 MiB/s, 117 objects/s recovering 2024-08-17T17:25:49.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:49.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:49.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.276 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:49.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:49.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:49.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:49.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:49.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:49.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:49.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:49.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:49.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:49.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:49.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:49.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:50.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:50.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:50.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:50.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:50.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:50.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:50.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:50.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:50.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:50.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:50.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:50.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:25:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:25:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:50.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:50.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:50.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:50.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:50.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:50.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:50.975 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:50 smithi152 ceph-mon[93456]: pgmap v863: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 19191/159882 objects degraded (12.003%); 2/159882 objects misplaced (0.001%); 11 MiB/s, 177 objects/s recovering 2024-08-17T17:25:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 19191/159882 objects degraded (12.003%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:51.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:51.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:51.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:50 smithi142 ceph-mon[106729]: pgmap v863: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 19191/159882 objects degraded (12.003%); 2/159882 objects misplaced (0.001%); 11 MiB/s, 177 objects/s recovering 2024-08-17T17:25:51.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 19191/159882 objects degraded (12.003%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:50 smithi142 ceph-mon[103614]: pgmap v863: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 19191/159882 objects degraded (12.003%); 2/159882 objects misplaced (0.001%); 11 MiB/s, 177 objects/s recovering 2024-08-17T17:25:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 19191/159882 objects degraded (12.003%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:51.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:51.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:51.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:51.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:51.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:51.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:51.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:51.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:51.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:51.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:51.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:51.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:51.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:51.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:52.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:52.063 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:25:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:25:51] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-08-17T17:25:52.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:52.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:52.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:52.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.342 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:52.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:52.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:52.614 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:52.622 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:52.754 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:52.792 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:52 smithi142 ceph-mon[106729]: pgmap v864: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18904/158301 objects degraded (11.942%); 2/158301 objects misplaced (0.001%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T17:25:52.793 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:52 smithi142 ceph-mon[103614]: pgmap v864: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18904/158301 objects degraded (11.942%); 2/158301 objects misplaced (0.001%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T17:25:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:52.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:52.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:53.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:52 smithi152 ceph-mon[93456]: pgmap v864: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 18904/158301 objects degraded (11.942%); 2/158301 objects misplaced (0.001%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T17:25:53.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:53.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:53.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:53.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:53.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:53.356 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:53.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:53.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:53.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:53.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:53.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:53.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:53.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:53.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:116253 10.8 10:1d55b2b8:::benchmark_data_smithi142_192453_object116252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:46.636710+0000", 2024-08-17T17:25:53.796 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 126.800311755, 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0213902429999999, 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 116253 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.797 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636710+0000", 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636710+0000", 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636712+0000", 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1890000000000001e-06 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636768+0000", 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5698000000000001e-05 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636782+0000", 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3881e-05 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636792+0000", 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5429999999999999e-06 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.800 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640545+0000", 2024-08-17T17:25:53.801 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037532910000000002 2024-08-17T17:25:53.801 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.801 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.801 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.801 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640553+0000", 2024-08-17T17:25:53.801 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6960000000000005e-06 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.646220+0000", 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.00566713 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.802 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.646818+0000", 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000598268 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.657964+0000", 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011146201 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.803 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.658101+0000", 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000136346 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:116255 10.8 10:1c6e316a:::benchmark_data_smithi142_192453_object116254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 126.800215177, 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.072248343, 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.804 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 116255 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.805 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0800000000000004e-07 2024-08-17T17:25:53.806 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636851+0000", 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3763000000000001e-05 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636860+0000", 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2280000000000004e-06 2024-08-17T17:25:53.807 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636867+0000", 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3380000000000001e-06 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640569+0000", 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037025130000000002 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.808 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640574+0000", 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1789999999999999e-06 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.647111+0000", 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.006536724 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:53.809 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.647199+0000", 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7706000000000004e-05 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.657395+0000", 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010196026 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.657453+0000", 2024-08-17T17:25:53.810 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8026000000000003e-05 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.708930+0000", 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.051476916999999997 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.709055+0000", 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000125315 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.811 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144807 10.8 10:12e5ad24:::benchmark_data_smithi142_192453_object19059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.832998+0000", 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 77.604023811000005, 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3307359860000001, 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.812 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144807 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.832998+0000", 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.813 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.832998+0000", 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833000+0000", 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.99e-06 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.814 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833006+0000", 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9730000000000002e-06 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833022+0000", 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5475999999999999e-05 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833029+0000", 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.498e-06 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.815 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.851615+0000", 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018585476 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.851882+0000", 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026769800000000002 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.816 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862610+0000", 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010727571999999999 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862694+0000", 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3969999999999997e-05 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862817+0000", 2024-08-17T17:25:53.817 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012301799999999999 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.895911+0000", 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.033093769000000002 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.896238+0000", 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000326967 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.818 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.158715+0000", 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2624777950000001 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.158913+0000", 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019719 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.163684+0000", 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0047715539999999999 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.819 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.163734+0000", 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0040000000000002e-05 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144822 10.5 10:a9630a4c:::benchmark_data_smithi142_192453_object19074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.861787+0000", 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 77.575235262999996, 2024-08-17T17:25:53.820 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3618416280000001, 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144822 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861787+0000", 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.821 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861787+0000", 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861788+0000", 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.221e-06 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.822 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861793+0000", 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5029999999999997e-06 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861807+0000", 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3886000000000001e-05 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861815+0000", 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8280000000000003e-06 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.823 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862564+0000", 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000748663 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862724+0000", 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015982799999999999 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:25:53.824 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862853+0000", 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012978299999999999 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.895758+0000", 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032904920999999997 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.895772+0000", 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3959e-05 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.825 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.188975+0000", 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.293202934 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.189529+0000", 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055367700000000003 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.826 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.827 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.223551+0000", 2024-08-17T17:25:53.827 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034022377999999999 2024-08-17T17:25:53.827 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.827 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.827 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.827 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.223628+0000", 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7046999999999996e-05 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.828 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:178984 10.5 10:a2f12403:::benchmark_data_smithi142_192453_object53236:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751405+0000", 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685616619000001, 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0189305549999998, 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 178984 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.829 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751405+0000", 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751405+0000", 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.830 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751407+0000", 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3170000000000001e-06 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751411+0000", 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5619999999999997e-06 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.831 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751422+0000", 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.095e-05 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751428+0000", 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9699999999999996e-06 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.832 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247757+0000", 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49632847200000002 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247782+0000", 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5482000000000001e-05 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774427+0000", 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.52664463699999997 2024-08-17T17:25:53.833 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774438+0000", 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0951999999999999e-05 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770249+0000", 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99581095200000003 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.834 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770336+0000", 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7261000000000004e-05 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179006 10.5 10:a5e93376:::benchmark_data_smithi142_192453_object53258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685584148, 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0192025789999999, 2024-08-17T17:25:53.835 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179006 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:25:53.836 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9400000000000001e-07 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.837 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751440+0000", 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9819999999999998e-06 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751445+0000", 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0710000000000002e-06 2024-08-17T17:25:53.838 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751449+0000", 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.031e-06 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247903+0000", 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49645393399999999 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.839 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247926+0000", 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2846000000000001e-05 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774536+0000", 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.52661009800000003 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.840 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774543+0000", 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1520000000000003e-06 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770583+0000", 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99603933700000002 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770640+0000", 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7734e-05 2024-08-17T17:25:53.841 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179009 10.15 10:adfe3263:::benchmark_data_smithi142_192453_object53261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751454+0000", 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685567643999999, 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.103763888, 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.842 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179009 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751454+0000", 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.843 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751454+0000", 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751455+0000", 2024-08-17T17:25:53.844 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0600000000000001e-07 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751456+0000", 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7460000000000001e-06 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751461+0000", 2024-08-17T17:25:53.845 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8539999999999996e-06 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751465+0000", 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8240000000000001e-06 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249382+0000", 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49791667099999998 2024-08-17T17:25:53.846 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249402+0000", 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9830000000000002e-05 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855145+0000", 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.60574357300000004 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855218+0000", 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3083999999999993e-05 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179011 10.13 10:cac77333:::benchmark_data_smithi142_192453_object53263:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685552470000001, 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1257471589999999, 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.848 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179011 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.849 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9700000000000003e-07 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.850 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751472+0000", 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1390000000000002e-06 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751477+0000", 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751481+0000", 2024-08-17T17:25:53.851 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0670000000000002e-06 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.218181+0000", 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.46670027200000003 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.218189+0000", 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4260000000000003e-06 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.852 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877127+0000", 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.65893728600000001 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877217+0000", 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9952999999999997e-05 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.853 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179016 10.5 10:a4006e9a:::benchmark_data_smithi142_192453_object53268:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751485+0000", 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685536575, 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1085708730000001, 2024-08-17T17:25:53.854 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179016 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751485+0000", 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.855 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751485+0000", 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751486+0000", 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.03e-07 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.856 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751487+0000", 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.64e-06 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751492+0000", 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0080000000000002e-06 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751497+0000", 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1710000000000004e-06 2024-08-17T17:25:53.857 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.248656+0000", 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49715965400000001 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.248673+0000", 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7258e-05 2024-08-17T17:25:53.858 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774566+0000", 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.52589242199999997 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774573+0000", 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0589999999999997e-06 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.859 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770664+0000", 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99609061799999998 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770773+0000", 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109517 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770911+0000", 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013743100000000001 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.860 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779028+0000", 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0081173779999999997 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779042+0000", 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.414e-05 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.861 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.859820+0000", 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.080778325999999998 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.859900+0000", 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9658000000000002e-05 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.862 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.860056+0000", 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015629000000000001 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179025 10.3 10:c56d5223:::benchmark_data_smithi142_192453_object53277:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751501+0000", 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685520775000001, 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.083066557, 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.863 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179025 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751501+0000", 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.864 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751501+0000", 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751502+0000", 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7399999999999999e-07 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.865 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751503+0000", 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6199999999999999e-06 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751508+0000", 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9169999999999997e-06 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751512+0000", 2024-08-17T17:25:53.866 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8920000000000002e-06 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.216846+0000", 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.46533451799999997 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.216856+0000", 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2410000000000001e-06 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.867 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.805891+0000", 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.58903523800000002 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.806054+0000", 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000163091 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.868 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.806177+0000", 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012318400000000001 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.811229+0000", 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0050514490000000004 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.819512+0000", 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0082837770000000008 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.834457+0000", 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014945007999999999 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.834516+0000", 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8554000000000002e-05 2024-08-17T17:25:53.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.834568+0000", 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1793999999999999e-05 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179027 10.15 10:a8f69768:::benchmark_data_smithi142_192453_object53279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:25:53.871 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 31.685505871, 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1410192640000001, 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179027 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.872 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3999999999999998e-07 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751518+0000", 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3960000000000001e-06 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.874 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751523+0000", 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1220000000000003e-06 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751529+0000", 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7899999999999996e-06 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249462+0000", 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49793338700000001 2024-08-17T17:25:53.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249473+0000", 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0618999999999999e-05 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855498+0000", 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.60602568700000004 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.877 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.877 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855628+0000", 2024-08-17T17:25:53.877 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012980199999999999 2024-08-17T17:25:53.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.877 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.877 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855770+0000", 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014220500000000001 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.890485+0000", 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034714936000000002 2024-08-17T17:25:53.878 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.890523+0000", 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8160000000000001e-05 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.892352+0000", 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001828367 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.879 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.892390+0000", 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8386000000000001e-05 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.892535+0000", 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000145167 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.880 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179051 10.18 10:1ac43054:::benchmark_data_smithi142_192453_object53303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.842100+0000", 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 30.594921640999999, 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9537706620000002, 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.881 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179051 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842100+0000", 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.882 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842100+0000", 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842102+0000", 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.807e-06 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.883 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842109+0000", 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3999999999999997e-06 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842125+0000", 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6248000000000001e-05 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842135+0000", 2024-08-17T17:25:53.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.045e-05 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842196+0000", 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0943999999999998e-05 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842211+0000", 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.429e-05 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.746081+0000", 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.90387036200000004 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.746091+0000", 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0304e-05 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.886 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.765556+0000", 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0194644129999999 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.765695+0000", 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013904100000000001 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:25:53.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.765830+0000", 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013585800000000001 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795604+0000", 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.029773871 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795734+0000", 2024-08-17T17:25:53.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012952800000000001 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795758+0000", 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4521000000000001e-05 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795799+0000", 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0237000000000001e-05 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795871+0000", 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2387999999999999e-05 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183179 10.18 10:1c594a47:::benchmark_data_smithi142_192453_object57431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:25:53.890 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.766992+0000", 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.670030010000001, 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1695684370000001, 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183179 2024-08-17T17:25:53.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766992+0000", 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766992+0000", 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766993+0000", 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1349999999999999e-06 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766998+0000", 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6430000000000003e-06 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767012+0000", 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4429e-05 2024-08-17T17:25:53.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767020+0000", 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4440000000000002e-06 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767100+0000", 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0448e-05 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767261+0000", 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016068699999999999 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767846+0000", 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058507100000000005 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.896 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793461+0000", 2024-08-17T17:25:53.896 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025615427 2024-08-17T17:25:53.896 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793468+0000", 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8199999999999999e-06 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.897 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785176+0000", 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99170805799999995 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785186+0000", 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.876e-06 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.898 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.931901+0000", 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1467151289999999 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.931968+0000", 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7092000000000001e-05 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936479+0000", 2024-08-17T17:25:53.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045107200000000002 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936560+0000", 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1458000000000005e-05 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.900 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183181 10.19 10:9d925f31:::benchmark_data_smithi142_192453_object57433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.769548+0000", 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.667474205000001, 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.040106049, 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.901 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183181 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769548+0000", 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769548+0000", 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769549+0000", 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.291e-06 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769554+0000", 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6190000000000002e-06 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769563+0000", 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5640000000000002e-06 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769570+0000", 2024-08-17T17:25:53.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4969999999999998e-06 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.780479+0000", 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010908800999999999 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.780842+0000", 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036355600000000001 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.781296+0000", 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045431300000000003 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795305+0000", 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014008270999999999 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795314+0000", 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9050000000000003e-06 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.809578+0000", 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0142647600000001 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.809654+0000", 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5471999999999998e-05 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.907 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183184 10.8 10:16da935b:::benchmark_data_smithi142_192453_object57436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.666383319000001, 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1660231830000001, 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.908 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183184 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.909 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6899999999999997e-07 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770642+0000", 2024-08-17T17:25:53.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1190000000000001e-06 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770651+0000", 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3580000000000003e-06 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770656+0000", 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7579999999999998e-06 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.771143+0000", 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000486279 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.771313+0000", 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170583 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:25:53.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.771913+0000", 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059953100000000004 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793423+0000", 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.021510226 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793434+0000", 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0453e-05 2024-08-17T17:25:53.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785060+0000", 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99162608100000005 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785075+0000", 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.502e-05 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.914 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.930179+0000", 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1451046819999999 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.930300+0000", 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121138 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936593+0000", 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062929860000000004 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936662+0000", 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8399999999999996e-05 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183185 10.19 10:9d6db525:::benchmark_data_smithi142_192453_object57437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:25:53.916 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.666351932000001, 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.07294892, 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183185 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.917 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.918 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6899999999999998e-07 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770672+0000", 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8300000000000001e-06 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770678+0000", 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3550000000000004e-06 2024-08-17T17:25:53.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770683+0000", 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0529999999999997e-06 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795338+0000", 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024655763000000001 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795344+0000", 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1379999999999998e-06 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.812566+0000", 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0172222440000001 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.812692+0000", 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012577399999999999 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.921 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.813081+0000", 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000389436 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826624+0000", 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013542903 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843473+0000", 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016849122000000001 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843525+0000", 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1595999999999999e-05 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843619+0000", 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4337000000000002e-05 2024-08-17T17:25:53.923 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183189 10.19 10:990f2256:::benchmark_data_smithi142_192453_object57441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.786460+0000", 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.650561614000001, 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.057317346, 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.924 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183189 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786460+0000", 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.925 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786460+0000", 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786462+0000", 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.455e-06 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786468+0000", 2024-08-17T17:25:53.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5549999999999999e-06 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786482+0000", 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3573e-05 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786490+0000", 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2069999999999995e-06 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795363+0000", 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0088728950000000004 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795368+0000", 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0409999999999997e-06 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.816415+0000", 2024-08-17T17:25:53.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0210472370000001 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.816550+0000", 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135088 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.817391+0000", 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00084071099999999995 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.929 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826630+0000", 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0092389719999999998 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843658+0000", 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017028002 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843701+0000", 2024-08-17T17:25:53.930 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2864999999999997e-05 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843778+0000", 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6744999999999998e-05 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183201 10.19 10:9f2fe008:::benchmark_data_smithi142_192453_object57453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.804721+0000", 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.632300746999999, 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0521881289999999, 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.932 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183201 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804721+0000", 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804721+0000", 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804722+0000", 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1510000000000001e-06 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804727+0000", 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2579999999999997e-06 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804736+0000", 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2990000000000002e-06 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804742+0000", 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.302e-06 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.935 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.805312+0000", 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00056939800000000004 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.805322+0000", 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0081000000000001e-05 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.819432+0000", 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0141100670000001 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.819562+0000", 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013033599999999999 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.819930+0000", 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000368045 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826635+0000", 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.006704659 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856588+0000", 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.029952880000000001 2024-08-17T17:25:53.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856794+0000", 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000206164 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856909+0000", 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011548900000000001 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183209 10.19 10:9d6a895e:::benchmark_data_smithi142_192453_object57461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.812976+0000", 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.624045812999999, 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.044152341, 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.940 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183209 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812976+0000", 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812976+0000", 2024-08-17T17:25:53.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812978+0000", 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.889e-06 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812985+0000", 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8120000000000002e-06 2024-08-17T17:25:53.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813002+0000", 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6755000000000001e-05 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813010+0000", 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813411+0000", 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00040115700000000002 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813419+0000", 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.799e-06 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.822029+0000", 2024-08-17T17:25:53.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0086097350000001 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.822159+0000", 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013015300000000001 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.822521+0000", 2024-08-17T17:25:53.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036132200000000002 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826640+0000", 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041192449999999997 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856970+0000", 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030330182000000001 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857027+0000", 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6888999999999997e-05 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857129+0000", 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000101544 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.947 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183218 10.19 10:9950a22e:::benchmark_data_smithi142_192453_object57470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.823832+0000", 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.613190034999999, 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.03347671, 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183218 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:25:53.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823832+0000", 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823832+0000", 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823833+0000", 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2750000000000001e-06 2024-08-17T17:25:53.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823838+0000", 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2769999999999999e-06 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823851+0000", 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3560999999999999e-05 2024-08-17T17:25:53.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823858+0000", 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2490000000000004e-06 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823925+0000", 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6742000000000006e-05 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:25:53.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823933+0000", 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6389999999999992e-06 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.824484+0000", 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0005513880000001 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.824622+0000", 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013785499999999999 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.825177+0000", 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055459499999999996 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826740+0000", 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001563191 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857177+0000", 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030437740000000001 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:25:53.954 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857227+0000", 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9391000000000002e-05 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857309+0000", 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1806999999999998e-05 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:25:53.955 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:25:53.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:53.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:53.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:54.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:54.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:54.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:54.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:54.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:54.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:54.497 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:54.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:54.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:54.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:54.662 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:54.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:54.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:54 smithi142 ceph-mon[106729]: pgmap v865: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 18083/146823 objects degraded (12.316%); 2/146823 objects misplaced (0.001%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:25:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:54 smithi142 ceph-mon[103614]: pgmap v865: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 18083/146823 objects degraded (12.316%); 2/146823 objects misplaced (0.001%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:25:54.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:54.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:54.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:55.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:55.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:54 smithi152 ceph-mon[93456]: pgmap v865: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 18083/146823 objects degraded (12.316%); 2/146823 objects misplaced (0.001%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:25:55.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:55.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:25:55.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:55.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:55.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:55.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:55.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:55.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:55.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:55.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:55.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:55.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:55.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:55.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:55.933 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:55 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 18083/146823 objects degraded (12.316%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:55.933 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:55 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 18083/146823 objects degraded (12.316%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:56.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:56.041 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:56.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:55 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 18083/146823 objects degraded (12.316%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:25:56.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:56.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:56.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:56.281 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:56.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:56.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:56.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:56.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:56.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:56.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:56.733 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:56.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:56.777 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:56 smithi142 ceph-mon[103614]: pgmap v866: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 17802/145980 objects degraded (12.195%); 2/145980 objects misplaced (0.001%); 8.4 MiB/s, 135 objects/s recovering 2024-08-17T17:25:56.777 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:56.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:56 smithi142 ceph-mon[106729]: pgmap v866: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 17802/145980 objects degraded (12.195%); 2/145980 objects misplaced (0.001%); 8.4 MiB/s, 135 objects/s recovering 2024-08-17T17:25:56.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:56.834 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:56.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:56.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:56.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:56.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:56.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:57.066 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:25:57.066 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:25:57.066 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-17T17:25:57.066 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:25:57.066 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:25:57.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-17T17:25:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:56 smithi152 ceph-mon[93456]: pgmap v866: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 17802/145980 objects degraded (12.195%); 2/145980 objects misplaced (0.001%); 8.4 MiB/s, 135 objects/s recovering 2024-08-17T17:25:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:57.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:57.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:57.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:57.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:57.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:57.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:57.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:57.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:57.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:57.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:57.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:57.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:57.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:57.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:57.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:57.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:57.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:58.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:58.117 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:58.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:58.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:58.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:58.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:58.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.402 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:58.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:58.503 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:58.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:58.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:58.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:58.689 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:25:58.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:58.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:58.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:58.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:58.958 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:58 smithi142 ceph-mon[103614]: pgmap v867: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 17217/140760 objects degraded (12.231%); 2/140760 objects misplaced (0.001%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T17:25:58.958 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4206185463' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:58.958 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:58 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:58.959 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:58 smithi142 ceph-mon[106729]: pgmap v867: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 17217/140760 objects degraded (12.231%); 2/140760 objects misplaced (0.001%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T17:25:58.959 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4206185463' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:58.960 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:58 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:59.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:59.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:58 smithi152 ceph-mon[93456]: pgmap v867: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 17217/140760 objects degraded (12.231%); 2/140760 objects misplaced (0.001%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T17:25:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4206185463' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:58 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:59.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:59.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:59.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:59.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:59.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:59.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:59.521 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:59.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:59.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:59.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:59.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:59.783 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:25:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:59 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:59 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:25:59 smithi142 ceph-mon[103614]: osdmap e267: 8 total, 7 up, 8 in 2024-08-17T17:25:59.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:59 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:59.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:59 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:59.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:25:59 smithi142 ceph-mon[106729]: osdmap e267: 8 total, 7 up, 8 in 2024-08-17T17:25:59.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:59.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:25:59.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:59.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:59 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:59.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:59 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:59.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:25:59 smithi152 ceph-mon[93456]: osdmap e267: 8 total, 7 up, 8 in 2024-08-17T17:26:00.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:00.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:00.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:00.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.208 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:00.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:00.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:26:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:26:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:00.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:00.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:00.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:00.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:00.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:00.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:00.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:00.759 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:00.862 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:00.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:00.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:01.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:00 smithi152 ceph-mon[93456]: pgmap v869: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 16938/133350 objects degraded (12.702%); 2/133350 objects misplaced (0.001%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T17:26:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:00 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16938/133350 objects degraded (12.702%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:01.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:00 smithi142 ceph-mon[106729]: pgmap v869: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 16938/133350 objects degraded (12.702%); 2/133350 objects misplaced (0.001%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T17:26:01.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:00 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16938/133350 objects degraded (12.702%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:00 smithi142 ceph-mon[103614]: pgmap v869: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 16938/133350 objects degraded (12.702%); 2/133350 objects misplaced (0.001%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T17:26:01.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:00 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16938/133350 objects degraded (12.702%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:01.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:01.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:01.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:01.286 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:01.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:01.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:01.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:01.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:01.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:01.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:01.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:01.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:01.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:01.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:01.849 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:26:01.849 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:01.886 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:26:01.886 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:01.920 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:26:01.920 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:01.921 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:26:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:26:01] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-08-17T17:26:01.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:01.958 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:26:01.958 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:01.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:01.986 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:26:01.986 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:02.016 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:26:02.016 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:02.042 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:26:02.042 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:02.042 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:26:02.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:02.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:02.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:02.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:02.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:02.346 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:02.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:02.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:02.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:02.598 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:02.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:02.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:02.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:02.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:02.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:02.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:02.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:02.928 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:26:02.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:02 smithi142 ceph-mon[103614]: pgmap v870: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 16702/131826 objects degraded (12.670%); 2/131826 objects misplaced (0.002%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T17:26:02.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/814275364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:02.997 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:02 smithi142 ceph-mon[106729]: pgmap v870: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 16702/131826 objects degraded (12.670%); 2/131826 objects misplaced (0.002%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T17:26:02.997 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/814275364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:03.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:03.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:02 smithi152 ceph-mon[93456]: pgmap v870: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 16702/131826 objects degraded (12.670%); 2/131826 objects misplaced (0.002%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T17:26:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/814275364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:26:03.112 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:26:03.155 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:03.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:03.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:03.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:03.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:03.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:03.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:03.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:03.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:03.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:03.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:03.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:04.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:04.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:04.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:04.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:04.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:04.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:04.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:04.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:04.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:04.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:04.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:04.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:04.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:04.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:04.852 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:04.852 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:04.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:04.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:04.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:04 smithi152 ceph-mon[93456]: pgmap v871: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 15873/119838 objects degraded (13.245%); 2/119838 objects misplaced (0.002%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:26:05.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:05.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:05.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:04 smithi142 ceph-mon[103614]: pgmap v871: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 15873/119838 objects degraded (13.245%); 2/119838 objects misplaced (0.002%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:26:05.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:04 smithi142 ceph-mon[106729]: pgmap v871: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 15873/119838 objects degraded (13.245%); 2/119838 objects misplaced (0.002%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:26:05.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:05.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:05.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:05.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:05.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:05.393 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:05.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:26:05.534 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:05.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.560 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:05.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:05.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:05.756 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 15873/119838 objects degraded (13.245%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:05.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 15873/119838 objects degraded (13.245%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:05.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:05.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:05.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:05.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:05.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 15873/119838 objects degraded (13.245%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:06.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:06.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:06.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:06.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:06.265 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:26:06.265 INFO:teuthology.orchestra.run.smithi142.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:26:06.265 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:26:06.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:06.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:06.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:06.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:06.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:06.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:06.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:06.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:06.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:06.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:07.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:07.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:07.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:06 smithi152 ceph-mon[93456]: pgmap v872: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 898 op/s; 15595/119004 objects degraded (13.105%); 2/119004 objects misplaced (0.002%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:26:07.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:06 smithi142 ceph-mon[106729]: pgmap v872: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 898 op/s; 15595/119004 objects degraded (13.105%); 2/119004 objects misplaced (0.002%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:26:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:06 smithi142 ceph-mon[103614]: pgmap v872: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 898 op/s; 15595/119004 objects degraded (13.105%); 2/119004 objects misplaced (0.002%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:26:07.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:07.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:07.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:07.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:07.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:07.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:07.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:07.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:07.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:07.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:07.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:07.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:07.853 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:07.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:07.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:07.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:08.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:08.098 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:08.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:08.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:08.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:08.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:08.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:08.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:08.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:08.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:08.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:08.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:08.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:08.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:08.816 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:08.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:08.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:09.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:09.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:08 smithi152 ceph-mon[93456]: pgmap v873: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 15057/113928 objects degraded (13.216%); 2/113928 objects misplaced (0.002%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:26:09.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:08 smithi142 ceph-mon[106729]: pgmap v873: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 15057/113928 objects degraded (13.216%); 2/113928 objects misplaced (0.002%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:26:09.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:08 smithi142 ceph-mon[103614]: pgmap v873: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 15057/113928 objects degraded (13.216%); 2/113928 objects misplaced (0.002%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:26:09.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:09.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:09.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:09.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:09.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:09.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:09.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:09.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:09.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:09.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:09.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:09.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:10.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:10.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:10.119 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:10.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:10.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:10.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:10.317 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:10.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:10.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:10.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:10.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:26:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:26:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:10.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:10.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:10.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:10.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:10.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:10 smithi142 ceph-mon[106729]: pgmap v874: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 14762/106140 objects degraded (13.908%); 2/106140 objects misplaced (0.002%); 7.8 MiB/s, 155 objects/s recovering 2024-08-17T17:26:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:10 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14762/106140 objects degraded (13.908%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:10 smithi142 ceph-mon[103614]: pgmap v874: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 14762/106140 objects degraded (13.908%); 2/106140 objects misplaced (0.002%); 7.8 MiB/s, 155 objects/s recovering 2024-08-17T17:26:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:10 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14762/106140 objects degraded (13.908%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:10.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:10.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:10 smithi152 ceph-mon[93456]: pgmap v874: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 14762/106140 objects degraded (13.908%); 2/106140 objects misplaced (0.002%); 7.8 MiB/s, 155 objects/s recovering 2024-08-17T17:26:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:10 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14762/106140 objects degraded (13.908%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:11.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:11.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:11.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:11.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:11.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:11.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:11.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:11.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:11.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:11.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:11.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:11.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:11.795 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:11.854 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:26:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:26:11] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-08-17T17:26:11.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:11.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:12.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:12.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:12.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:12.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:12.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:12.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:12.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:12.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:12.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:12.682 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:12 smithi142 ceph-mon[106729]: pgmap v875: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 14459/104400 objects degraded (13.850%); 2/104400 objects misplaced (0.002%); 7.1 MiB/s, 140 objects/s recovering 2024-08-17T17:26:12.682 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:12.683 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:12.683 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:12 smithi142 ceph-mon[103614]: pgmap v875: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 14459/104400 objects degraded (13.850%); 2/104400 objects misplaced (0.002%); 7.1 MiB/s, 140 objects/s recovering 2024-08-17T17:26:12.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:12.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:12 smithi152 ceph-mon[93456]: pgmap v875: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 14459/104400 objects degraded (13.850%); 2/104400 objects misplaced (0.002%); 7.1 MiB/s, 140 objects/s recovering 2024-08-17T17:26:12.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:12.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:12.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:12.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:12.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:13.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:13.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:13.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:13.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:13.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:13.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:13.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:13.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:13.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:13.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:13.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:13.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:14.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:14.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:14.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:14.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:14.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:14.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:14.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:14.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:14.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:14.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:14.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:14.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:14.728 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:14.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:14.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:14.913 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:14.929 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:14 smithi142 ceph-mon[103614]: pgmap v876: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 13851/94836 objects degraded (14.605%); 2/94836 objects misplaced (0.002%); 8.7 MiB/s, 166 objects/s recovering 2024-08-17T17:26:14.929 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:14 smithi142 ceph-mon[106729]: pgmap v876: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 13851/94836 objects degraded (14.605%); 2/94836 objects misplaced (0.002%); 8.7 MiB/s, 166 objects/s recovering 2024-08-17T17:26:15.015 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:15.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:14 smithi152 ceph-mon[93456]: pgmap v876: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 13851/94836 objects degraded (14.605%); 2/94836 objects misplaced (0.002%); 8.7 MiB/s, 166 objects/s recovering 2024-08-17T17:26:15.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:15.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:15.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.548 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:15.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:15.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:15.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:15.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:15.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 13851/94836 objects degraded (14.605%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:16.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 13851/94836 objects degraded (14.605%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:16.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:16.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 13851/94836 objects degraded (14.605%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:16.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:16.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:16.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:16.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:16.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:16.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:16.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:16.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:16.601 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:26:16.602 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:26:16.602 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2024-08-17T17:26:16.602 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2024-08-17T17:26:16.602 DEBUG:teuthology.orchestra.run.smithi152:osd.7> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T17:26:16.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl start ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:26:16.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:16.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:16 smithi152 ceph-mon[93456]: pgmap v877: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 13297/91401 objects degraded (14.548%); 2/91401 objects misplaced (0.002%); 4.9 MiB/s, 152 objects/s recovering 2024-08-17T17:26:16.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:16.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:16.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:16.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:16.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:16.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:17.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:17.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:17.072 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:16 smithi152 systemd[1]: Starting Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:26:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:16 smithi142 ceph-mon[106729]: pgmap v877: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 13297/91401 objects degraded (14.548%); 2/91401 objects misplaced (0.002%); 4.9 MiB/s, 152 objects/s recovering 2024-08-17T17:26:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:16 smithi142 ceph-mon[103614]: pgmap v877: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+clean+remapped, 21 active+undersized, 5 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 55 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 13297/91401 objects degraded (14.548%); 2/91401 objects misplaced (0.002%); 4.9 MiB/s, 152 objects/s recovering 2024-08-17T17:26:17.171 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:17.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:17.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:17.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:17.404 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 podman[148788]: 2024-08-17 17:26:16.984098609 +0000 UTC m=+0.019565918 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:26:17.404 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 podman[148788]: 2024-08-17 17:26:17.108025109 +0000 UTC m=+0.143492410 container create 368c7fae48ab7536097aa6dbc715f8fc45bb9505d4e6d8d2fccbb5f1f8352a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:26:17.404 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 podman[148788]: 2024-08-17 17:26:17.403484513 +0000 UTC m=+0.438951825 container init 368c7fae48ab7536097aa6dbc715f8fc45bb9505d4e6d8d2fccbb5f1f8352a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, RELEASE=quincy-f2c39d4, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:26:17.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:17.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:17.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:17.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:17.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:17.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:17.747 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 podman[148788]: 2024-08-17 17:26:17.414117921 +0000 UTC m=+0.449585235 container start 368c7fae48ab7536097aa6dbc715f8fc45bb9505d4e6d8d2fccbb5f1f8352a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:26:17.747 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 podman[148788]: 2024-08-17 17:26:17.503532796 +0000 UTC m=+0.539000105 container attach 368c7fae48ab7536097aa6dbc715f8fc45bb9505d4e6d8d2fccbb5f1f8352a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:26:17.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:17.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:17.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:17.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:18.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:18.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:26:18.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:26:18.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:26:18.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[148815]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:26:18.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:17 smithi152 bash[148788]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:26:18.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:18.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:18.278 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:18.278 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:18.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:18.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:18.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:18.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:18.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:18.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:18 smithi152 podman[148788]: 2024-08-17 17:26:18.215613696 +0000 UTC m=+1.251081005 container died 368c7fae48ab7536097aa6dbc715f8fc45bb9505d4e6d8d2fccbb5f1f8352a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-08-17T17:26:18.582 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:18.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:18.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:18.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:18.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:18.829 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:18 smithi142 ceph-mon[103614]: pgmap v878: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11638/86526 objects degraded (13.450%); 2/86526 objects misplaced (0.002%); 6.4 MiB/s, 183 objects/s recovering 2024-08-17T17:26:18.830 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:18 smithi142 ceph-mon[106729]: pgmap v878: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11638/86526 objects degraded (13.450%); 2/86526 objects misplaced (0.002%); 6.4 MiB/s, 183 objects/s recovering 2024-08-17T17:26:18.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:18.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:19.007 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:18 smithi152 ceph-mon[93456]: pgmap v878: 99 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 11638/86526 objects degraded (13.450%); 2/86526 objects misplaced (0.002%); 6.4 MiB/s, 183 objects/s recovering 2024-08-17T17:26:19.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:19.032 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:19.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:19.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:19.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:19.275 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 podman[149017]: 2024-08-17 17:26:19.007097265 +0000 UTC m=+1.114000192 container remove 368c7fae48ab7536097aa6dbc715f8fc45bb9505d4e6d8d2fccbb5f1f8352a20 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True) 2024-08-17T17:26:19.275 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 podman[149221]: 2024-08-17 17:26:19.143872171 +0000 UTC m=+0.024074987 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:26:19.276 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 podman[149221]: 2024-08-17 17:26:19.275034865 +0000 UTC m=+0.155237653 container create bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:26:19.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:19.312 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:19.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:19.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:19.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:19.562 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 podman[149221]: 2024-08-17 17:26:19.562108069 +0000 UTC m=+0.442310876 container init bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:26:19.576 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:19.665 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:26:19.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:26:19.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:19.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:19.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:19.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:26:19.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:19.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 podman[149221]: 2024-08-17 17:26:19.569498058 +0000 UTC m=+0.449700853 container start bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0) 2024-08-17T17:26:19.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 ceph-osd[149290]: -- 172.21.15.152:0/2528249376 <== mon.1 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55f03acef4a0 con 0x55f03ab77000 2024-08-17T17:26:19.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 ceph-osd[149290]: -- 172.21.15.152:0/2528249376 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f03aba41a0 con 0x55f03ab77000 2024-08-17T17:26:19.824 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 bash[149221]: bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 2024-08-17T17:26:19.824 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 systemd[1]: Started Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:26:19.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3482280371' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:19.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:19 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:19.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:19.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:20.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:20.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:20.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3482280371' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:20.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:19 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:20.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3482280371' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:20.124 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:19 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:20.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:20.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:20.213 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:19 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[149251]: 2024-08-17T17:26:19.879+0000 7f3c1fb04440 -1 Falling back to public interface 2024-08-17T17:26:20.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:20.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:20.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:20.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:20.551 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:26:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:26:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:20.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:20.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:20.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:20.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:20.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:20.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:20 smithi152 ceph-mon[93456]: pgmap v879: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 10656/79311 objects degraded (13.436%); 740/79311 objects misplaced (0.933%); 5.7 MiB/s, 208 objects/s recovering 2024-08-17T17:26:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:20 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:20 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:20 smithi152 ceph-mon[93456]: osdmap e268: 8 total, 7 up, 8 in 2024-08-17T17:26:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:20 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10656/79311 objects degraded (13.436%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:20.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:20.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:20.966 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[103614]: pgmap v879: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 10656/79311 objects degraded (13.436%); 740/79311 objects misplaced (0.933%); 5.7 MiB/s, 208 objects/s recovering 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[103614]: osdmap e268: 8 total, 7 up, 8 in 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10656/79311 objects degraded (13.436%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[106729]: pgmap v879: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 10656/79311 objects degraded (13.436%); 740/79311 objects misplaced (0.933%); 5.7 MiB/s, 208 objects/s recovering 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:20.967 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[106729]: osdmap e268: 8 total, 7 up, 8 in 2024-08-17T17:26:20.968 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:20 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10656/79311 objects degraded (13.436%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:21.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:21.051 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:26:21.051 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:21.065 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:21.086 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:26:21.087 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:21.124 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:26:21.124 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:21.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:21.163 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:26:21.164 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:21.193 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:26:21.193 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:21.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:21.221 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:26:21.222 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:21.253 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:21.264 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:26:21.265 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:21.265 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:26:21.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:21.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:21.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:21.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:21.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:21.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:21.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:21.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:21.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:21.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.889 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:21.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:21.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:22.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:22.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:22.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3341174324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:22.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3341174324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:22.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:26:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:26:21] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-08-17T17:26:22.124 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3341174324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:22.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:22.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:22.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:22.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:22.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.603 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:22.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:22.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:22.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:22.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:26:22.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:22.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:22.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:22.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:22 smithi152 ceph-mon[93456]: 10.13 scrub starts 2024-08-17T17:26:22.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:22 smithi152 ceph-mon[93456]: pgmap v881: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10401/77799 objects degraded (13.369%); 740/77799 objects misplaced (0.951%); 5.2 MiB/s, 191 objects/s recovering 2024-08-17T17:26:22.978 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:22 smithi142 ceph-mon[106729]: 10.13 scrub starts 2024-08-17T17:26:22.978 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:22 smithi142 ceph-mon[106729]: pgmap v881: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10401/77799 objects degraded (13.369%); 740/77799 objects misplaced (0.951%); 5.2 MiB/s, 191 objects/s recovering 2024-08-17T17:26:22.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:22 smithi142 ceph-mon[103614]: 10.13 scrub starts 2024-08-17T17:26:22.978 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:22 smithi142 ceph-mon[103614]: pgmap v881: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 12 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10401/77799 objects degraded (13.369%); 740/77799 objects misplaced (0.951%); 5.2 MiB/s, 191 objects/s recovering 2024-08-17T17:26:23.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:23.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:23.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:23.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:23.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:23.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:23.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:23.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:23.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:23.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:23.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:23.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:23.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:23.692 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:23.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:23.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:23.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:23 smithi142 ceph-mon[106729]: 10.18 scrub starts 2024-08-17T17:26:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:23 smithi142 ceph-mon[103614]: 10.18 scrub starts 2024-08-17T17:26:23.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:23.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:23.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:23 smithi152 ceph-mon[93456]: 10.18 scrub starts 2024-08-17T17:26:24.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:24.104 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:26:24.104 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:26:24.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:24.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:24.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:24.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:24.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:24.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:24.438 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2024-08-17T17:26:24.438 INFO:teuthology.orchestra.run.smithi152.stderr:0 2024-08-17T17:26:24.438 INFO:teuthology.orchestra.run.smithi152.stderr:1 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:2 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore bluefs device info [] 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:config diff 2024-08-17T17:26:24.439 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2024-08-17T17:26:24.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:24.528 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:24.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:85915 10.4 e233/222 10:26360449:::benchmark_data_smithi142_192453_object85914:head v 233'5364, mlcod=233'5362)", 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:17.465575+0000", 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.11096289599999, 2024-08-17T17:26:24.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.108773988, 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465575+0000", 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465575+0000", 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465577+0000", 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.389e-06 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465610+0000", 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3352999999999997e-05 2024-08-17T17:26:24.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465628+0000", 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7476999999999999e-05 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465635+0000", 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2239999999999998e-06 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.584 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465694+0000", 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9613e-05 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.465714+0000", 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9065000000000001e-05 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.471355+0000", 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0056410419999999998 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574349+0000", 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.102994825 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:85920 10.7 e233/222 10:e9594680:::benchmark_data_smithi142_192453_object85919:head v 233'5337, mlcod=233'5335)", 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:17.466218+0000", 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.110320105, 2024-08-17T17:26:24.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.113836324, 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.466218+0000", 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.466218+0000", 2024-08-17T17:26:24.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.466219+0000", 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4099999999999997e-07 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.466277+0000", 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8037e-05 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.466287+0000", 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0176e-05 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.466294+0000", 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.392e-06 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574489+0000", 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10819508799999999 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574507+0000", 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8102e-05 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.579984+0000", 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054768999999999998 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.580055+0000", 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0788000000000001e-05 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:85921 10.f e233/222 10:f781cde8:::benchmark_data_smithi142_192453_object85920:head v 233'5453, mlcod=233'5451)", 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:17.468235+0000", 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.10830372199999, 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.111745491, 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.591 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.468235+0000", 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.468235+0000", 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.592 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.468235+0000", 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7899999999999997e-07 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.468285+0000", 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9434000000000001e-05 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.468296+0000", 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1351e-05 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.468303+0000", 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9009999999999997e-06 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574518+0000", 2024-08-17T17:26:24.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10621516 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574537+0000", 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8369999999999999e-05 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.579923+0000", 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053858300000000003 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.579980+0000", 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7565999999999997e-05 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:85916 10.5 e233/222 10:a7e467c4:::benchmark_data_smithi142_192453_object85915:head v 233'5494, mlcod=233'5490)", 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:17.469702+0000", 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.10683665600001, 2024-08-17T17:26:24.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.117539798, 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.469702+0000", 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.469702+0000", 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.469702+0000", 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7599999999999996e-07 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.469730+0000", 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7399e-05 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574246+0000", 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10451635500000001 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574262+0000", 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5461999999999999e-05 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.599 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574323+0000", 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1447999999999998e-05 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.574353+0000", 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9808e-05 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.587204+0000", 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012850751000000001 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:17.587241+0000", 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7799000000000002e-05 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:89513 10.f e233/222 10:fd96efd5:::benchmark_data_smithi142_192453_object89512:head v 233'5671, mlcod=233'5667)", 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:20.907970+0000", 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.66856818599999, 2024-08-17T17:26:24.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.181509693, 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.907970+0000", 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.907970+0000", 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.907971+0000", 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9299999999999997e-07 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.908025+0000", 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4551e-05 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.908034+0000", 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9870000000000005e-06 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.908040+0000", 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6810000000000003e-06 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.908086+0000", 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5686999999999998e-05 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.908100+0000", 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4426e-05 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089428+0000", 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18132736499999999 2024-08-17T17:26:24.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089480+0000", 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2302999999999997e-05 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.606 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:89518 10.7 e233/222 10:e26c6570:::benchmark_data_smithi142_192453_object89517:head v 233'5548, mlcod=233'5546)", 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:20.911794+0000", 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.66474436600001, 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18053741700000001, 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.607 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.911794+0000", 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.911794+0000", 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.911795+0000", 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7800000000000001e-07 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.911834+0000", 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9236000000000002e-05 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.911845+0000", 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0805000000000001e-05 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.911851+0000", 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2940000000000002e-06 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912154+0000", 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00030276499999999998 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912171+0000", 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7418000000000002e-05 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.092278+0000", 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18010637800000001 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.092331+0000", 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3842999999999999e-05 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:89515 10.f e233/222 10:f80eb3e9:::benchmark_data_smithi142_192453_object89514:head v 233'5672, mlcod=233'5669)", 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:20.912808+0000", 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.663730558, 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18503683900000001, 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.612 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.613 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.613 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.613 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912808+0000", 2024-08-17T17:26:24.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912808+0000", 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912808+0000", 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3799999999999997e-07 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912857+0000", 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9064000000000003e-05 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912867+0000", 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9899999999999992e-06 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.912873+0000", 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0170000000000002e-06 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089521+0000", 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.17664719400000001 2024-08-17T17:26:24.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089537+0000", 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6181999999999999e-05 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.097781+0000", 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.008244187 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.097845+0000", 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3566999999999996e-05 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:89536 10.4 e233/222 10:20160454:::benchmark_data_smithi142_192453_object89535:head v 233'5596, mlcod=233'5593)", 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:20.915329+0000", 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.661209681, 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18021348000000001, 2024-08-17T17:26:24.618 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.915329+0000", 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.915329+0000", 2024-08-17T17:26:24.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.915329+0000", 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3699999999999999e-07 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.915352+0000", 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2374e-05 2024-08-17T17:26:24.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089431+0000", 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.17407895800000001 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089444+0000", 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3512e-05 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089503+0000", 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8464000000000001e-05 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089529+0000", 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5732999999999999e-05 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.095498+0000", 2024-08-17T17:26:24.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059690009999999998 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.095542+0000", 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4601000000000002e-05 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:89524 10.7 e233/222 10:e8a32be5:::benchmark_data_smithi142_192453_object89523:head v 233'5549, mlcod=233'5546)", 2024-08-17T17:26:24.623 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:20.920090+0000", 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.65644878800001, 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.17768848200000001, 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.920090+0000", 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.920090+0000", 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.920091+0000", 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2669999999999999e-06 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.920143+0000", 2024-08-17T17:26:24.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1836000000000003e-05 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089428+0000", 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.16928488999999999 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089444+0000", 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6733000000000001e-05 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089514+0000", 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0178000000000005e-05 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.089533+0000", 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8728000000000001e-05 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.097657+0000", 2024-08-17T17:26:24.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0081237299999999991 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.097778+0000", 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012112000000000001 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:127793 10.d e244/240 10:b4c93bf8:::benchmark_data_smithi142_192453_object2045:head v 244'8038, mlcod=0'0)", 2024-08-17T17:26:24.628 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:05.212104+0000", 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 139.36443418600001, 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.076673369000000005, 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.212104+0000", 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.212104+0000", 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.212105+0000", 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4600000000000003e-07 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.212111+0000", 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9599999999999997e-06 2024-08-17T17:26:24.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.212138+0000", 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7217e-05 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.212146+0000", 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7239999999999992e-06 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.219799+0000", 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0076527050000000001 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.219847+0000", 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7839000000000001e-05 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.288735+0000", 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.068888282999999995 2024-08-17T17:26:24.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:05.288778+0000", 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2694999999999997e-05 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:132934 10.c e244/240 10:32f3988c:::benchmark_data_smithi142_192453_object7186:head v 244'8359, mlcod=227'458)", 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:14.311121+0000", 2024-08-17T17:26:24.633 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 130.26541757199999, 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.077306322999999996, 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311121+0000", 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.634 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311121+0000", 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311121+0000", 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7100000000000002e-07 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311125+0000", 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.968e-06 2024-08-17T17:26:24.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311134+0000", 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4400000000000005e-06 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311139+0000", 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9219999999999996e-06 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311203+0000", 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4173999999999999e-05 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.311275+0000", 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1995000000000004e-05 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.388253+0000", 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.076978104000000006 2024-08-17T17:26:24.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:14.388427+0000", 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017424899999999999 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:139354 10.2 e246/240 10:4e7e8dcd:::benchmark_data_smithi142_192453_object13606:head v 246'8696, mlcod=0'0)", 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:26.007120+0000", 2024-08-17T17:26:24.638 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 118.569418246, 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.086177744000000001, 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.007120+0000", 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.007120+0000", 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.007121+0000", 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.08e-06 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.007129+0000", 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8099999999999998e-06 2024-08-17T17:26:24.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.007154+0000", 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5474e-05 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.007163+0000", 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0800000000000006e-06 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.022274+0000", 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015111378999999999 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.022303+0000", 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8605000000000001e-05 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.092904+0000", 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.070601518000000002 2024-08-17T17:26:24.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.093298+0000", 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000393798 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:139360 10.2 e246/240 10:4599a8fe:::benchmark_data_smithi142_192453_object13612:head v 246'8697, mlcod=0'0)", 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:26.017755+0000", 2024-08-17T17:26:24.643 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 118.55878355500001, 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.075582044000000001, 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.017755+0000", 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.017755+0000", 2024-08-17T17:26:24.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.017756+0000", 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3629999999999999e-06 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.017765+0000", 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6119999999999999e-06 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.017782+0000", 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7070000000000001e-05 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.017791+0000", 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6150000000000003e-06 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.033291+0000", 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015499511000000001 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.033344+0000", 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3483000000000003e-05 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.093304+0000", 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.059960029999999997 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.093337+0000", 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2360000000000002e-05 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:139587 10.c e246/240 10:35075aa2:::benchmark_data_smithi142_192453_object13839:head v 246'8786, mlcod=227'871)", 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:26.420078+0000", 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 118.15645989399999, 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.085585312999999996, 2024-08-17T17:26:24.648 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.420078+0000", 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.420078+0000", 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.420080+0000", 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8059999999999999e-06 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.420092+0000", 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1561e-05 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.650 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.420113+0000", 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0950000000000001e-05 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.420123+0000", 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9769999999999995e-06 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.449232+0000", 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.029109586 2024-08-17T17:26:24.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.449260+0000", 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8045999999999999e-05 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.505621+0000", 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.056360562000000003 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:26.505664+0000", 2024-08-17T17:26:24.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2824999999999997e-05 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:143623 10.d e247/240 10:b515f9c4:::benchmark_data_smithi142_192453_object17875:head v 247'9089, mlcod=0'0)", 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:33.688309+0000", 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 110.888229813, 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.079591505000000007, 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.653 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.688309+0000", 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.688309+0000", 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.688310+0000", 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.818e-06 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.688321+0000", 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1052e-05 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.688341+0000", 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9511e-05 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.688352+0000", 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.112e-05 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.704180+0000", 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015828321999999999 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.704208+0000", 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7239000000000001e-05 2024-08-17T17:26:24.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.767860+0000", 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.063652637999999997 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:33.767900+0000", 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9805000000000003e-05 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:144814 10.2 e247/240 10:499d4ad0:::benchmark_data_smithi142_192453_object19066:head v 247'9034, mlcod=0'0)", 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.846655+0000", 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 108.729883296, 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.076191025999999995, 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.846655+0000", 2024-08-17T17:26:24.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.846655+0000", 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.846656+0000", 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1689999999999999e-06 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.846665+0000", 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1560000000000001e-06 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.846723+0000", 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7565999999999997e-05 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.846733+0000", 2024-08-17T17:26:24.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7079999999999992e-06 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.876990+0000", 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030257108000000001 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.877013+0000", 2024-08-17T17:26:24.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3252e-05 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.922492+0000", 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.045479104999999999 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.922846+0000", 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035396199999999998 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:174986 10.f e254/222 10:faef2e0e:::benchmark_data_smithi142_192453_object49238:head v 254'11065, mlcod=254'11063)", 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:16.670975+0000", 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 67.905563673000003, 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.089543267999999995, 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.670975+0000", 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.670975+0000", 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.670976+0000", 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7899999999999997e-07 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.670981+0000", 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.942e-06 2024-08-17T17:26:24.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.670993+0000", 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1078e-05 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.671000+0000", 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2559999999999996e-06 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.671337+0000", 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00033761000000000002 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.671357+0000", 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9108000000000001e-05 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.682793+0000", 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011436145999999999 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.760518+0000", 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.077725248999999996 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:174992 10.3 e255/222 10:c5f2a46e:::benchmark_data_smithi142_192453_object49244:head v 255'10744, mlcod=254'10742)", 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:16.682657+0000", 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 67.893881733000001, 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.088457201999999999, 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.668 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.682657+0000", 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.682657+0000", 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.682657+0000", 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0999999999999998e-07 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.682666+0000", 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5299999999999996e-06 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.760696+0000", 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078029790000000002 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.760705+0000", 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9509999999999995e-06 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.760785+0000", 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0635000000000003e-05 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.760805+0000", 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9765000000000001e-05 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.771013+0000", 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010207567000000001 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:16.771114+0000", 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000101254 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:196027 10.c e264/262 10:3290f7fc:::benchmark_data_smithi142_192453_object70279:head v 264'12307, mlcod=230'3591)", 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:44.567570+0000", 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 40.008968062999998, 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.088146554000000002, 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.673 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.567570+0000", 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.567570+0000", 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.567571+0000", 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1600000000000001e-07 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.567577+0000", 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.392e-06 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.567591+0000", 2024-08-17T17:26:24.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3200000000000001e-05 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.567598+0000", 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5390000000000002e-06 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.599815+0000", 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032217281 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.599833+0000", 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7853e-05 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.655671+0000", 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.055837688000000003 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.655717+0000", 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5884999999999999e-05 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.677 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:196189 10.c e264/262 10:3674debb:::benchmark_data_smithi142_192453_object70441:head v 264'12321, mlcod=230'3591)", 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:44.800580+0000", 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 39.775958727000003, 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.085306168000000002, 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.800580+0000", 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.800580+0000", 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.800580+0000", 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7800000000000001e-07 2024-08-17T17:26:24.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.800588+0000", 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3159999999999999e-06 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.800730+0000", 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014253200000000001 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:26:24.680 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.800738+0000", 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.191e-06 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.818243+0000", 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017504319000000001 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.818265+0000", 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2238999999999998e-05 2024-08-17T17:26:24.681 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.885773+0000", 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.067507674000000004 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:44.885886+0000", 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000113119 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.682 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:26:24.683 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:26:24.683 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:26:24.689 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:24 smithi142 ceph-mon[103614]: pgmap v882: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 9865/68967 objects degraded (14.304%); 740/68967 objects misplaced (1.073%); 5.0 MiB/s, 238 objects/s recovering 2024-08-17T17:26:24.690 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:24 smithi142 ceph-mon[103614]: 10.3 scrub starts 2024-08-17T17:26:24.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:24.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:24.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:24 smithi152 ceph-mon[93456]: pgmap v882: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 9865/68967 objects degraded (14.304%); 740/68967 objects misplaced (1.073%); 5.0 MiB/s, 238 objects/s recovering 2024-08-17T17:26:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:24 smithi152 ceph-mon[93456]: 10.3 scrub starts 2024-08-17T17:26:24.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:24.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:25.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:24 smithi142 ceph-mon[106729]: pgmap v882: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 9865/68967 objects degraded (14.304%); 740/68967 objects misplaced (1.073%); 5.0 MiB/s, 238 objects/s recovering 2024-08-17T17:26:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:24 smithi142 ceph-mon[106729]: 10.3 scrub starts 2024-08-17T17:26:25.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:25.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:25.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:25.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:25.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:25.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:26:25.433 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2024-08-17T17:26:25.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:25.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:25.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:25.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:25.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:25.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:25.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:25 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 9865/68967 objects degraded (14.304%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:25 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 9865/68967 objects degraded (14.304%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:25.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:25.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:25.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:26.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:26.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:26:26.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:25 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 9865/68967 objects degraded (14.304%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:26.075 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:26.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:26:26.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:26.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:26.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:26.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:26.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:26.510 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:26.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:26.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:26.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:26.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:26 smithi152 ceph-mon[93456]: pgmap v883: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 9467/65847 objects degraded (14.377%); 605/65847 objects misplaced (0.919%); 5.1 MiB/s, 183 objects/s recovering 2024-08-17T17:26:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:26.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:26.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:27.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:27.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:27.099 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:26 smithi142 ceph-mon[106729]: pgmap v883: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 9467/65847 objects degraded (14.377%); 605/65847 objects misplaced (0.919%); 5.1 MiB/s, 183 objects/s recovering 2024-08-17T17:26:27.099 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:27.099 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:26 smithi142 ceph-mon[103614]: pgmap v883: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 9467/65847 objects degraded (14.377%); 605/65847 objects misplaced (0.919%); 5.1 MiB/s, 183 objects/s recovering 2024-08-17T17:26:27.100 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:27.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:27.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:27.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:27.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:27.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:27.483 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:27.566 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:27.584 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:27.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:27.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:27.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:27.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:27.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:28.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.035 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:28.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:28.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:28.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:28.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:28.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:28.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:28.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:28.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:28.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:28.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:28.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:28.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:28.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:28.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:28.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:28.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:28.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:28.967 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:28 smithi142 ceph-mon[106729]: 10.13 scrub starts 2024-08-17T17:26:28.968 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:28 smithi142 ceph-mon[106729]: pgmap v884: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 8921/60753 objects degraded (14.684%); 605/60753 objects misplaced (0.996%); 4.9 MiB/s, 172 objects/s recovering 2024-08-17T17:26:28.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:28 smithi142 ceph-mon[103614]: 10.13 scrub starts 2024-08-17T17:26:28.968 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:28 smithi142 ceph-mon[103614]: pgmap v884: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 8921/60753 objects degraded (14.684%); 605/60753 objects misplaced (0.996%); 4.9 MiB/s, 172 objects/s recovering 2024-08-17T17:26:29.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:29.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:28 smithi152 ceph-mon[93456]: 10.13 scrub starts 2024-08-17T17:26:29.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:28 smithi152 ceph-mon[93456]: pgmap v884: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 8921/60753 objects degraded (14.684%); 605/60753 objects misplaced (0.996%); 4.9 MiB/s, 172 objects/s recovering 2024-08-17T17:26:29.130 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:29.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.134 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:29.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:29.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:29.282 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:29.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:29.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:29.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:29.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:29.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:29.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.558 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:29.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:29.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:29 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[149251]: 2024-08-17T17:26:29.247+0000 7f3c1fb04440 -1 osd.7 260 log_to_monitors true 2024-08-17T17:26:29.659 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:29.706 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[103614]: 10.18 scrub starts 2024-08-17T17:26:29.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:29.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:29.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:29.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:29 smithi152 ceph-mon[93456]: 10.18 scrub starts 2024-08-17T17:26:29.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:29 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:29.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:29 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:29.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:29 smithi152 ceph-mon[93456]: from='osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:26:29.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:29 smithi152 ceph-mon[93456]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:26:29.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:30.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:30.041 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[106729]: 10.18 scrub starts 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[106729]: from='osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[106729]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[103614]: from='osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:26:30.042 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:29 smithi142 ceph-mon[103614]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:26:30.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:30.142 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:30.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:30.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:30.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:30.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:30.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:26:30.496 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[149251]: 2024-08-17T17:26:30.220+0000 7f3c140a9640 -1 osd.7 260 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:26:30.496 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:26:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:30.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:30.539 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:30.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:30.651 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:26:30.652 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:30.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:30.697 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:26:30.697 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:30.730 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:26:30.730 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:30.760 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:26:30.760 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:30.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:30.794 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:26:30.794 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:30.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:30.823 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:26:30.823 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: 10.3 scrub starts 2024-08-17T17:26:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: pgmap v885: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 8799/53742 objects degraded (16.373%); 484/53742 objects misplaced (0.901%); 4.9 MiB/s, 129 objects/s recovering 2024-08-17T17:26:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:26:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: from='osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:26:30.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: osdmap e269: 8 total, 7 up, 8 in 2024-08-17T17:26:30.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:26:30.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3171642549' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:30.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:30.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 8799/53742 objects degraded (16.373%), 20 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:30.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3849698604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:30.853 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:26:30.853 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:30.853 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:26:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: 10.3 scrub starts 2024-08-17T17:26:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: pgmap v885: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 8799/53742 objects degraded (16.373%); 484/53742 objects misplaced (0.901%); 4.9 MiB/s, 129 objects/s recovering 2024-08-17T17:26:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:26:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: from='osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: osdmap e269: 8 total, 7 up, 8 in 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3171642549' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 8799/53742 objects degraded (16.373%), 20 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3849698604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: 10.3 scrub starts 2024-08-17T17:26:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: pgmap v885: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 8799/53742 objects degraded (16.373%); 484/53742 objects misplaced (0.901%); 4.9 MiB/s, 129 objects/s recovering 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: from='osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: osdmap e269: 8 total, 7 up, 8 in 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3171642549' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 8799/53742 objects degraded (16.373%), 20 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:30.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3849698604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:30.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:30.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:31.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:31.197 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:26:31.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:31.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:31.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:31.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:31.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:31.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:31.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:31.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:31.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:31.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:31.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:31.980 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:26:31] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-08-17T17:26:31.980 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:31.980 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[106729]: osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009] boot 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[106729]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[103614]: osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009] boot 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[103614]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:26:31.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:26:31.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:31.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:26:32.007 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:26:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:31 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:31 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:26:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:31 smithi152 ceph-mon[93456]: osd.7 [v2:172.21.15.152:6824/3835016009,v1:172.21.15.152:6825/3835016009] boot 2024-08-17T17:26:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:31 smithi152 ceph-mon[93456]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:26:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:26:32.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:32.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:32.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:32.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:32.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:32.365 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:32.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:32.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:32.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:32.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:32.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:32.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:32.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:32.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:33.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:33.110 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:33.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.251 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:33.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:33.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:33.377 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:33.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:33.482 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:33 smithi142 ceph-mon[106729]: pgmap v888: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8536/52188 objects degraded (16.356%); 484/52188 objects misplaced (0.927%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:26:33.483 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:33 smithi142 ceph-mon[106729]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:26:33.483 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:33 smithi142 ceph-mon[106729]: 10.13 scrub starts 2024-08-17T17:26:33.483 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:33 smithi142 ceph-mon[103614]: pgmap v888: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8536/52188 objects degraded (16.356%); 484/52188 objects misplaced (0.927%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:26:33.483 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:33 smithi142 ceph-mon[103614]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:26:33.483 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:33 smithi142 ceph-mon[103614]: 10.13 scrub starts 2024-08-17T17:26:33.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:33.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:33 smithi152 ceph-mon[93456]: pgmap v888: 99 pgs: 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 56 active+clean; 1.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8536/52188 objects degraded (16.356%); 484/52188 objects misplaced (0.927%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T17:26:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:33 smithi152 ceph-mon[93456]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:26:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:33 smithi152 ceph-mon[93456]: 10.13 scrub starts 2024-08-17T17:26:33.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:33.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:33.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:33.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:33.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:33.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:33.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:33.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:34.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:34.014 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:34.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:34.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:34.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:34.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:34.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:34.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:34.324 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:34.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: 10.12 scrub starts 2024-08-17T17:26:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: 10.13 scrub ok 2024-08-17T17:26:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: 10.12 scrub ok 2024-08-17T17:26:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:26:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: 10.12 scrub starts 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: 10.13 scrub ok 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: 10.12 scrub ok 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:26:34.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:26:34.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:34 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:34.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:34.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:34.523 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: 10.12 scrub starts 2024-08-17T17:26:34.523 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: 10.13 scrub ok 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: 10.12 scrub ok 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:26:34.524 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:34 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:34.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:34.546 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:34.546 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:34.559 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:34.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:34.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:34.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:34.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:34.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:26:34.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:34.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:34.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:35.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:35.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:35.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:35.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:35.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:35.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:35.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:35 smithi152 ceph-mon[93456]: pgmap v891: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 16 active+undersized, 4 active+recovery_wait+degraded, 9 active+undersized+degraded, 62 active+clean; 975 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5123/46839 objects degraded (10.937%); 484/46839 objects misplaced (1.033%); 2.9 MiB/s, 73 objects/s recovering 2024-08-17T17:26:35.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:35 smithi152 ceph-mon[93456]: pgmap v892: 99 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+clean+remapped, 16 active+undersized, 2 active+recovery_wait+degraded, 9 active+undersized+degraded, 62 active+clean; 950 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 4869/45675 objects degraded (10.660%); 2/45675 objects misplaced (0.004%); 3.7 MiB/s, 97 objects/s recovering 2024-08-17T17:26:35.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:35 smithi152 ceph-mon[93456]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:26:35.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:35 smithi152 ceph-mon[93456]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:26:35.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:26:35.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:35.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:35.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[106729]: pgmap v891: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 16 active+undersized, 4 active+recovery_wait+degraded, 9 active+undersized+degraded, 62 active+clean; 975 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5123/46839 objects degraded (10.937%); 484/46839 objects misplaced (1.033%); 2.9 MiB/s, 73 objects/s recovering 2024-08-17T17:26:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[106729]: pgmap v892: 99 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+clean+remapped, 16 active+undersized, 2 active+recovery_wait+degraded, 9 active+undersized+degraded, 62 active+clean; 950 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 4869/45675 objects degraded (10.660%); 2/45675 objects misplaced (0.004%); 3.7 MiB/s, 97 objects/s recovering 2024-08-17T17:26:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[106729]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:26:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[106729]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:26:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[103614]: pgmap v891: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+degraded+remapped, 2 active+clean+remapped, 16 active+undersized, 4 active+recovery_wait+degraded, 9 active+undersized+degraded, 62 active+clean; 975 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5123/46839 objects degraded (10.937%); 484/46839 objects misplaced (1.033%); 2.9 MiB/s, 73 objects/s recovering 2024-08-17T17:26:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[103614]: pgmap v892: 99 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+clean+remapped, 16 active+undersized, 2 active+recovery_wait+degraded, 9 active+undersized+degraded, 62 active+clean; 950 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 4869/45675 objects degraded (10.660%); 2/45675 objects misplaced (0.004%); 3.7 MiB/s, 97 objects/s recovering 2024-08-17T17:26:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[103614]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:26:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:35 smithi142 ceph-mon[103614]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:26:35.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:35.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:35.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:35.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:35.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:35.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:36.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:36.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:36.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:36.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:36.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:36.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:36.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:36.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4869/45675 objects degraded (10.660%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:36.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4869/45675 objects degraded (10.660%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:36.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.470 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:36.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:36.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4869/45675 objects degraded (10.660%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:36.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:36.703 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:36.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:36.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:36.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:36.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:36.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:37.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:37.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:37.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:37.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:37.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:37.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:37.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:37.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:37.379 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:26:37.379 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:26:37.379 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:26:37.379 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:26:37.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:37 smithi152 ceph-mon[93456]: pgmap v894: 99 pgs: 2 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+degraded, 87 active+clean; 935 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 6596/44916 objects degraded (14.685%); 3.2 MiB/s, 83 objects/s recovering 2024-08-17T17:26:37.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:37.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:37.572 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:37 smithi142 ceph-mon[106729]: pgmap v894: 99 pgs: 2 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+degraded, 87 active+clean; 935 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 6596/44916 objects degraded (14.685%); 3.2 MiB/s, 83 objects/s recovering 2024-08-17T17:26:37.573 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:37 smithi142 ceph-mon[103614]: pgmap v894: 99 pgs: 2 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+degraded, 87 active+clean; 935 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 6596/44916 objects degraded (14.685%); 3.2 MiB/s, 83 objects/s recovering 2024-08-17T17:26:37.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:37.604 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:37.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:37.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:37.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:37.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:37.917 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:37.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:38.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:38.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:38.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:38.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:38.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:38.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:38.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:38.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:38.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:38.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:38.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:38.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:38.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:38.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:38.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:38.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:38.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:38.961 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:26:38.961 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:38.986 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:26:38.986 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:38.994 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:39.026 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:26:39.026 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:39.029 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:39.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:39.050 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:26:39.050 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:39.065 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:26:39.066 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:39.081 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:26:39.081 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:39.100 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:26:39.100 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:39.131 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:26:39.132 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:39.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:39.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:39.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:39.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:39.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:39 smithi142 ceph-mon[106729]: pgmap v895: 99 pgs: 2 peering, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 800 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 14836/38460 objects degraded (38.575%); 2.5 MiB/s, 80 objects/s recovering 2024-08-17T17:26:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1321482697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:39.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:39 smithi142 ceph-mon[103614]: pgmap v895: 99 pgs: 2 peering, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 800 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 14836/38460 objects degraded (38.575%); 2.5 MiB/s, 80 objects/s recovering 2024-08-17T17:26:39.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1321482697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:39.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:39.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:39.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:39 smithi152 ceph-mon[93456]: pgmap v895: 99 pgs: 2 peering, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 800 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 14836/38460 objects degraded (38.575%); 2.5 MiB/s, 80 objects/s recovering 2024-08-17T17:26:39.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1321482697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:39.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:39.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:39.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:39.744 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:39.834 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:26:39.834 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:26:39.834 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:26:39.834 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:26:39.834 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:26:39.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:39.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:39.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:40.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:40.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:40.202 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:40.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:40.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:40.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:40.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:40.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:40.503 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:26:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:26:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:40.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:26:40.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:40.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:40.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:40.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:40.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:40.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:40.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:40.988 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:41.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:41.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:41.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:41.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:41.231 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:26:41.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:41.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:41.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:41.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:41.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:41.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:41 smithi152 ceph-mon[93456]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 733 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 16020/35229 objects degraded (45.474%); 503 KiB/s, 100 objects/s recovering 2024-08-17T17:26:41.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16020/35229 objects degraded (45.474%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:41.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3979248355' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:41.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:41.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-mon[106729]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 733 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 16020/35229 objects degraded (45.474%); 503 KiB/s, 100 objects/s recovering 2024-08-17T17:26:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16020/35229 objects degraded (45.474%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3979248355' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-mon[103614]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 733 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 16020/35229 objects degraded (45.474%); 503 KiB/s, 100 objects/s recovering 2024-08-17T17:26:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16020/35229 objects degraded (45.474%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3979248355' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:41.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:41.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:41.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:41.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:41.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:41.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:42.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:42.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:26:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:26:41] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.51.0" 2024-08-17T17:26:42.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:42.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:42.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:42.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:42.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3979248355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[106729]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3979248355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[103614]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:26:42.477 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:42.478 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:42.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:42.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:42 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:42.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3979248355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:42.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:42 smithi152 ceph-mon[93456]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:26:42.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:42.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:42.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:42.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:42.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:42.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:42.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:42.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:42.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:42.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:42.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:42.989 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73423 9.10 9.364f5070 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.638087+0000", 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 507.34647901900001, 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96902246400000003, 2024-08-17T17:26:42.990 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73423 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638087+0000", 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:42.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638087+0000", 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638085+0000", 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638219+0000", 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013416799999999999 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638229+0000", 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5980000000000003e-06 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638234+0000", 2024-08-17T17:26:42.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0200000000000002e-06 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638275+0000", 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0731e-05 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.638281+0000", 2024-08-17T17:26:42.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.657e-06 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607099+0000", 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96881813800000005 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:42.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607109+0000", 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7890000000000006e-06 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73452 9.10 9.1b94c10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.654997+0000", 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 507.329568843, 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95221170799999999, 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:42.996 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73452 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.654997+0000", 2024-08-17T17:26:42.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.654997+0000", 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.654995+0000", 2024-08-17T17:26:42.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655078+0000", 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2665999999999998e-05 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655090+0000", 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1800000000000001e-05 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:42.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655094+0000", 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3970000000000004e-06 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655139+0000", 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4725000000000003e-05 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.655146+0000", 2024-08-17T17:26:43.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6240000000000002e-06 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607202+0000", 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95205562600000004 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607209+0000", 2024-08-17T17:26:43.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4579999999999998e-06 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73460 9.10 9.6fcedb10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.659057+0000", 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 507.32550879600001, 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94826010100000002, 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.002 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73460 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659057+0000", 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659057+0000", 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659055+0000", 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659082+0000", 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.728e-05 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659096+0000", 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3103e-05 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659102+0000", 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9460000000000003e-06 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659157+0000", 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5708000000000003e-05 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.659164+0000", 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9949999999999999e-06 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607305+0000", 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94814032699999995 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607317+0000", 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2401e-05 2024-08-17T17:26:43.007 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73489 9.10 9.408e6a50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.677505+0000", 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 507.30706103199998, 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92987554699999997, 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.008 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73489 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677505+0000", 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677505+0000", 2024-08-17T17:26:43.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677503+0000", 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677640+0000", 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013677000000000001 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677649+0000", 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0380000000000005e-06 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677654+0000", 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9400000000000001e-06 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677702+0000", 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7562999999999998e-05 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.677709+0000", 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5299999999999999e-06 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607348+0000", 2024-08-17T17:26:43.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92963922200000004 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607380+0000", 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1744e-05 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:73517 9.10 9.fb63b790 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2024-08-17T17:26:43.013 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:18:15.691458+0000", 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 507.29310799699999, 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91600119800000002, 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 73517 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691458+0000", 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691458+0000", 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691456+0000", 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691544+0000", 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7583999999999994e-05 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691558+0000", 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4141000000000001e-05 2024-08-17T17:26:43.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691564+0000", 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3330000000000002e-06 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691616+0000", 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1677e-05 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:15.691623+0000", 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4479999999999997e-06 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607426+0000", 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91580259200000003 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:18:16.607459+0000", 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2879999999999997e-05 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:130930 9.10 9:0f3273ea:::benchmark_data_smithi142_157452_object130929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.632116+0000", 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 456.35244920999997, 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.009814414, 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.019 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130930 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632116+0000", 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632116+0000", 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632115+0000", 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632293+0000", 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017897599999999999 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632310+0000", 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6932e-05 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.632318+0000", 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1160000000000001e-06 2024-08-17T17:26:43.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.634785+0000", 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024670349999999998 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.634794+0000", 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3039999999999994e-06 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.637459+0000", 2024-08-17T17:26:43.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.002664835 2024-08-17T17:26:43.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.637550+0000", 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1077999999999998e-05 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.637640+0000", 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0031999999999995e-05 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641657+0000", 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004017222 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641666+0000", 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7329999999999992e-06 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641843+0000", 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000177565 2024-08-17T17:26:43.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641894+0000", 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0785999999999998e-05 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641931+0000", 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6743999999999999e-05 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:130938 9.10 9:0e8a7329:::benchmark_data_smithi142_157452_object130937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.637870+0000", 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 456.34669599099999, 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.005439572, 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:43.028 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130938 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637870+0000", 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637870+0000", 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637868+0000", 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637959+0000", 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0857000000000001e-05 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637975+0000", 2024-08-17T17:26:43.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6239000000000002e-05 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.637982+0000", 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4250000000000003e-06 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.638046+0000", 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4596000000000003e-05 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.638058+0000", 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1234e-05 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.639258+0000", 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001200818 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.639345+0000", 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6588000000000005e-05 2024-08-17T17:26:43.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.639453+0000", 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107642 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641780+0000", 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002326846 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.641943+0000", 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000162941 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643245+0000", 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001302847 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643274+0000", 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8538e-05 2024-08-17T17:26:43.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643309+0000", 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5363e-05 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.45800.0:130971 9.10 9:0d797390:::benchmark_data_smithi142_157452_object130970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:19:06.668996+0000", 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 456.31556971499998, 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97444199300000001, 2024-08-17T17:26:43.035 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.45800", 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130971 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.668996+0000", 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.668996+0000", 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.668994+0000", 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669035+0000", 2024-08-17T17:26:43.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1689e-05 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669055+0000", 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0049000000000001e-05 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669063+0000", 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2690000000000002e-06 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669125+0000", 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2682999999999996e-05 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:06.669136+0000", 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0825999999999999e-05 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.640571+0000", 2024-08-17T17:26:43.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97143519 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.640640+0000", 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8612e-05 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.640712+0000", 2024-08-17T17:26:43.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2284000000000001e-05 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643217+0000", 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025048829999999998 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643226+0000", 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6920000000000008e-06 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643354+0000", 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012790800000000001 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643400+0000", 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6555999999999999e-05 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:19:07.643438+0000", 2024-08-17T17:26:43.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7691000000000002e-05 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116286 10.4 10:23b25cf9:::benchmark_data_smithi142_192453_object116285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:46.664127+0000", 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 176.320439064, 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98948754699999997, 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.043 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116286 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664127+0000", 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664127+0000", 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664125+0000", 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664186+0000", 2024-08-17T17:26:43.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0749999999999999e-05 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664200+0000", 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3976e-05 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664206+0000", 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7100000000000004e-06 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664250+0000", 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4048000000000002e-05 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664256+0000", 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0240000000000003e-06 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.634933+0000", 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97067708200000002 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.637306+0000", 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023726519999999998 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.637412+0000", 2024-08-17T17:26:43.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106477 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.646572+0000", 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091603299999999995 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.646581+0000", 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6759999999999996e-06 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653536+0000", 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069550829999999999 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653573+0000", 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6442000000000001e-05 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653614+0000", 2024-08-17T17:26:43.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1631000000000002e-05 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116345 10.4 10:20c60145:::benchmark_data_smithi142_192453_object116344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:46.732878+0000", 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 176.251688126, 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92081792600000001, 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.051 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116345 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732878+0000", 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732878+0000", 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732876+0000", 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732906+0000", 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9699000000000001e-05 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732921+0000", 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5262000000000001e-05 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732927+0000", 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6860000000000003e-06 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732999+0000", 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2241000000000002e-05 2024-08-17T17:26:43.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.733008+0000", 2024-08-17T17:26:43.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8179999999999993e-06 2024-08-17T17:26:43.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.640666+0000", 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90765785600000004 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.640812+0000", 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000145883 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.640981+0000", 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016897799999999999 2024-08-17T17:26:43.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.646599+0000", 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056188089999999998 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653485+0000", 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068855139999999997 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653651+0000", 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0001663 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653677+0000", 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5490000000000002e-05 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653695+0000", 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8845999999999999e-05 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:144825 10.1 10:8856b1b2:::benchmark_data_smithi142_192453_object19077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:35.865517+0000", 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 127.119049014, 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.157562472, 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.059 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144825 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865517+0000", 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865517+0000", 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865515+0000", 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865523+0000", 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9109999999999993e-06 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865537+0000", 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3396e-05 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865543+0000", 2024-08-17T17:26:43.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7519999999999999e-06 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865919+0000", 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037529399999999997 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.866059+0000", 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014045600000000001 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2024-08-17T17:26:43.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.866199+0000", 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013964500000000001 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.877260+0000", 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011061720000000001 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.883033+0000", 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057721539999999998 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.883042+0000", 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3689999999999996e-06 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.001960+0000", 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1189178639999999 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:26:43.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.002076+0000", 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116146 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.022819+0000", 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020743167999999999 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.023079+0000", 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025994599999999999 2024-08-17T17:26:43.066 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:144827 10.6 10:61040ce9:::benchmark_data_smithi142_192453_object19079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:35.893612+0000", 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 127.090953644, 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2032246719999999, 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.067 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144827 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893612+0000", 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893612+0000", 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893611+0000", 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893617+0000", 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.635e-06 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893629+0000", 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1505000000000001e-05 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893636+0000", 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1790000000000002e-06 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893679+0000", 2024-08-17T17:26:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2938999999999997e-05 2024-08-17T17:26:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893684+0000", 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4920000000000001e-06 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.089536+0000", 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1958518410000001 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.089632+0000", 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5643000000000006e-05 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.089722+0000", 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9754999999999997e-05 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.092998+0000", 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032764909999999999 2024-08-17T17:26:43.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.093681+0000", 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00068252099999999995 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.096625+0000", 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029443440000000002 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.096662+0000", 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7126000000000003e-05 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.096837+0000", 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017454999999999999 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179013 10.14 10:2fc210a1:::benchmark_data_smithi142_192453_object53265:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.761475+0000", 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 81.223090901000006, 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.061772216, 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179013 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.077 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761475+0000", 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761475+0000", 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761473+0000", 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761482+0000", 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6840000000000008e-06 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761497+0000", 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4990999999999999e-05 2024-08-17T17:26:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761505+0000", 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2289999999999997e-06 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090806+0000", 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32930165099999997 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090816+0000", 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6940000000000007e-06 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823175+0000", 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.73235919699999996 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823247+0000", 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1911999999999999e-05 2024-08-17T17:26:43.081 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179014 10.11 10:8e447741:::benchmark_data_smithi142_192453_object53266:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 81.223054383999994, 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0591031639999999, 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179014 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:26:43.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761514+0000", 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7099999999999999e-06 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761520+0000", 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.93e-06 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761523+0000", 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5159999999999999e-06 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.097043+0000", 2024-08-17T17:26:43.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33552037699999998 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.097050+0000", 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.820532+0000", 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72348204800000004 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.820614+0000", 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2399999999999997e-05 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179019 10.4 10:21f666e9:::benchmark_data_smithi142_192453_object53271:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 81.223038629000001, 2024-08-17T17:26:43.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0361314450000001, 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179019 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:26:43.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761529+0000", 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0609999999999998e-06 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761534+0000", 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2379999999999997e-06 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761537+0000", 2024-08-17T17:26:43.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3529999999999999e-06 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.079599+0000", 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31806203500000002 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.079607+0000", 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.661e-06 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.797577+0000", 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.71796951399999998 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.797658+0000", 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1886999999999997e-05 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179033 10.14 10:2d675116:::benchmark_data_smithi142_192453_object53285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.845669+0000", 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 81.138896466999995, 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98075968000000002, 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179033 2024-08-17T17:26:43.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845669+0000", 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845669+0000", 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845667+0000", 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845678+0000", 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.079e-05 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845701+0000", 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3144000000000001e-05 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845709+0000", 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8830000000000006e-06 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090830+0000", 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24512108099999999 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090834+0000", 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8659999999999999e-06 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823498+0000", 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.73266395200000001 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823608+0000", 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110109 2024-08-17T17:26:43.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823949+0000", 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034181199999999999 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.825713+0000", 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017633939999999999 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826128+0000", 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041489700000000001 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826350+0000", 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000222315 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826383+0000", 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3222000000000001e-05 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826429+0000", 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5630000000000002e-05 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183186 10.1 10:8272017f:::benchmark_data_smithi142_192453_object57438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.770927+0000", 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 75.213638298999996, 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.041528228, 2024-08-17T17:26:43.100 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183186 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770927+0000", 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770927+0000", 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770926+0000", 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770931+0000", 2024-08-17T17:26:43.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3939999999999998e-06 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770943+0000", 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1646e-05 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770948+0000", 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7930000000000003e-06 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770992+0000", 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3686999999999997e-05 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.771095+0000", 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102514 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.771464+0000", 2024-08-17T17:26:43.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036939699999999999 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.773319+0000", 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001855338 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.774672+0000", 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013531299999999999 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.797741+0000", 2024-08-17T17:26:43.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023068381999999998 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.797748+0000", 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6329999999999996e-06 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812380+0000", 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014632024 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812456+0000", 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5165000000000003e-05 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183207 10.1 10:8661facc:::benchmark_data_smithi142_192453_object57459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.812883+0000", 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 75.171682723999993, 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014551669, 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.107 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183207 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812883+0000", 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812883+0000", 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812882+0000", 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812886+0000", 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4920000000000004e-06 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812896+0000", 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0750000000000004e-06 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812899+0000", 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6500000000000002e-06 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812950+0000", 2024-08-17T17:26:43.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1099e-05 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812958+0000", 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4229999999999999e-06 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812477+0000", 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99951941 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812567+0000", 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0138999999999999e-05 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812990+0000", 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00042253899999999999 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.814513+0000", 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001523555 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827371+0000", 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012857337 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827392+0000", 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.088e-05 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827435+0000", 2024-08-17T17:26:43.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3016999999999998e-05 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183215 10.11 10:8c02d523:::benchmark_data_smithi142_192453_object57467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.818751+0000", 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 75.165815007000006, 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0085936609999999, 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.114 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183215 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818751+0000", 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818751+0000", 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818750+0000", 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818754+0000", 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.87e-06 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818765+0000", 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0712e-05 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818769+0000", 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1899999999999997e-06 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818812+0000", 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2886000000000002e-05 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818821+0000", 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.551e-06 2024-08-17T17:26:43.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.810908+0000", 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99208728599999996 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.811041+0000", 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013248399999999999 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.811251+0000", 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021044299999999999 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.813943+0000", 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026920339999999998 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827250+0000", 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013307401 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827290+0000", 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0039000000000001e-05 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827344+0000", 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3897999999999998e-05 2024-08-17T17:26:43.121 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183216 10.11 10:8e43b7c1:::benchmark_data_smithi142_192453_object57468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 75.164869287000002, 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0077557509999999, 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:43.122 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183216 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:26:43.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819700+0000", 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6070000000000001e-06 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819711+0000", 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0214000000000001e-05 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819716+0000", 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5670000000000002e-06 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819752+0000", 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5376999999999997e-05 2024-08-17T17:26:43.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819756+0000", 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8300000000000003e-06 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812388+0000", 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99263158200000001 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812498+0000", 2024-08-17T17:26:43.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110532 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812694+0000", 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019582900000000001 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.814418+0000", 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017239149999999999 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827382+0000", 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012964128 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827408+0000", 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5443999999999999e-05 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827452+0000", 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4357000000000002e-05 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.128 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.129 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.129 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:43.129 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:43.129 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:26:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:43.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:43.231 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:43.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:43.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:43 smithi142 ceph-mon[106729]: pgmap v898: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 714 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 16020/34311 objects degraded (46.691%); 2 B/s, 75 objects/s recovering 2024-08-17T17:26:43.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:43 smithi142 ceph-mon[103614]: pgmap v898: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 714 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 16020/34311 objects degraded (46.691%); 2 B/s, 75 objects/s recovering 2024-08-17T17:26:43.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:43.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:43.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:43.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:43 smithi152 ceph-mon[93456]: pgmap v898: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 714 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 16020/34311 objects degraded (46.691%); 2 B/s, 75 objects/s recovering 2024-08-17T17:26:43.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:43.647 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:26:43.647 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:26:43.647 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:26:43.647 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.489375 2024-08-17T17:26:43.647 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 1 0.4893746192739675 2024-08-17T17:26:43.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:43.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:43.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:43.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:43.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:43.973 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:44.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:26:44.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:44.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:44.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:44.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:44.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:44.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:44.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:44.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:44.530 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:44.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:44.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:44.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:44.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:44.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:44.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:44.796 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:44.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:44.898 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:44.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:45.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:45.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:45.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:45.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:45.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:45 smithi152 ceph-mon[93456]: pgmap v899: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 463 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 16020/22296 objects degraded (71.851%); 2 B/s, 110 objects/s recovering 2024-08-17T17:26:45.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:45.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:45.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:45.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:45 smithi142 ceph-mon[106729]: pgmap v899: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 463 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 16020/22296 objects degraded (71.851%); 2 B/s, 110 objects/s recovering 2024-08-17T17:26:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:45 smithi142 ceph-mon[103614]: pgmap v899: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 463 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 16020/22296 objects degraded (71.851%); 2 B/s, 110 objects/s recovering 2024-08-17T17:26:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:45.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:45.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:45.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:45.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:45.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:45.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:46.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:46.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:46.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:46.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.196 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:46.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:46.277 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:46.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:46.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:46.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:46.455 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16020/22296 objects degraded (71.851%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:46.455 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16020/22296 objects degraded (71.851%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:46.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:46.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:46.529 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16020/22296 objects degraded (71.851%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:247541 10.b 10:dabbafbe:::benchmark_data_smithi142_192453_object121793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.513330+0000", 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.014926119999999999, 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014957198999999999, 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:26:46.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 247541 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513330+0000", 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513330+0000", 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513328+0000", 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513335+0000", 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1600000000000003e-06 2024-08-17T17:26:46.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513349+0000", 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4379000000000001e-05 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513354+0000", 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4240000000000001e-06 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.516706+0000", 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003351733 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.516827+0000", 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121315 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.516967+0000", 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013922699999999999 2024-08-17T17:26:46.534 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:247537 10.e e274/222 10:7e495231:::benchmark_data_smithi142_192453_object121789:head v 274'15422, mlcod=274'15418)", 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.517743+0000", 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.010512264, 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010619777, 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.517743+0000", 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.517743+0000", 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.517743+0000", 2024-08-17T17:26:46.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.517747+0000", 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3390000000000003e-06 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.517753+0000", 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9819999999999996e-06 2024-08-17T17:26:46.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.517757+0000", 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3440000000000001e-06 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.518062+0000", 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00030525600000000001 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.518079+0000", 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6748999999999998e-05 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:247540 10.e e274/222 10:7bf39be6:::benchmark_data_smithi142_192453_object121792:head v 274'15423, mlcod=274'15419)", 2024-08-17T17:26:46.539 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.519615+0000", 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0086408860000000004, 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088033030000000002, 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519615+0000", 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519615+0000", 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519614+0000", 2024-08-17T17:26:46.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519620+0000", 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8320000000000002e-06 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519630+0000", 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0113e-05 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519635+0000", 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1440000000000002e-06 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.525050+0000", 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005414678 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.525073+0000", 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3037e-05 2024-08-17T17:26:46.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:247544 10.b 10.1cfafa7b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.519939+0000", 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0083166909999999993, 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085316809999999993, 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:46.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 247544 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519939+0000", 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519939+0000", 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519938+0000", 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519942+0000", 2024-08-17T17:26:46.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2690000000000001e-06 2024-08-17T17:26:46.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519953+0000", 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0293e-05 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.519958+0000", 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9729999999999997e-06 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.549 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:247548 10.17 10:ea759d66:::benchmark_data_smithi142_192453_object121800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-08-17T17:26:46.549 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.520749+0000", 2024-08-17T17:26:46.549 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0075069719999999998, 2024-08-17T17:26:46.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077745779999999999, 2024-08-17T17:26:46.549 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 247548 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520749+0000", 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520749+0000", 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520748+0000", 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:46.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520753+0000", 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6909999999999996e-06 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520762+0000", 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7529999999999996e-06 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520767+0000", 2024-08-17T17:26:46.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7550000000000002e-06 2024-08-17T17:26:46.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.521385+0000", 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00061747799999999997 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.521483+0000", 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8245000000000006e-05 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:247550 10.d 10:bd145cb8:::benchmark_data_smithi142_192453_object121802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.520779+0000", 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0074763479999999998, 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078110180000000003, 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:26:46.555 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 247550 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520779+0000", 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520779+0000", 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520779+0000", 2024-08-17T17:26:46.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520781+0000", 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.278e-06 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520787+0000", 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3029999999999999e-06 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520797+0000", 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0506000000000001e-05 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.520938+0000", 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014093100000000001 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.521060+0000", 2024-08-17T17:26:46.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121859 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.521176+0000", 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011654099999999999 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526524+0000", 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053480749999999999 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.560 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36309.0:247541 10.b e274/248)", 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.523839+0000", 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0044171820000000004, 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048213900000000001, 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.523839+0000", 2024-08-17T17:26:46.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.523839+0000", 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.523838+0000", 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:46.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.523844+0000", 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8409999999999996e-06 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.523859+0000", 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4888e-05 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.523865+0000", 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2160000000000003e-06 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:247533 10.d 10:b105e75b:::benchmark_data_smithi142_192453_object121785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.504268+0000", 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.023988206000000001, 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024441764000000001, 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.564 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 247533 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.504268+0000", 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.504268+0000", 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.504267+0000", 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.504269+0000", 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0260000000000002e-06 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.504274+0000", 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.994e-06 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.504277+0000", 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.05e-06 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513619+0000", 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093416000000000003 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513749+0000", 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013056100000000001 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.513897+0000", 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014799 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526517+0000", 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012620002999999999 2024-08-17T17:26:46.569 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36309.0:247541 10.b e274/248)", 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.526908+0000", 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001347974, 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018740390000000001, 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526908+0000", 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526908+0000", 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526907+0000", 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526912+0000", 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6240000000000001e-06 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526924+0000", 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1901e-05 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.526929+0000", 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9810000000000003e-06 2024-08-17T17:26:46.573 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:247553 10.7 10:e60598e5:::benchmark_data_smithi142_192453_object121805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.527080+0000", 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001175394, 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001748926, 2024-08-17T17:26:46.574 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 247553 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527080+0000", 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527080+0000", 2024-08-17T17:26:46.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527079+0000", 2024-08-17T17:26:46.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:46.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527085+0000", 2024-08-17T17:26:46.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9259999999999997e-06 2024-08-17T17:26:46.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527094+0000", 2024-08-17T17:26:46.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7450000000000001e-06 2024-08-17T17:26:46.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527098+0000", 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2309999999999998e-06 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:26:46.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527521+0000", 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00042284899999999999 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.527635+0000", 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011377799999999999 2024-08-17T17:26:46.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528232+0000", 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00059754399999999998 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.582 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.583 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36309.0:247533 10.d e274/248)", 2024-08-17T17:26:46.583 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.528875+0000", 2024-08-17T17:26:46.583 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9993806, 2024-08-17T17:26:46.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8546000000000001e-05, 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528875+0000", 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528875+0000", 2024-08-17T17:26:46.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528874+0000", 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528880+0000", 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3140000000000003e-06 2024-08-17T17:26:46.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528890+0000", 2024-08-17T17:26:46.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0936e-05 2024-08-17T17:26:46.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528896+0000", 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1320000000000002e-06 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36309.0:247550 10.d e274/248)", 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:46.528911+0000", 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9993453, 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3988000000000001e-05, 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:26:46.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528911+0000", 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528911+0000", 2024-08-17T17:26:46.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528910+0000", 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528912+0000", 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0669999999999999e-06 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528918+0000", 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2109999999999997e-06 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:26:46.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:46.528921+0000", 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0690000000000002e-06 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.592 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:26:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:26:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 12 2024-08-17T17:26:46.593 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:26:46.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:46.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:46.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:46.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:46.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:46.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:46.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:47.015 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:47.053 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:47.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:47.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:26:47.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:47.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:47.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:26:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.334 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:47.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:47.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:47.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:47.566 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:47 smithi142 ceph-mon[103614]: pgmap v900: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 430 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 16020/20688 objects degraded (77.436%); 0 B/s, 110 objects/s recovering 2024-08-17T17:26:47.567 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:47 smithi142 ceph-mon[106729]: pgmap v900: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 430 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 16020/20688 objects degraded (77.436%); 0 B/s, 110 objects/s recovering 2024-08-17T17:26:47.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:47 smithi152 ceph-mon[93456]: pgmap v900: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 430 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 16020/20688 objects degraded (77.436%); 0 B/s, 110 objects/s recovering 2024-08-17T17:26:47.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:47.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:47.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:47.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:47.822 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:47.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:47.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:47.985 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:48.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:48.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:48.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:48.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:48.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:48.226 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:26:48.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:48.362 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:48.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:48.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:48.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:48.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.636 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:48.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:48.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:48.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:48.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:48.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:49.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:49.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:49.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:49.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:49.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:49.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:49.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:49.442 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:49 smithi152 ceph-mon[93456]: pgmap v901: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 234 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 16020/11274 objects degraded (142.097%); 0 B/s, 101 objects/s recovering 2024-08-17T17:26:49.442 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1627944638' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675} : dispatch 2024-08-17T17:26:49.442 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675} : dispatch 2024-08-17T17:26:49.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:49.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:49.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:49.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:49 smithi142 ceph-mon[106729]: pgmap v901: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 234 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 16020/11274 objects degraded (142.097%); 0 B/s, 101 objects/s recovering 2024-08-17T17:26:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1627944638' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675} : dispatch 2024-08-17T17:26:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675} : dispatch 2024-08-17T17:26:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:49 smithi142 ceph-mon[103614]: pgmap v901: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 234 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 16020/11274 objects degraded (142.097%); 0 B/s, 101 objects/s recovering 2024-08-17T17:26:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1627944638' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675} : dispatch 2024-08-17T17:26:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675} : dispatch 2024-08-17T17:26:49.639 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.1 primary-affinity to 0.489375 (8320712) 2024-08-17T17:26:49.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:49.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:49.839 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:49.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:49.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:50.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:50.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:50.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:50.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:50.192 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:26:50.192 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:26:50.192 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:26:50.192 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:26:50.192 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:26:50.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:50.301 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:26:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:26:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:50.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:50.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:50.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:50.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:50.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:50.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675}]': finished 2024-08-17T17:26:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:50 smithi152 ceph-mon[93456]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:26:50.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:50.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675}]': finished 2024-08-17T17:26:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:50 smithi142 ceph-mon[106729]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:26:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.4893746192739675}]': finished 2024-08-17T17:26:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:50 smithi142 ceph-mon[103614]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:26:50.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:50.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:50.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:50.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:50.867 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 125745 objects 2024-08-17T17:26:50.867 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :169.263 2024-08-17T17:26:50.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:50.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.007 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:51.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:51.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:51.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:51.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:51.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:51.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:51.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:51.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:51.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:51 smithi152 ceph-mon[93456]: pgmap v903: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 168 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 16020/8106 objects degraded (197.631%); 0 B/s, 104 objects/s recovering 2024-08-17T17:26:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:51 smithi152 ceph-mon[93456]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:26:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16020/8106 objects degraded (197.631%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:51.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-mon[106729]: pgmap v903: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 168 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 16020/8106 objects degraded (197.631%); 0 B/s, 104 objects/s recovering 2024-08-17T17:26:51.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-mon[106729]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:26:51.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16020/8106 objects degraded (197.631%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:51.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-mon[103614]: pgmap v903: 99 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 87 active+clean; 168 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 16020/8106 objects degraded (197.631%); 0 B/s, 104 objects/s recovering 2024-08-17T17:26:51.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-mon[103614]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:26:51.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16020/8106 objects degraded (197.631%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:51.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:51.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:26:51.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:51.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:51.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:51.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:51.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:26:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:26:51] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.51.0" 2024-08-17T17:26:51.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.883 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:51.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:51.975 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:51.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:52.007 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:26:52.008 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:52.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:52.009 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:52.029 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:26:52.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:52.048 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:26:52.049 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:52.066 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:26:52.066 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:52.087 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:26:52.088 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:52.110 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:26:52.110 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:52.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:52.131 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:26:52.132 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:52.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:52.150 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:26:52.151 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:52.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.257 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:52.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:52.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:52.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:52.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[106729]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1022326872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[106729]: pgmap v906: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 86 active+clean; 152 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 15386/7350 objects degraded (209.333%); 107 KiB/s, 84 objects/s recovering 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[106729]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[103614]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1022326872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[103614]: pgmap v906: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 86 active+clean; 152 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 15386/7350 objects degraded (209.333%); 107 KiB/s, 84 objects/s recovering 2024-08-17T17:26:52.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:52 smithi142 ceph-mon[103614]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:26:52.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:52 smithi152 ceph-mon[93456]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:26:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1022326872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:52 smithi152 ceph-mon[93456]: pgmap v906: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 86 active+clean; 152 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 15386/7350 objects degraded (209.333%); 107 KiB/s, 84 objects/s recovering 2024-08-17T17:26:52.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:52 smithi152 ceph-mon[93456]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:26:52.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:52.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:52.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:52.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:52.862 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-08-17T17:26:52.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-08-17T17:26:52.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:52.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:52.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:53.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:53.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:53.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:53.134 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:53.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:53.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:53.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:53.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:53.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:53.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:53.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:53.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:53.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:53.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:53.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:53.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:54.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:54.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:54.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:54.151 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:54.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:54.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:54.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:54.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:54.469 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:54.469 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:54.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:54.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:54.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:54.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:54.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:54.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:54.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:54.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:54.976 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:54.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:55.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:55.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:55.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:55.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:55.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:26:55.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:55.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:55.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:55.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:55.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:55 smithi142 ceph-mon[106729]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 86 active+clean; 25 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 14419/1239 objects degraded (1163.761%); 390 KiB/s, 134 objects/s recovering 2024-08-17T17:26:55.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:55 smithi142 ceph-mon[103614]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 86 active+clean; 25 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 14419/1239 objects degraded (1163.761%); 390 KiB/s, 134 objects/s recovering 2024-08-17T17:26:55.468 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:55.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.546 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:26:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:55 smithi152 ceph-mon[93456]: pgmap v908: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 86 active+clean; 25 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 14419/1239 objects degraded (1163.761%); 390 KiB/s, 134 objects/s recovering 2024-08-17T17:26:55.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:55.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:55.720 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:55.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:55.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:55.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:55.944 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:26:55.944 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:26:55.945 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-08-17T17:26:55.945 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:26:55.945 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:26:55.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-08-17T17:26:55.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:55.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:56.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:56.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:56.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:56.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:56.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.362 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:56.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:56.381 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:56.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:56.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:56.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:56.487 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:26:56.487 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1852972982' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14419/1239 objects degraded (1163.761%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[103614]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1852972982' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14419/1239 objects degraded (1163.761%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:56.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:56.489 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:56 smithi142 ceph-mon[106729]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1852972982' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14419/1239 objects degraded (1163.761%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:56 smithi152 ceph-mon[93456]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:26:56.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:56.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:56.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:56.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:56.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:56.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:56.859 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:56.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:56.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:56.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:57.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:57.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:57.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:57.230 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:57.332 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:57.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:57.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:57.478 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:57 smithi152 ceph-mon[93456]: pgmap v911: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 23 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 12728/1143 objects degraded (1113.561%); 262 KiB/s, 531 objects/s recovering 2024-08-17T17:26:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:57 smithi152 ceph-mon[93456]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:26:57.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:57.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:57.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[106729]: pgmap v911: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 23 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 12728/1143 objects degraded (1113.561%); 262 KiB/s, 531 objects/s recovering 2024-08-17T17:26:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[106729]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:26:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[103614]: pgmap v911: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 23 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 12728/1143 objects degraded (1113.561%); 262 KiB/s, 531 objects/s recovering 2024-08-17T17:26:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:26:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:57 smithi142 ceph-mon[103614]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:26:57.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:57.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:57.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:57.830 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:57.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:57.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:57.975 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:58.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:58.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:58.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:58.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:58.262 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_5' removed 2024-08-17T17:26:58.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:58.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:58.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2161744123' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:26:58.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2161744123' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:26:58.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:58.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:58.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2161744123' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:26:58.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:58.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:58.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:58.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:58.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:58.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:58.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:58.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:58.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:58.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:59.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:59.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:59.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:59.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:59.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:59.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:59.342 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:59.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[106729]: pgmap v913: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 23 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 11002/1143 objects degraded (962.555%); 217 KiB/s, 692 objects/s recovering 2024-08-17T17:26:59.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2161744123' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:26:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[106729]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:26:59.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2523866125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[103614]: pgmap v913: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 23 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 11002/1143 objects degraded (962.555%); 217 KiB/s, 692 objects/s recovering 2024-08-17T17:26:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2161744123' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:26:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[103614]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:26:59.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:26:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2523866125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:59.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:59.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:26:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:59.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:59 smithi152 ceph-mon[93456]: pgmap v913: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 23 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 11002/1143 objects degraded (962.555%); 217 KiB/s, 692 objects/s recovering 2024-08-17T17:26:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2161744123' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:26:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:59 smithi152 ceph-mon[93456]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:26:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:26:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2523866125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:59.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:59.674 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:26:59.674 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:26:59.675 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-08-17T17:26:59.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_6 16 2024-08-17T17:26:59.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:59.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:59.768 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:26:59.768 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:26:59.768 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:26:59.768 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:26:59.768 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:26:59.768 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:59.769 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:59.770 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:26:59.770 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:26:59.770 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:59.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:59.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:59.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:59.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:59.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:26:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:00.008 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:00.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:00.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:00.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:00.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:00.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:00.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:00.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:00 smithi142 ceph-mon[106729]: pgmap v915: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:00.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:00 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11002/1143 objects degraded (962.555%), 7 pgs degraded, 1 pg undersized) 2024-08-17T17:27:00.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:00 smithi142 ceph-mon[103614]: pgmap v915: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:00.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:00 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11002/1143 objects degraded (962.555%), 7 pgs degraded, 1 pg undersized) 2024-08-17T17:27:00.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:00.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:00.550 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:27:00.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:00 smithi152 ceph-mon[93456]: pgmap v915: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:00 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11002/1143 objects degraded (962.555%), 7 pgs degraded, 1 pg undersized) 2024-08-17T17:27:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:27:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:27:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:00.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:00.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:00.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:00.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:00.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:00.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:00.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:01.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:01.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:01.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:01.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:01.342 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:01.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:01.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:01.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:01.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:01.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:01.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:01.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:01.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:01.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:01.837 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:01.857 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:01.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:01.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:01.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:01.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:01.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:02.025 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:27:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:27:01] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:27:02.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:02.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:02.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:02.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:02.235 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:02.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:02.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:02.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:02.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:02.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:02.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:02.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:02.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:02.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:02.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:02.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.999 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:03.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:03.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:03.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:03.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:03.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:03.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:03.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:03.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.355 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:03.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:03.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:03 smithi142 ceph-mon[106729]: pgmap v916: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:03.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:03 smithi142 ceph-mon[103614]: pgmap v916: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:03.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:03.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:03 smithi152 ceph-mon[93456]: pgmap v916: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:03.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:03.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:03.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:03.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:03.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:03.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:03.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:03.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:03.968 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:03.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:03.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:04.077 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:04.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:04.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:04.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:04.290 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_6' created 2024-08-17T17:27:04.334 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:04.357 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:04.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:04.464 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1433315218' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:27:04.464 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:27:04.465 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1433315218' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:27:04.465 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:27:04.485 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:04.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:04.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1433315218' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:27:04.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:27:04.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:04.609 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:04.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:04.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:04.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:04.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:04.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:04.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:04.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:04.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:05.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:05.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:05.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:05.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:05.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:05.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:05.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:05.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:05.466 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:05.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:05.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:05.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:05 smithi152 ceph-mon[93456]: pgmap v917: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:05.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:05 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-08-17T17:27:05.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:05 smithi152 ceph-mon[93456]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:27:05.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:05.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:05 smithi142 ceph-mon[106729]: pgmap v917: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:05 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-08-17T17:27:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:05 smithi142 ceph-mon[106729]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:27:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:05 smithi142 ceph-mon[103614]: pgmap v917: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:05 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-08-17T17:27:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:05 smithi142 ceph-mon[103614]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:27:05.637 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:27:05.637 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:27:05.637 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:27:05.637 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:27:05.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:05.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:05.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:05.838 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:05.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:05.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-08-17T17:27:05.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:05.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:06.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:06.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:06.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:06.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:06.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:06.303 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:27:06.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:06.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:06.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.478 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:06.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:06.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:06 smithi152 ceph-mon[93456]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:27:06.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1820447787' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:06.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:06 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:06.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:06.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:06.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:06 smithi142 ceph-mon[106729]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:27:06.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1820447787' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:06.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:06 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:06 smithi142 ceph-mon[103614]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:27:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1820447787' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:06 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:06.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:06.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:06.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:06.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:06.848 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:06.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:06.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:06.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:07.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:07.096 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:07.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:07.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:07.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:07.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:07.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:27:07.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:07.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:07.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:07.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:07.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:07.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:07.518 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:27:07.518 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:27:07.518 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:27:07.518 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:07 smithi152 ceph-mon[93456]: pgmap v920: 89 pgs: 9 creating+peering, 6 unknown, 74 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:07.518 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:07 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:27:07.518 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:07 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:07.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:07 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:07.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:07 smithi152 ceph-mon[93456]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:27:07.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:07 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:07.532 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:27:07.533 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:07.539 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:07.543 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:27:07.543 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[106729]: pgmap v920: 89 pgs: 9 creating+peering, 6 unknown, 74 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[106729]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[103614]: pgmap v920: 89 pgs: 9 creating+peering, 6 unknown, 74 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:07.544 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[103614]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:27:07.545 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:07 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:07.561 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:27:07.561 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:07.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:07.583 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:27:07.583 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:07.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:07.599 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:27:07.599 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:07.614 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:27:07.614 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:07.629 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:27:07.630 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:07.644 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:27:07.645 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:07.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:07.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:07.816 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:07.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:07.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:07.881 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:07.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:07.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:08.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:08.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:08.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:08.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:08.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:08.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:08.331 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:08.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3449112781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:08.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3449112781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:08.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:08.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:08.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:08.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3449112781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:08.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:08.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:08.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:08.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:08.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:08.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:08.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:08.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:08.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:09.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:09.072 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:09.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:09.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:09.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:09.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:09.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:27:09.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:09 smithi142 ceph-mon[106729]: pgmap v922: 89 pgs: 9 creating+peering, 3 unknown, 77 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:09.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:09 smithi142 ceph-mon[103614]: pgmap v922: 89 pgs: 9 creating+peering, 3 unknown, 77 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:09.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:09.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:09 smithi152 ceph-mon[93456]: pgmap v922: 89 pgs: 9 creating+peering, 3 unknown, 77 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:09.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:09.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:09.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:09.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:09.692 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:09.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:09.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:09.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.941 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:09.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:10.014 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:10.014 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:10.022 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:10.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:10.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:10.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:10.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:10.186 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:10.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:10.311 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:10.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:10.318 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-08-17T17:27:10.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:10.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:10.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:10.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:10.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:10.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:10.564 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/862936878' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:27:10.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:10 smithi152 ceph-mon[93456]: pgmap v923: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:10.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:27:10.565 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:27:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:27:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:10.577 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/862936878' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:27:10.577 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:10 smithi142 ceph-mon[106729]: pgmap v923: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:10.577 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:27:10.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/862936878' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:27:10.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:10 smithi142 ceph-mon[103614]: pgmap v923: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:10.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:27:10.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:27:10.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:10.682 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:10.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:10.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:10.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:10.846 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:10.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:10.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:10.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:11.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:11.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:11.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:11.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:11.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:11.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:11.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:11.299 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:11.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:27:11.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:11 smithi142 ceph-mon[103614]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:27:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:11 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:27:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:11 smithi142 ceph-mon[106729]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:27:11.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:11.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:11.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:11.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:11.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:11.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:11 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:27:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:11 smithi152 ceph-mon[93456]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:27:11.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:11.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:11.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:11.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:11.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:11.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:11.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:12.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:12.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:27:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:27:11] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-08-17T17:27:12.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:12.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:12.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.294 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:12.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:12.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:12.416 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:12.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:12.430 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:27:12.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:12.524 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:12.524 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:12.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:12.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:12.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:12.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:12.684 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:12.684 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:12.684 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[106729]: pgmap v925: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:12.684 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2777510094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:12.684 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:12.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:12.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:12.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[103614]: pgmap v925: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:12.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2777510094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:12.685 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:12.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:12.785 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:12 smithi152 ceph-mon[93456]: pgmap v925: 89 pgs: 89 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2777510094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:12.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:12.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:13.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:13.032 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:13.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:13.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:13.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:13.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:13.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:27:13.180 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:13.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:13.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:13.336 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:27:13.337 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:27:13.337 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-17T17:27:13.337 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:27:13.337 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:27:13.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-17T17:27:13.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:13.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:13.469 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:13.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:13.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:13.568 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-08-17T17:27:13.572 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:27:13.572 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:27:13.618 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:27:13.618 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:27:13.618 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_227468 2024-08-17T17:27:13.618 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:13.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:27:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:27:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[106729]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:27:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:27:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:13 smithi142 ceph-mon[103614]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:27:13.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:13.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:13.739 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:13 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:27:13.739 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:13 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:13.739 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:13 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:13.739 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:13 smithi152 ceph-mon[93456]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:27:13.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:13.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:13.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:13.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:13.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:14.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:14.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:14.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:14.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:14.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:14.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:14.358 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:14.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:14.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:14.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:14.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:14.618 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 4005 3989 249.299 249.312 0.00269872 0.00399337 2024-08-17T17:27:14.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:14.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:14 smithi152 ceph-mon[93456]: pgmap v927: 89 pgs: 89 active+clean; 3.0 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 316 KiB/s wr, 4 op/s 2024-08-17T17:27:14.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:14.749 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:14 smithi142 ceph-mon[103614]: pgmap v927: 89 pgs: 89 active+clean; 3.0 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 316 KiB/s wr, 4 op/s 2024-08-17T17:27:14.749 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:14 smithi142 ceph-mon[106729]: pgmap v927: 89 pgs: 89 active+clean; 3.0 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 316 KiB/s wr, 4 op/s 2024-08-17T17:27:14.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:14.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:14.913 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:14.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:15.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:15.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:15.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:15.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:27:15.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:15.203 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:15.228 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:15.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:15.277 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:27:15.277 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:15.315 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:27:15.315 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:15.324 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:15.324 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:15.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:15.350 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:27:15.350 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:15.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:15.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:15.386 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:27:15.387 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:15.421 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:27:15.421 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:15.452 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:27:15.452 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:15.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:15.485 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:27:15.486 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:15.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:15.521 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:27:15.522 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:15.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:15.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 7128 7112 222.204 195.188 0.00597645 0.00449095 2024-08-17T17:27:15.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/804270443' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:15.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/804270443' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:15.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:15.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/804270443' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:15.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:15.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:15.909 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:16.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:16.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:27:16.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:16.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:16.188 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:16.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:16.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:16.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:16.447 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:16.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:16.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:16.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:16.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:16.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 10480 10464 217.964 209.5 0.00358234 0.00457942 2024-08-17T17:27:16.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:16.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:16.746 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:16 smithi152 ceph-mon[93456]: pgmap v928: 89 pgs: 89 active+clean; 186 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s 2024-08-17T17:27:16.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:16 smithi142 ceph-mon[103614]: pgmap v928: 89 pgs: 89 active+clean; 186 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s 2024-08-17T17:27:16.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:16 smithi142 ceph-mon[106729]: pgmap v928: 89 pgs: 89 active+clean; 186 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 371 op/s 2024-08-17T17:27:16.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:16.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:16.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:17.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:17.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:17.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:17.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:17.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:17.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:17.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:17.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:17.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:17.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:17.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:17.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 13739 13723 214.39 203.688 0.00793409 0.00465737 2024-08-17T17:27:17.653 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:17.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:17.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:17.808 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:17.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:17.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:17.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:17.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:18.004 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:17 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:18.004 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:17 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:18.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:17 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:18.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:18.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:18.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:18.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:18.350 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:27:18.351 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:27:18.351 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:27:18.351 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:27:18.351 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:27:18.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:18.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:18.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:18.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 15 16498 16483 206.01 172.5 0.00509235 0.00484901 2024-08-17T17:27:18.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:18.662 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:18.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:18.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:18.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:18.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:18.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:19.018 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:18 smithi142 ceph-mon[103614]: pgmap v929: 89 pgs: 89 active+clean; 354 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s 2024-08-17T17:27:19.018 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:18 smithi142 ceph-mon[106729]: pgmap v929: 89 pgs: 89 active+clean; 354 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s 2024-08-17T17:27:19.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:19.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:18 smithi152 ceph-mon[93456]: pgmap v929: 89 pgs: 89 active+clean; 354 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s 2024-08-17T17:27:19.075 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:19.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:19.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:27:19.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:19.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:19.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:19.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:19.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:19.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:19.558 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:19.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:19.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 18944 18928 197.141 152.812 0.00513505 0.00506699 2024-08-17T17:27:19.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:19.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:19.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:19.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:19.894 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:27:19.895 INFO:teuthology.orchestra.run.smithi142.stdout: "success": "" 2024-08-17T17:27:19.895 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:27:19.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:20.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:20.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:20.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:20.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:20.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:20.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:20.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:20.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:20.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:27:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:27:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:20.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 21626 21610 192.923 167.625 0.0024164 0.00517648 2024-08-17T17:27:20.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:20.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:20.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:20.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:20.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:20.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:21.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:21.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:21.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:21.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:21.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:21.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:21.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:21.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:21.420 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:21 smithi152 ceph-mon[93456]: pgmap v930: 89 pgs: 89 active+clean; 905 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:27:21.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:21.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:21.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:21.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 24241 24225 189.236 163.438 0.00605095 0.00527871 2024-08-17T17:27:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:21 smithi142 ceph-mon[103614]: pgmap v930: 89 pgs: 89 active+clean; 905 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:27:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:21 smithi142 ceph-mon[106729]: pgmap v930: 89 pgs: 89 active+clean; 905 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:27:21.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:21.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:21.709 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:21.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:21.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:21.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:21.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:22.005 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:27:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:27:21] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-08-17T17:27:22.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:22.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:22.116 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:27:22.116 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:27:22.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236374 10.c e272/271 10:31cd58ca:::benchmark_data_smithi142_192453_object110626:head v 272'14874, mlcod=0'0)", 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.511116+0000", 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.603472003999997, 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.100306045, 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.511116+0000", 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.511116+0000", 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.511116+0000", 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:22.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.511122+0000", 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.054e-06 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.511131+0000", 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7110000000000007e-06 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.511138+0000", 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2389999999999999e-06 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.523320+0000", 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012182246000000001 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.523340+0000", 2024-08-17T17:27:22.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9987999999999999e-05 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611386+0000", 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.088046489000000006 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611422+0000", 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6015e-05 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.121 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236396 10.c e272/271 10:394b0712:::benchmark_data_smithi142_192453_object110648:head v 272'14875, mlcod=0'0)", 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.513933+0000", 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.600655218, 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.097514292000000002, 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.513933+0000", 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.513933+0000", 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.513932+0000", 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.513948+0000", 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6130999999999999e-05 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.513960+0000", 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1566000000000001e-05 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.513965+0000", 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5400000000000003e-06 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.529906+0000", 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015940651 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.529925+0000", 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9162000000000001e-05 2024-08-17T17:27:22.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611431+0000", 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.081505460000000002 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611447+0000", 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6583999999999999e-05 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.126 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236406 10.c e272/271 10:3d21c5b4:::benchmark_data_smithi142_192453_object110658:head v 272'14876, mlcod=246'8583)", 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.520731+0000", 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.593857446000001, 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.090733003000000007, 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.520731+0000", 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.520731+0000", 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.520730+0000", 2024-08-17T17:27:22.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.520739+0000", 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2119999999999992e-06 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.520754+0000", 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4707e-05 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.520761+0000", 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4889999999999998e-06 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536146+0000", 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015385088 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536171+0000", 2024-08-17T17:27:22.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4879999999999999e-05 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611450+0000", 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.075278510000000007 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611464+0000", 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4119e-05 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.131 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236415 10.c e272/271 10:34af8f9f:::benchmark_data_smithi142_192453_object110667:head v 272'14877, mlcod=246'8583)", 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.525360+0000", 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.589228532999996, 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.091394650999999993, 2024-08-17T17:27:22.132 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.133 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.133 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.525360+0000", 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.525360+0000", 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.525359+0000", 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.525367+0000", 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4989999999999994e-06 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.525381+0000", 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3549e-05 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.525388+0000", 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2479999999999999e-06 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591698+0000", 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.066310392999999995 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591722+0000", 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3227999999999999e-05 2024-08-17T17:27:22.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.616708+0000", 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024985825999999999 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.616754+0000", 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6798000000000002e-05 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.137 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236438 10.9 e272/271 10:97f20084:::benchmark_data_smithi142_192453_object110690:head v 272'12948, mlcod=260'11250)", 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.534359+0000", 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.580229002999999, 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.057339235000000002, 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534359+0000", 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534359+0000", 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534358+0000", 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534368+0000", 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4790000000000002e-06 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534382+0000", 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4187e-05 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534389+0000", 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2590000000000003e-06 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534456+0000", 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7119000000000006e-05 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.534479+0000", 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3274000000000001e-05 2024-08-17T17:27:22.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591670+0000", 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.057190316999999997 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591698+0000", 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8696000000000001e-05 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.142 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236440 10.11 e272/271 10:89ed8f1c:::benchmark_data_smithi142_192453_object110692:head v 272'13169, mlcod=260'11439)", 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.536516+0000", 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.578072251000002, 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.055200115000000001, 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536516+0000", 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536516+0000", 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536515+0000", 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536523+0000", 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7100000000000007e-06 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536535+0000", 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2300000000000001e-05 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536541+0000", 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.823e-06 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536765+0000", 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022430700000000001 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.536786+0000", 2024-08-17T17:27:22.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0874e-05 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591701+0000", 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.054915481000000002 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591716+0000", 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4652e-05 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.147 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236441 10.19 e272/271 10:9dfe1020:::benchmark_data_smithi142_192453_object110693:head v 272'12889, mlcod=260'11253)", 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.538014+0000", 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.576573975999999, 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.060372351999999997, 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.538014+0000", 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.538014+0000", 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.538013+0000", 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.538022+0000", 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1880000000000002e-06 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.538034+0000", 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2326e-05 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591665+0000", 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.053631307000000003 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591708+0000", 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2388999999999997e-05 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591727+0000", 2024-08-17T17:27:22.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9766999999999999e-05 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.598369+0000", 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066413929999999998 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.598387+0000", 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7872000000000001e-05 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.152 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:236435 10.c e272/271 10:3c05ffcd:::benchmark_data_smithi142_192453_object110687:head v 272'14878, mlcod=246'8583)", 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:33.539777+0000", 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.574811117000003, 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.077002301999999995, 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.539777+0000", 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.539777+0000", 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.539776+0000", 2024-08-17T17:27:22.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.539785+0000", 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0040000000000005e-06 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591673+0000", 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.051888020999999999 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.591682+0000", 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2040000000000003e-06 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611495+0000", 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019812434 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.611523+0000", 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7940999999999999e-05 2024-08-17T17:27:22.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.616759+0000", 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052366449999999998 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:33.616779+0000", 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0231999999999999e-05 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.157 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:241150 10.c e273/271 10:3c8c39da:::benchmark_data_smithi142_192453_object115402:head v 273'15151, mlcod=246'8583)", 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:39.357155+0000", 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 42.757433265000003, 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039284564000000001, 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.357155+0000", 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.357155+0000", 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.357154+0000", 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.357162+0000", 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2409999999999995e-06 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.357179+0000", 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6438000000000001e-05 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.357186+0000", 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9920000000000001e-06 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.360085+0000", 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028995650000000002 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.360100+0000", 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4464000000000001e-05 2024-08-17T17:27:22.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.396337+0000", 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036237254000000003 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:39.396440+0000", 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102608 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:245397 10.c e274/271 10:3cd3b41a:::benchmark_data_smithi142_192453_object119649:head v 274'15430, mlcod=246'8583)", 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:44.052277+0000", 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.062311455, 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042424420999999997, 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.052277+0000", 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.052277+0000", 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.052276+0000", 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.052282+0000", 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.925e-06 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.052290+0000", 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5440000000000001e-06 2024-08-17T17:27:22.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.052293+0000", 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5159999999999999e-06 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.056981+0000", 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046883990000000002 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.057006+0000", 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4743999999999999e-05 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.094657+0000", 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037651153999999999 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:44.094701+0000", 2024-08-17T17:27:22.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3841999999999998e-05 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:248469 10.c e274/271 10:3db6147e:::benchmark_data_smithi142_192453_object122721:head v 274'15604, mlcod=246'8583)", 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:47.463047+0000", 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.651541055999999, 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046543275000000002, 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.168 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463047+0000", 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463047+0000", 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463046+0000", 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463054+0000", 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5460000000000003e-06 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463071+0000", 2024-08-17T17:27:22.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7022000000000001e-05 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463077+0000", 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.511e-06 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463283+0000", 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000205183 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463340+0000", 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7212e-05 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.509453+0000", 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046113267999999999 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.509590+0000", 2024-08-17T17:27:22.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013729700000000001 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:248471 10.c e274/271 10:3a3ff002:::benchmark_data_smithi142_192453_object122723:head v 274'15605, mlcod=246'8583)", 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:47.463091+0000", 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.651497245000002, 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046626402999999997, 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463091+0000", 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463091+0000", 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463091+0000", 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463095+0000", 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0489999999999997e-06 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463101+0000", 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.562e-06 2024-08-17T17:27:22.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.463105+0000", 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9759999999999997e-06 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.471248+0000", 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081423880000000004 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.471304+0000", 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6677e-05 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.509594+0000", 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038289801999999998 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.509717+0000", 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012323600000000001 2024-08-17T17:27:22.177 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36309.0:248473 10.c e274/271 10:35965a8f:::benchmark_data_smithi142_192453_object122725:head v 274'15606, mlcod=246'8583)", 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:47.467830+0000", 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.646757802000003, 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.04873856, 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.467830+0000", 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.467830+0000", 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.467830+0000", 2024-08-17T17:27:22.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.467839+0000", 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9910000000000008e-06 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.467886+0000", 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7503000000000002e-05 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.467893+0000", 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8299999999999998e-06 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.479514+0000", 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01162121 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.479537+0000", 2024-08-17T17:27:22.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3213e-05 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.516428+0000", 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036890436999999998 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:47.516569+0000", 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014127500000000001 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.182 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:16429 11.f 11:faf076b3:::benchmark_data_smithi142_227468_object16428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:18.558672+0000", 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.5559160190000001, 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039649338999999999, 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:27:22.183 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 16429 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558672+0000", 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558672+0000", 2024-08-17T17:27:22.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558671+0000", 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558725+0000", 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3647000000000003e-05 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558736+0000", 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1616e-05 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558742+0000", 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6729999999999997e-06 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558864+0000", 2024-08-17T17:27:22.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012170700000000001 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.558972+0000", 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108179 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559105+0000", 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000132607 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559742+0000", 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000637786 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.560746+0000", 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010039300000000001 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.598234+0000", 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037487254999999997 2024-08-17T17:27:22.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.598287+0000", 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3062999999999997e-05 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.598322+0000", 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4864999999999999e-05 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.189 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:16432 11.4 11:2fc3ff03:::benchmark_data_smithi142_227468_object16431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:18.559391+0000", 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.5551972469999997, 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039591999000000003, 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 16432 2024-08-17T17:27:22.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559391+0000", 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559391+0000", 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559390+0000", 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559463+0000", 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2873999999999994e-05 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559473+0000", 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7359999999999996e-06 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559477+0000", 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9600000000000002e-06 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559517+0000", 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0111000000000003e-05 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.559629+0000", 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111972 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.560101+0000", 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00047184400000000002 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.560850+0000", 2024-08-17T17:27:22.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074958000000000002 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.561482+0000", 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00063205399999999995 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.598908+0000", 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037425420000000001 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.598952+0000", 2024-08-17T17:27:22.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4075000000000001e-05 2024-08-17T17:27:22.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:18.598983+0000", 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1117e-05 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:20002 11.f 11:f81b89ed:::benchmark_data_smithi142_227468_object20001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:19.994146+0000", 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.120442653, 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046686515999999997, 2024-08-17T17:27:22.197 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20002 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994146+0000", 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994146+0000", 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994145+0000", 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994184+0000", 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9780000000000002e-05 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994196+0000", 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2198000000000001e-05 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994202+0000", 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0140000000000004e-06 2024-08-17T17:27:22.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994342+0000", 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013934700000000001 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994522+0000", 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000180018 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:22.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.994690+0000", 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000168467 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995649+0000", 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00095839799999999995 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040706+0000", 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.045057 2024-08-17T17:27:22.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040735+0000", 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9366e-05 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040791+0000", 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.622e-05 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040832+0000", 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0774e-05 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:20005 11.5 11:a89eaf17:::benchmark_data_smithi142_227468_object20004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:19.995752+0000", 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.1188360670000002, 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046736919000000002, 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.204 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20005 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995752+0000", 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995752+0000", 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995751+0000", 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995803+0000", 2024-08-17T17:27:22.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2098000000000002e-05 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995815+0000", 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1402e-05 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995820+0000", 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7350000000000001e-06 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995859+0000", 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8923e-05 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.995962+0000", 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103047 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.996084+0000", 2024-08-17T17:27:22.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012171 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.041369+0000", 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.045284778999999997 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.041407+0000", 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7706000000000001e-05 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.042412+0000", 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010049989999999999 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.042457+0000", 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5955999999999998e-05 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.042489+0000", 2024-08-17T17:27:22.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1578999999999997e-05 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36603.0:20002 11.f e287/283 ondisk, result = 0)", 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:19.997087+0000", 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.1175016179999999, 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043750684999999997, 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997087+0000", 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997087+0000", 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997086+0000", 2024-08-17T17:27:22.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997090+0000", 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3290000000000004e-06 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040688+0000", 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043597576999999998 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040695+0000", 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7939999999999997e-06 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040726+0000", 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1229999999999997e-05 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040731+0000", 2024-08-17T17:27:22.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7380000000000002e-06 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040837+0000", 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106635 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.215 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36603.0:20005 11.5 e287/283 ondisk, result = 0)", 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:19.997245+0000", 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.1173429060000002, 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.044163101000000003, 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997245+0000", 2024-08-17T17:27:22.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997245+0000", 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997245+0000", 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997248+0000", 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7450000000000001e-06 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040757+0000", 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043508719000000001 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040766+0000", 2024-08-17T17:27:22.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6580000000000008e-06 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.041399+0000", 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00063328199999999999 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.041403+0000", 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9550000000000002e-06 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.041408+0000", 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3369999999999999e-06 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36603.0:20002 11.f e287/283 ondisk, result = 0)", 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:19.997761+0000", 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.1168269949999998, 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042947122999999997, 2024-08-17T17:27:22.220 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997761+0000", 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997761+0000", 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997761+0000", 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:19.997768+0000", 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9330000000000004e-06 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:27:22.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040345+0000", 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042576900000000001 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040364+0000", 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9834999999999999e-05 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040691+0000", 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032685100000000002 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040699+0000", 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6170000000000001e-06 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:20.040708+0000", 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5380000000000008e-06 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:27:22.224 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:27:22.225 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:27:22.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:22.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:22.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:22.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:22.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:22.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:22.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:22.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 26163 26147 181.556 120.125 0.00223024 0.00544609 2024-08-17T17:27:22.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:22.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:22.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:22.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:22.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:22.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:22.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:22.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:22.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:27:23.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:23.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:23.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:23.179 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:23.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:23.281 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:23.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:23.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:23.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:23.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:23.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:23.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:23 smithi152 ceph-mon[93456]: pgmap v931: 89 pgs: 89 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:27:23.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:23 smithi142 ceph-mon[106729]: pgmap v931: 89 pgs: 89 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:27:23.614 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:23 smithi142 ceph-mon[103614]: pgmap v931: 89 pgs: 89 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:27:23.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 28190 28174 176.068 126.688 0.00223436 0.00567061 2024-08-17T17:27:23.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:23.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:23.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:23.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:23.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:23.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:24.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:24.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:24.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:24.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:24.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:24.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:24.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:24.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:24.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:24.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:24.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:24.619 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 29770 29754 169.038 98.75 0.0058757 0.00590626 2024-08-17T17:27:24.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:24.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:24.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:24.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:24.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:24.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:25.029 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:25.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:25.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:25.163 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:25.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:25.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:25 smithi142 ceph-mon[106729]: pgmap v932: 89 pgs: 89 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:27:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:25 smithi142 ceph-mon[103614]: pgmap v932: 89 pgs: 89 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:27:25.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:25.427 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:25.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:25.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:25 smithi152 ceph-mon[93456]: pgmap v932: 89 pgs: 89 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:27:25.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 31717 31701 165.092 121.688 0.00801691 0.00605175 2024-08-17T17:27:25.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:25.680 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:27:25.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:27:25.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:25.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:25.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:25.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:25.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:26.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:26.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:26.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:26.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:26.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:26.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:26.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:26.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:26.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:26.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:26.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:26.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 33591 33575 161.401 117.125 0.00344741 0.00618842 2024-08-17T17:27:26.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:26.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:26.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:26.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:26.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:26.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:26.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:27.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:27.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:27.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:27.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:27.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:27.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:27.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:27.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:27.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:27.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:27 smithi152 ceph-mon[93456]: pgmap v933: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-08-17T17:27:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:27.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 34632 34616 154.52 65.0625 0.0138708 0.00646732 2024-08-17T17:27:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:27 smithi142 ceph-mon[103614]: pgmap v933: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-08-17T17:27:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:27 smithi142 ceph-mon[106729]: pgmap v933: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-08-17T17:27:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:27.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:27.662 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:27.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:27.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:27.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:27.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:28.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:28.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:28.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:28.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:28.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:28.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:28.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:28.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:28.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:28.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:28.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 36720 36704 152.918 130.5 0.00202847 0.00651902 2024-08-17T17:27:28.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:28.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:28.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:28.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:28.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:29.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:29.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:29.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:29.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:29.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:29.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:29 smithi142 ceph-mon[106729]: pgmap v934: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:27:29.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:29 smithi142 ceph-mon[103614]: pgmap v934: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:27:29.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:29.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:29.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:29.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:29.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:29.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:29 smithi152 ceph-mon[93456]: pgmap v934: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:27:29.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 38601 38585 150.707 117.562 0.0178241 0.0066301 2024-08-17T17:27:29.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:29.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:29.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:29.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:29.912 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:29.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:29.925 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:29.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:29.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:30.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:30.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:30.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:30.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:30.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:30.300 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:27:30.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:30.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/189837285' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:30.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:30.318 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/189837285' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:30.319 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:30.355 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:30.456 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:30.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/189837285' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:27:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:27:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:30.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:30.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 40189 40173 147.68 99.25 0.00729913 0.00676561 2024-08-17T17:27:30.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:30.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:30.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:30.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:30.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:30.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:30.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.039 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:31.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:31.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:27:31.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:31.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:31.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:31.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:31.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:31.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:31.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[106729]: pgmap v935: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[106729]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/137820039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[103614]: pgmap v935: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:31.523 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:31.524 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[103614]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:27:31.524 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/137820039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:31.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:31 smithi152 ceph-mon[93456]: pgmap v935: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-08-17T17:27:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:31 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:31 smithi152 ceph-mon[93456]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:27:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/137820039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:31.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:31.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:31.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 42055 42039 145.954 116.625 0.00528609 0.00684623 2024-08-17T17:27:31.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:31.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:31.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:31.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:31.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:27:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:27:31] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:27:31.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:31.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:31.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:32.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:32.117 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:32.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:32.252 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:32.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:32.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:32.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:32.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:32.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:32.621 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 43741 43725 143.818 105.375 0.0045297 0.00694824 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:32.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:32.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:32.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:32.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:32.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:33.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:33.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:33.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:33.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:33.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:33.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:33.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:33.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:33.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:33.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:33 smithi152 ceph-mon[93456]: pgmap v937: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-17T17:27:33.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:33.619 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:33 smithi142 ceph-mon[103614]: pgmap v937: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-17T17:27:33.620 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:33 smithi142 ceph-mon[106729]: pgmap v937: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-17T17:27:33.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:27:33.620709+0000 min lat: 0.00101647 max lat: 0.270521 avg lat: 0.00702357 2024-08-17T17:27:33.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:33.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 45548 45532 142.273 112.938 0.00824785 0.00702357 2024-08-17T17:27:33.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:33.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:33.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:33.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:33.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:34.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:34.030 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:34.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:34.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:34.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:34.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:34.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:34.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:34.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:34.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:34.569 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:34.620 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 47308 47292 140.736 110 0.00455548 0.00710043 2024-08-17T17:27:34.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:34.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:34.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:34.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:34.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:34.950 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:34.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:34.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:35.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:35.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:35.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:35.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:35.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:35.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:35.458 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:35 smithi142 ceph-mon[106729]: pgmap v938: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:27:35.458 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:35 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:27:35.459 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:35 smithi142 ceph-mon[103614]: pgmap v938: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:27:35.459 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:35 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:27:35.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:35.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:35 smithi152 ceph-mon[93456]: pgmap v938: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-17T17:27:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:35 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:27:35.621 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 48835 48819 138.676 95.4375 0.00313912 0.00717787 2024-08-17T17:27:35.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:35.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:35.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:35.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:35.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:35.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:35.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:35.994 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:27:36.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:36.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:36.131 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:36.131 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:36.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:36.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:36.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:36.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:36.332 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/709378353' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:27:36.332 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:36 smithi142 ceph-mon[106729]: pgmap v939: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:27:36.333 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:27:36.335 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/709378353' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:27:36.335 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:36 smithi142 ceph-mon[103614]: pgmap v939: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:27:36.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:36.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:36.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/709378353' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:27:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:36 smithi152 ceph-mon[93456]: pgmap v939: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:27:36.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:36.621 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 50140 50124 136.193 81.5625 0.00697004 0.00733698 2024-08-17T17:27:36.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:36.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:36.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:36.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:36.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:36.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:37.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:37.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:37.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:37.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:37.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:37.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:37.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:37.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:27:37.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:37.561 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:37.622 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 51791 51775 134.81 103.188 0.00689627 0.00741205 2024-08-17T17:27:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/709378353' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:27:37.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:37 smithi142 ceph-mon[106729]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:27:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/709378353' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:27:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:37 smithi142 ceph-mon[103614]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:27:37.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:37.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.694 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:37.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:37.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/709378353' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:27:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:37 smithi152 ceph-mon[93456]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:27:37.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:37.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:37.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:37.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:38.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:38.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:38.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:38.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:38.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:38.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:38.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:38.479 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:38.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:38 smithi152 ceph-mon[93456]: pgmap v941: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-17T17:27:38.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:38 smithi142 ceph-mon[106729]: pgmap v941: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-17T17:27:38.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:38 smithi142 ceph-mon[103614]: pgmap v941: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-17T17:27:38.624 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 53223 53207 132.989 89.5 0.0090508 0.0075138 2024-08-17T17:27:38.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:38.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:38.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:38.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:38.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:38.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:39.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:39.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:39.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:39.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:39.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:39.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:39.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:39.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:39.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:39.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:39.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:39.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:27:39.550 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:39.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:39.592 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:39.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:39.625 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 54652 54636 131.309 89.3125 0.0236788 0.00760836 2024-08-17T17:27:39.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:39.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:39.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:39.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:39.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:40.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:40.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:40.017 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:40.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:40.131 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:40.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:40.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:40.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:40.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:40.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:40.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:40.539 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:40.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:27:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:27:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:40.624 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 56114 56098 129.83 91.375 0.00804356 0.00769749 2024-08-17T17:27:40.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:40.684 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:40.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:40.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:40.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:40.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:41.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:41.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:41.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:41.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:41.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:41.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:41.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:41.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:41.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:41.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:41 smithi152 ceph-mon[93456]: pgmap v942: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:27:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:41.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-mon[106729]: pgmap v942: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:27:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-mon[103614]: pgmap v942: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T17:27:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:41.624 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 57500 57484 128.287 86.625 0.005808 0.00778208 2024-08-17T17:27:41.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:41.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:41.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:41.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:41.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:41.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:41.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:42.010 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:42.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:27:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:27:41] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-08-17T17:27:42.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:42.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:42.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:42.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:42.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:42.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:42.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:42.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:42.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:42 smithi152 ceph-mon[93456]: pgmap v943: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-17T17:27:42.624 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 58929 58913 126.943 89.3125 0.00760981 0.0078716 2024-08-17T17:27:42.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:42.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:42.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:42.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:42.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:42 smithi142 ceph-mon[106729]: pgmap v943: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-17T17:27:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:42 smithi142 ceph-mon[103614]: pgmap v943: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-17T17:27:42.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:42.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:43.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:43.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:43.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:43.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:43.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:43.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:43.624 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 60298 60282 125.563 85.5625 0.00924123 0.00795904 2024-08-17T17:27:43.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:43.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:43.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:43.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:43.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:43.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:43.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:44.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:44.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:44.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:44.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:44.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:44.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:44.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:44.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.552 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:44.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:44.624 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 61663 61647 124.265 85.3125 0.00616326 0.00803862 2024-08-17T17:27:44.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:44.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:44.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.810 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:44.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:44.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:44.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:44.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:45.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:45.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:45.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:45.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:45.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:45.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:45.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:45.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:45.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:45.505 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:45 smithi152 ceph-mon[93456]: pgmap v944: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-08-17T17:27:45.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:45 smithi142 ceph-mon[106729]: pgmap v944: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-08-17T17:27:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:45 smithi142 ceph-mon[103614]: pgmap v944: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-08-17T17:27:45.626 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 62879 62863 122.751 76 0.00797611 0.00814128 2024-08-17T17:27:45.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:45.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:45.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:45.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:45.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:45.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:46.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:46.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:46.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:46.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:46.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:46.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:46.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:46.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:46.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:46.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:46.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:46.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:46.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:46.626 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 15 64405 64390 121.923 95.4375 0.00632623 0.0081969 2024-08-17T17:27:46.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:46.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:46.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:46.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:46.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:46.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:46.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:47.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:47.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:47.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:47.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:47.285 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:47.387 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:47.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:47.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:47.540 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:47 smithi142 ceph-mon[106729]: pgmap v945: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:27:47.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:47 smithi142 ceph-mon[103614]: pgmap v945: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:27:47.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:47.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:47 smithi152 ceph-mon[93456]: pgmap v945: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:27:47.627 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 65649 65633 120.62 77.6875 0.0134953 0.00828513 2024-08-17T17:27:47.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:47.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:47.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:47.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:47.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:47.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:48.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:48.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:48.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:48.235 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:48.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:48.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:48.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:48.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:48.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:48.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:48.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:48.627 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 67121 67105 119.802 92 0.0195554 0.00834202 2024-08-17T17:27:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.650 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:48.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:48.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:48.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:48.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:48.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:48.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:49.035 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:49.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:49.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:49.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:49.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:49.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:49.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:49.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:49 smithi142 ceph-mon[103614]: pgmap v946: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T17:27:49.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:49 smithi142 ceph-mon[106729]: pgmap v946: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T17:27:49.472 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:27:49.472 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.503 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:49.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:49.514 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:27:49.515 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:49.548 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:27:49.549 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:49 smithi152 ceph-mon[93456]: pgmap v946: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T17:27:49.581 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:27:49.581 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:49.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:49.615 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:27:49.615 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:49.626 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 68438 68422 118.761 82.3125 0.00557492 0.00841492 2024-08-17T17:27:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.636 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:49.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:49.652 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:27:49.653 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:49.694 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:27:49.695 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:49.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:49.740 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:27:49.741 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:49.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:49.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:49.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:49.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:49.994 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:50.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:50.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:50.299 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:27:50.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:50.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:50.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:50.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4235944918' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1058086072' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:27:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:27:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:50.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4235944918' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1058086072' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4235944918' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1058086072' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.627 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 69613 69597 117.536 73.4375 0.0156648 0.00850359 2024-08-17T17:27:50.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:50.746 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:50.848 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:50.883 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:50.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:51.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:51.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:51.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:51.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:51.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:51.286 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:51.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:51.433 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:51.452 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:27:51.453 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:27:51.453 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:27:51.453 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:27:51.453 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:27:51.535 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:51.541 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[103614]: pgmap v947: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[103614]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[106729]: pgmap v947: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:51.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-mon[106729]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:27:51.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:51 smithi152 ceph-mon[93456]: pgmap v947: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T17:27:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:51 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:51 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:51 smithi152 ceph-mon[93456]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:27:51.627 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 70975 70959 116.683 85.125 0.00917356 0.00856521 2024-08-17T17:27:51.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:51.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:51.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:51.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:51.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:27:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:27:51] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-08-17T17:27:51.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:52.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:52.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:52.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:52.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:52.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:52.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:52.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:52.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:52.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:52.627 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 72322 72306 115.849 84.1875 0.0113446 0.0086269 2024-08-17T17:27:52.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:52.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:52.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:52.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:52.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:52.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:52.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:53.067 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:53.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:53.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:53.229 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.236 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:53.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:53.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:27:53.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:53.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:53.466 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:53.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:53.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:53.534 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:53 smithi142 ceph-mon[106729]: pgmap v949: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:27:53.535 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:53 smithi142 ceph-mon[103614]: pgmap v949: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:27:53.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:53 smithi152 ceph-mon[93456]: pgmap v949: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-17T17:27:53.628 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:27:53.627779+0000 min lat: 0.00101647 max lat: 0.270521 avg lat: 0.00869496 2024-08-17T17:27:53.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:53.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 73587 73571 114.929 79.0625 0.0169045 0.00869496 2024-08-17T17:27:53.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:53.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:53.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:53.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:53.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:53.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:53.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:54.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:27:54.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:54.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:54.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:54.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:54.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:54.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:54.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:54.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:54.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 74855 74839 114.058 79.25 0.0117328 0.00876242 2024-08-17T17:27:54.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:54.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:54.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:54.790 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:54.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:54.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:54.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:54.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:54.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:55.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:55.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:55.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:55.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:55.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:55.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:55.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:55.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:55.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:55.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:55.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:55 smithi152 ceph-mon[93456]: pgmap v950: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T17:27:55.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:55.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:55 smithi142 ceph-mon[103614]: pgmap v950: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T17:27:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:55 smithi142 ceph-mon[106729]: pgmap v950: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T17:27:55.627 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 75894 75878 112.889 64.9375 0.0117955 0.00885347 2024-08-17T17:27:55.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:55.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:55.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:55.885 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:55.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:56.010 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:56.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:56.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:56.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:56.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:56.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:56.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:56.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:56.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:56.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:56.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:56.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:56.628 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 77179 77163 112.131 80.3125 0.0199875 0.00891338 2024-08-17T17:27:56.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:56.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:56.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:56.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:56.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:56.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:57.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:57.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:57.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:57.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:57.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:57.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:57.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:57.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:57 smithi142 ceph-mon[106729]: pgmap v951: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:27:57.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:57.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:57.498 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:57 smithi142 ceph-mon[103614]: pgmap v951: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:27:57.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:57.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:57.545 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:57.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:57 smithi152 ceph-mon[93456]: pgmap v951: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:27:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:27:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:57.628 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 78383 78367 111.291 75.25 0.0136498 0.00897995 2024-08-17T17:27:57.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:57.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:57.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:57.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:57.815 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:57.917 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:57.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:58.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:58.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:58.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:58.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:58.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:58.324 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:27:58.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:58.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:58.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:58.476 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:58.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:58.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/392349125' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:27:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:58 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:27:58.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:58.609 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:58.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/392349125' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:27:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:58 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:27:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/392349125' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:27:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:58 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:27:58.628 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 79620 79604 110.536 77.3125 0.010171 0.00904188 2024-08-17T17:27:58.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:58.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:58.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:58.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:58.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:58.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:58.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.035 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:59.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:59.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:59.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:59.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:59.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:59.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:59.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:59.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:59.541 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:59 smithi142 ceph-mon[103614]: pgmap v952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:27:59.541 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:59 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:27:59.541 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:27:59 smithi142 ceph-mon[103614]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:27:59.541 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:59 smithi142 ceph-mon[106729]: pgmap v952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:27:59.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:59 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:27:59.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:27:59 smithi142 ceph-mon[106729]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:27:59.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:59 smithi152 ceph-mon[93456]: pgmap v952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:27:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:59 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:27:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:27:59 smithi152 ceph-mon[93456]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:27:59.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:59.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 80893 80877 109.863 79.5625 0.0146757 0.00909762 2024-08-17T17:27:59.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:59.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:27:59.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:59.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:59.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:59.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:59.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:27:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:00.025 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:00.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:28:00.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:00.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:00.220 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:00.287 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:00.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:00.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:00.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:00.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:00 smithi142 ceph-mon[106729]: pgmap v954: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:28:00.539 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:00 smithi142 ceph-mon[103614]: pgmap v954: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:28:00.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:00.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:00 smithi152 ceph-mon[93456]: pgmap v954: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:28:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:28:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:28:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:00.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:00.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 81985 81969 108.977 68.25 0.0107645 0.00917133 2024-08-17T17:28:00.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:00.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:00.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:00.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:00.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:00.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:00.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:00.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:201367 9.b 9:dcdb63f3:::benchmark_data_smithi142_157452_object69246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:28:00.996 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:20:23.599574+0000", 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 457.23851308399998, 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26723699200000001, 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 201367 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:00.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599574+0000", 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599574+0000", 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:00.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599576+0000", 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9740000000000001e-06 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599582+0000", 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9020000000000004e-06 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599596+0000", 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4318000000000001e-05 2024-08-17T17:28:00.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599604+0000", 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7249999999999997e-06 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599671+0000", 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7658999999999997e-05 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599802+0000", 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013095199999999999 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.599930+0000", 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000127515 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.604450+0000", 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045197350000000004 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.619982+0000", 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015531942 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866677+0000", 2024-08-17T17:28:01.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.24669524200000001 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866761+0000", 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4036000000000007e-05 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866811+0000", 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9991999999999998e-05 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.003 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:201375 9.b 9:d12412ad:::benchmark_data_smithi142_157452_object69254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:20:23.616229+0000", 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 457.22185748499999, 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25071760900000001, 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:28:01.004 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 201375 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616229+0000", 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616229+0000", 2024-08-17T17:28:01.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616231+0000", 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2309999999999999e-06 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616235+0000", 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5090000000000004e-06 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616246+0000", 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0934999999999999e-05 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616252+0000", 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3959999999999997e-06 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616290+0000", 2024-08-17T17:28:01.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8390999999999999e-05 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616388+0000", 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7591999999999997e-05 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.616469+0000", 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1032000000000006e-05 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.619952+0000", 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034836279999999999 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866635+0000", 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.24668289500000001 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866848+0000", 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00021329000000000001 2024-08-17T17:28:01.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866916+0000", 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7562000000000001e-05 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.866947+0000", 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1148000000000001e-05 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.45800.0:201378 9.8 e184/160 9:136f3371:::benchmark_data_smithi142_157452_object69257:head v 184'8235, mlcod=184'8233)", 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:20:23.620815+0000", 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 457.21727185700001, 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25139726899999998, 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620815+0000", 2024-08-17T17:28:01.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620815+0000", 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620816+0000", 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7300000000000005e-07 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620822+0000", 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4999999999999996e-06 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620833+0000", 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0556e-05 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620840+0000", 2024-08-17T17:28:01.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9299999999999997e-06 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620907+0000", 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7299999999999996e-05 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.620926+0000", 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8841000000000001e-05 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.872175+0000", 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25124939000000002 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.872212+0000", 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6979e-05 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.015 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:201381 9.9 9:92908963:::benchmark_data_smithi142_157452_object69260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:20:23.622703+0000", 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 457.21538390299997, 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25546075000000001, 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 201381 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622703+0000", 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622703+0000", 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622705+0000", 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.55e-06 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622710+0000", 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4670000000000003e-06 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622723+0000", 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3159999999999999e-05 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622731+0000", 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1710000000000002e-06 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622800+0000", 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8249000000000003e-05 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.622914+0000", 2024-08-17T17:28:01.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114267 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.623008+0000", 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4387000000000003e-05 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.867514+0000", 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.244505789 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.867548+0000", 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3899000000000002e-05 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.877967+0000", 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010418857 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.878091+0000", 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012368200000000001 2024-08-17T17:28:01.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.878164+0000", 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3271999999999999e-05 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.45800.0:201382 9.b 9:d10367fe:::benchmark_data_smithi142_157452_object69261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:20:23.623260+0000", 2024-08-17T17:28:01.022 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 457.21482734400001, 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27727157499999999, 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.45800", 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/578244091", 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 201382 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.623260+0000", 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.623260+0000", 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.623260+0000", 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5000000000000001e-07 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.623265+0000", 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4560000000000002e-06 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.863847+0000", 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.240582295 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.863860+0000", 2024-08-17T17:28:01.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3142000000000001e-05 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.867027+0000", 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031663870000000001 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.867156+0000", 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012902700000000001 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.867274+0000", 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011857 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.879638+0000", 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01236369 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.883399+0000", 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037612679999999999 2024-08-17T17:28:01.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.900435+0000", 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017036000999999999 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.900488+0000", 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2472000000000002e-05 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:20:23.900531+0000", 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3417000000000001e-05 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:46022 10.8 10:14a24480:::benchmark_data_smithi142_192453_object46021:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:22:49.553295+0000", 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 311.284791842, 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.20448577100000001, 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.029 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 46022 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553295+0000", 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553295+0000", 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553296+0000", 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5899999999999996e-07 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553321+0000", 2024-08-17T17:28:01.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5709999999999999e-05 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553329+0000", 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5070000000000003e-06 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553334+0000", 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1039999999999998e-06 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565791+0000", 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012456662 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565955+0000", 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000164403 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.566109+0000", 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015354299999999999 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.571533+0000", 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054244209999999996 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.576375+0000", 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048423520000000003 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.757671+0000", 2024-08-17T17:28:01.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18129589700000001 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.757735+0000", 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3748e-05 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.757781+0000", 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5865000000000002e-05 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.035 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:46025 10.8 10:170b7d78:::benchmark_data_smithi142_192453_object46024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:22:49.553405+0000", 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 311.28468243600003, 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.20592275500000001, 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 46025 2024-08-17T17:28:01.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553405+0000", 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553405+0000", 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553405+0000", 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6899999999999998e-07 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553498+0000", 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2793000000000004e-05 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553507+0000", 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0389999999999993e-06 2024-08-17T17:28:01.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.553512+0000", 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5269999999999998e-06 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.567954+0000", 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014441918999999999 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.568117+0000", 2024-08-17T17:28:01.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000162471 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.568375+0000", 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000257908 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.576381+0000", 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080062710000000006 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.576419+0000", 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8427999999999999e-05 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.759226+0000", 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18280681800000001 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.759280+0000", 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3801000000000002e-05 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.759327+0000", 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7311000000000002e-05 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:46029 10.8 10:1868b121:::benchmark_data_smithi142_192453_object46028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:22:49.565619+0000", 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 311.27246757400002, 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19386162400000001, 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.042 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 46029 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565619+0000", 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565619+0000", 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565620+0000", 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6699999999999999e-07 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565645+0000", 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4674000000000001e-05 2024-08-17T17:28:01.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565651+0000", 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6889999999999998e-06 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.565656+0000", 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2800000000000003e-06 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.571562+0000", 2024-08-17T17:28:01.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059057040000000003 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.571720+0000", 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015760500000000001 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.571875+0000", 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015544800000000001 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.576385+0000", 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045102679999999996 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.577657+0000", 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012718569999999999 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.759401+0000", 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18174369200000001 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.759447+0000", 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5985000000000001e-05 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:22:49.759481+0000", 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3955000000000003e-05 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.048 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:89536 10.4 10:20160454:::benchmark_data_smithi142_192453_object89535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:20.913275+0000", 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 279.92481202099998, 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18275671099999999, 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.049 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 89536 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.913275+0000", 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.913275+0000", 2024-08-17T17:28:01.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.913276+0000", 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3540000000000001e-06 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.913298+0000", 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2149000000000001e-05 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.913308+0000", 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2599999999999994e-06 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.913313+0000", 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7760000000000003e-06 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.914279+0000", 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00096567400000000002 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.914445+0000", 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000166157 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.914573+0000", 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000127389 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.917640+0000", 2024-08-17T17:28:01.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030670749999999998 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:20.920614+0000", 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029740499999999998 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.095861+0000", 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.175247503 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.095993+0000", 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013209499999999999 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:21.096032+0000", 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8229000000000003e-05 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179059 10.16 10:6ca643db:::benchmark_data_smithi142_192453_object53311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.874961+0000", 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.963125916, 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1128011550000001, 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179059 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.056 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874961+0000", 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874961+0000", 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874962+0000", 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.466e-06 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874968+0000", 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7629999999999997e-06 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874981+0000", 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2499e-05 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874988+0000", 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.875063+0000", 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5281e-05 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.875075+0000", 2024-08-17T17:28:01.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1722999999999999e-05 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.977731+0000", 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.102656842 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.977994+0000", 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026225899999999998 2024-08-17T17:28:01.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.978306+0000", 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00031236099999999999 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980318+0000", 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002012167 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980412+0000", 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3225000000000003e-05 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987210+0000", 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067988019999999996 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987324+0000", 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114079 2024-08-17T17:28:01.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987762+0000", 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000437792 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179060 10.6 10:6267ad6a:::benchmark_data_smithi142_192453_object53312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.877493+0000", 2024-08-17T17:28:01.063 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.96059421699999, 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0776005580000001, 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179060 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877493+0000", 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877493+0000", 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877495+0000", 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7379999999999999e-06 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877507+0000", 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1565e-05 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877531+0000", 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4179999999999999e-05 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877543+0000", 2024-08-17T17:28:01.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1841000000000001e-05 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877631+0000", 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7466999999999995e-05 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877643+0000", 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2224e-05 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.776800+0000", 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.89915759500000003 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.777008+0000", 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020719900000000001 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.777249+0000", 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024190299999999999 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876915+0000", 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.099665980000000001 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876952+0000", 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6426e-05 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.954737+0000", 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.077785629999999994 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.954847+0000", 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010928400000000001 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955093+0000", 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000246526 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.070 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179070 10.6 10:673852e0:::benchmark_data_smithi142_192453_object53322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.939158+0000", 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.898929369, 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.016300097, 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.071 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179070 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939158+0000", 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939158+0000", 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939160+0000", 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.04e-06 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939165+0000", 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6280000000000002e-06 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939180+0000", 2024-08-17T17:28:01.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4820999999999999e-05 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939190+0000", 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9709999999999999e-06 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939262+0000", 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1489999999999995e-05 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939275+0000", 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.385e-05 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779505+0000", 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.84022981600000002 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779727+0000", 2024-08-17T17:28:01.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022187800000000001 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779969+0000", 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024200799999999999 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876965+0000", 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.096995998 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.877076+0000", 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111371 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955159+0000", 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078082562999999994 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955254+0000", 2024-08-17T17:28:01.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4983999999999997e-05 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955458+0000", 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020367900000000001 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179074 10.6 10:60355f2a:::benchmark_data_smithi142_192453_object53326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.078 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.959773+0000", 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.87831411799999, 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99600560299999996, 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179074 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959773+0000", 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959773+0000", 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959775+0000", 2024-08-17T17:28:01.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7630000000000001e-06 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959780+0000", 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8270000000000003e-06 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959796+0000", 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.519e-05 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959804+0000", 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4489999999999999e-06 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959885+0000", 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0570999999999998e-05 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959899+0000", 2024-08-17T17:28:01.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4100999999999999e-05 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.781785+0000", 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.82188591899999996 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.782008+0000", 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022341700000000001 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.782252+0000", 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024394299999999999 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876976+0000", 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.094723940000000006 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.877172+0000", 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019610600000000001 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955515+0000", 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078342785999999998 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955590+0000", 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5014999999999999e-05 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955778+0000", 2024-08-17T17:28:01.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018857600000000001 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179093 10.16 10:69bfd028:::benchmark_data_smithi142_192453_object53345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:23.033659+0000", 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.804427557, 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95483545400000003, 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.086 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179093 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033659+0000", 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033659+0000", 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033661+0000", 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.547e-06 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033667+0000", 2024-08-17T17:28:01.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1029999999999998e-06 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034263+0000", 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059557700000000002 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034271+0000", 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2549999999999997e-06 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034347+0000", 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5828000000000007e-05 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034354+0000", 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9639999999999997e-06 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980469+0000", 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.94611584100000001 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980680+0000", 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00021035800000000001 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980941+0000", 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026073300000000003 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987843+0000", 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0069019479999999998 2024-08-17T17:28:01.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987877+0000", 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4063999999999998e-05 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987950+0000", 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3293000000000005e-05 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.988217+0000", 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026665499999999997 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.988495+0000", 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00027828800000000002 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179098 10.16 10:680f5353:::benchmark_data_smithi142_192453_object53350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:23.042973+0000", 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.79511391099999, 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95446910100000004, 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.094 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179098 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042973+0000", 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042973+0000", 2024-08-17T17:28:01.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042975+0000", 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.823e-06 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042981+0000", 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5740000000000001e-06 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042997+0000", 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5534999999999999e-05 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.043008+0000", 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0831e-05 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.043072+0000", 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3793000000000004e-05 2024-08-17T17:28:01.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.043081+0000", 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9260000000000006e-06 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.982770+0000", 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93968903199999998 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.983144+0000", 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00037481999999999999 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.985041+0000", 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0018962899999999999 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.989001+0000", 2024-08-17T17:28:01.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039599580000000004 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.991100+0000", 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002099007 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.997105+0000", 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060057089999999997 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.997189+0000", 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3978000000000001e-05 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.997442+0000", 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025282499999999998 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179107 10.6 10:6057202c:::benchmark_data_smithi142_192453_object53359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:23.076805+0000", 2024-08-17T17:28:01.101 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 157.76128146100001, 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.87953849799999995, 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179107 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076805+0000", 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076805+0000", 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076807+0000", 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.139e-06 2024-08-17T17:28:01.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076811+0000", 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0679999999999998e-06 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076822+0000", 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0886999999999999e-05 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076826+0000", 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7720000000000002e-06 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076884+0000", 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7207000000000003e-05 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076893+0000", 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5610000000000004e-06 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.874948+0000", 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.79805484299999996 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.875101+0000", 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015309000000000001 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:28:01.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.875259+0000", 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015764899999999999 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.877266+0000", 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020069229999999999 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955941+0000", 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078675373000000007 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.956007+0000", 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5870999999999993e-05 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.956089+0000", 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1760000000000003e-05 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.956344+0000", 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025535500000000002 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:29101 11.9 e287/283 11:91525119:::benchmark_data_smithi142_227468_object29100:head v 287'1828, mlcod=287'1825)", 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 36.747489340000001, 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26308380100000001, 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.109 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3300000000000001e-07 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090642+0000", 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3078999999999999e-05 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090653+0000", 2024-08-17T17:28:01.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1972e-05 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090661+0000", 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3549999999999999e-06 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090738+0000", 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7309000000000001e-05 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090757+0000", 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8936e-05 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092123+0000", 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001365478 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353681+0000", 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26155883899999999 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.113 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:29102 11.1 e287/283 11:8390c806:::benchmark_data_smithi142_227468_object29101:head v 287'1859, mlcod=287'1857)", 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.092007+0000", 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 36.746080196999998, 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26468164500000002, 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.114 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092007+0000", 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092007+0000", 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092008+0000", 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4799999999999997e-07 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092206+0000", 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000198228 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353636+0000", 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26143067199999998 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353654+0000", 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7646999999999999e-05 2024-08-17T17:28:01.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353728+0000", 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3374000000000006e-05 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353753+0000", 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5216e-05 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.356643+0000", 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028901360000000002 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.356688+0000", 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5524e-05 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.118 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:29103 11.9 e287/283 11:950c9362:::benchmark_data_smithi142_227468_object29102:head v 287'1829, mlcod=287'1825)", 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.092977+0000", 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 36.745109644000003, 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26484094000000002, 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092977+0000", 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092977+0000", 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092978+0000", 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0359999999999999e-06 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.093003+0000", 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4321e-05 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353634+0000", 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.260631157 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353654+0000", 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9958000000000001e-05 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353739+0000", 2024-08-17T17:28:01.121 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5712999999999997e-05 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353754+0000", 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4806e-05 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357766+0000", 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040113789999999998 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.122 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357818+0000", 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.257e-05 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:81743 11.9 e291/283 11:91d21a7f:::benchmark_data_smithi142_227468_object81742:head v 291'5212, mlcod=291'5208)", 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:00.412710+0000", 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.42537736700000001, 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032698496000000001, 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:01.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.412710+0000", 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.412710+0000", 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.412711+0000", 2024-08-17T17:28:01.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1710000000000001e-06 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.412736+0000", 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5228e-05 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.412752+0000", 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6262000000000001e-05 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:01.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.412760+0000", 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8269999999999998e-06 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.421507+0000", 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0087465570000000003 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.421561+0000", 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4382000000000002e-05 2024-08-17T17:28:01.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.445269+0000", 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.023707514999999998 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:00.445408+0000", 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013955400000000001 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.127 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:01.128 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:01.128 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:28:01.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:01.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:01.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:01.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:01.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:01.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:01.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:01.564 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:01.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 82801 82785 107.769 51 0.00332913 0.00923741 2024-08-17T17:28:01.666 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:01.688 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:01.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:01.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:01.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:01.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:01.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:01.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:02.043 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:28:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:28:01] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-08-17T17:28:02.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:02.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:02.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:02.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:02.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:02.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:02.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:02.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:02.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:02.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:02.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:02.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:02.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 83749 83733 106.779 59.25 0.0127038 0.00936072 2024-08-17T17:28:02.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:02.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:02.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:02.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:02.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:03.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:03.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:03.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:03.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:03.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:03.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:03.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:03.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:03.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:03.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:03.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:03 smithi152 ceph-mon[93456]: pgmap v955: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:28:03.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:03 smithi142 ceph-mon[106729]: pgmap v955: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:28:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:03 smithi142 ceph-mon[103614]: pgmap v955: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:28:03.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 84973 84957 106.174 76.5 0.0122417 0.00941407 2024-08-17T17:28:03.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:03.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:03.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:03.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:03.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:03.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:04.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:04.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:04.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:04.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:04.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:04.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:04.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:04.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:04.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:04.429 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:04.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:04.560 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:04.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 86146 86130 105.529 73.3125 0.014071 0.0094711 2024-08-17T17:28:04.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:04.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:04.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:04.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:04.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:04.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:04.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:04.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:05.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:05.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:05.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:05.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:05.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:05.349 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:05.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:05 smithi142 ceph-mon[106729]: pgmap v956: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:28:05.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:05 smithi142 ceph-mon[103614]: pgmap v956: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:28:05.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:05.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:05.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:05 smithi152 ceph-mon[93456]: pgmap v956: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T17:28:05.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:05.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:05.629 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 87350 87334 104.947 75.25 0.0162664 0.00952369 2024-08-17T17:28:05.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:05.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:05.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:05.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:05.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:05.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:06.025 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:06.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:06.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:06.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:06.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:06.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:06.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:06.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:06.550 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:06.594 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:06.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 88488 88472 104.306 71.125 0.0224049 0.00958243 2024-08-17T17:28:06.696 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:06.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:06.834 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:06.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:06.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:07.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:07.015 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:07.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:28:07.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:07.156 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:07.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:07.291 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:07 smithi142 ceph-mon[103614]: pgmap v957: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:28:07.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:07.416 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:07.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:07.546 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:07 smithi142 ceph-mon[106729]: pgmap v957: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:28:07.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:07.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:07 smithi152 ceph-mon[93456]: pgmap v957: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:28:07.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:07.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:07.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 89514 89498 103.561 64.125 0.00886282 0.00965154 2024-08-17T17:28:07.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:07.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:07.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:07.848 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:28:07.849 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:07.884 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:28:07.885 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:07.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:07.934 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:28:07.935 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:07.977 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:28:07.978 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:07.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:08.024 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:28:08.024 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:08.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:08.069 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:28:08.069 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:08.098 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:28:08.098 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:08.134 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:28:08.135 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:08.167 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:08.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:08.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:08.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:08.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3576984982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3576984982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:08.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:08.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:08.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:08.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:08.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3576984982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:08.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:08.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 90523 90507 102.825 63.0625 0.0210547 0.00972056 2024-08-17T17:28:08.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:08.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:08.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:08.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:08.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:08.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:08.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:08.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:09.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:09.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:09.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:09.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:09.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:09.316 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:28:09.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:09.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:09.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:09.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:09.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:09 smithi152 ceph-mon[93456]: pgmap v958: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:28:09.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2849107801' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:09 smithi142 ceph-mon[106729]: pgmap v958: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:28:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2849107801' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:09 smithi142 ceph-mon[103614]: pgmap v958: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:28:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2849107801' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:09.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 91685 91669 102.286 72.625 0.00837172 0.00977176 2024-08-17T17:28:09.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:09.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:09.734 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:09.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:09.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:09.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:10.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:10.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:10.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:10.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:10.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:10.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:10.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:10.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:10.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:10 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2849107801' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:10 smithi152 ceph-mon[93456]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:28:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:28:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:28:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:10 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2849107801' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:10 smithi142 ceph-mon[106729]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:28:10.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:10 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:10.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2849107801' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:10.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:10 smithi142 ceph-mon[103614]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:28:10.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:10.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 92798 92782 101.711 69.5625 0.00978649 0.00982719 2024-08-17T17:28:10.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:10.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:10.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:10.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:10.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:11.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:11.079 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:11.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:11.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:28:11.180 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:11.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:11.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:11.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:11.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:11.470 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.523 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:11.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:11.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:11 smithi152 ceph-mon[93456]: pgmap v960: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:28:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:11 smithi142 ceph-mon[106729]: pgmap v960: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:28:11.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:11 smithi142 ceph-mon[103614]: pgmap v960: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:28:11.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:11.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 93821 93805 101.06 63.9375 0.00825654 0.00988979 2024-08-17T17:28:11.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:11.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:11.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:11.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:11.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:12.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:12.057 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:28:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:28:11] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:28:12.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:12.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:12.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:12.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:12.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:12.334 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:12.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:12.436 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:12.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:12.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:12.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:12.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:12.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:12.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:12.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:12.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 94853 94837 100.44 64.5 0.0154606 0.00995157 2024-08-17T17:28:12.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:12.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:12.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:12.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:12.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:12.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:12.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:12.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:12.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:13.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:13.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:13.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:13.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:13.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:13.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:13.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:13.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:13.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:13.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:13 smithi142 ceph-mon[103614]: pgmap v961: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:28:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:13 smithi142 ceph-mon[103614]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:28:13.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:13 smithi142 ceph-mon[106729]: pgmap v961: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:28:13.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:13 smithi142 ceph-mon[106729]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:28:13.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:28:13.632156+0000 min lat: 0.00101647 max lat: 0.452582 avg lat: 0.0100146 2024-08-17T17:28:13.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:28:13.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 95848 95832 99.8026 62.1875 0.014228 0.0100146 2024-08-17T17:28:13.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:13.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:13.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:13 smithi152 ceph-mon[93456]: pgmap v961: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T17:28:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:13 smithi152 ceph-mon[93456]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:28:13.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:28:13.844 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:13.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:13.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:14.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:14.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:14.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:14.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:14.346 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:28:14.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:14.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:14.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:14.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:14.534 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1288917962' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:14 smithi142 ceph-mon[106729]: pgmap v962: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:28:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1288917962' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:14 smithi142 ceph-mon[103614]: pgmap v962: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:28:14.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 96708 96692 99.0477 53.75 0.0280046 0.0100919 2024-08-17T17:28:14.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:14.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:14.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:14.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:14.792 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1288917962' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:14 smithi152 ceph-mon[93456]: pgmap v962: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T17:28:14.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:14.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:15.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:15.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:15.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:15.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:15.316 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:15.316 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:15.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:15.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:15.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1288917962' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:28:15.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:15 smithi142 ceph-mon[103614]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:28:15.418 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1288917962' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:28:15.418 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:15 smithi142 ceph-mon[106729]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:28:15.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:15.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:15.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:15.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 97627 97611 98.3765 57.4375 0.00843451 0.01016 2024-08-17T17:28:15.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:15.741 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:15.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:15.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1288917962' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:28:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:15 smithi152 ceph-mon[93456]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:28:15.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:15.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:16.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:16.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:16.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:16.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:16.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:16.249 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:28:16.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:16.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:16.426 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:16.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:16.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:16.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:16 smithi142 ceph-mon[103614]: pgmap v964: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:28:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:16 smithi142 ceph-mon[106729]: pgmap v964: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:28:16.632 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 98668 98652 97.8467 65.0625 0.0202878 0.0102147 2024-08-17T17:28:16.707 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:16.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:16.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:16 smithi152 ceph-mon[93456]: pgmap v964: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:28:16.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:16.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:17.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:17.109 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:17.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:17.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:17.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:17.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.409 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:17.510 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:17.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:17.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 99738 99722 97.3628 66.875 0.0164093 0.0102661 2024-08-17T17:28:17.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:17.655 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:17.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:28:17.696 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:17.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:17.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:17.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:17.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:17.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:18.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:18.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:18.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:18.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:18.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:18.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:18.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:18.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:18.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:18.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:18.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:18.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 100745 100729 96.8331 62.9375 0.0212972 0.0103216 2024-08-17T17:28:18.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:18.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:18.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.809 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:18.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:18.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:18.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:18.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:19.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:19.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:19.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:19.182 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:28:19.182 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:19.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.212 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:28:19.212 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:19.230 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:19.249 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:28:19.249 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:19.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:19.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:19.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:19 smithi142 ceph-mon[106729]: pgmap v965: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:28:19.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3794337644' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:19.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:19 smithi142 ceph-mon[103614]: pgmap v965: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:28:19.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3794337644' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:19.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:19.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:19.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:19 smithi152 ceph-mon[93456]: pgmap v965: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:28:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3794337644' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:19.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:19.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:19.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:19.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 101724 101708 96.293 61.1875 0.0216717 0.0103799 2024-08-17T17:28:19.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:19.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:19.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:19.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:20.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:20.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:20.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:20.193 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:20.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:20.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:20.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:20.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:20.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:28:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:28:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:20.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:20.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 102769 102753 95.8306 65.3125 0.0066126 0.0104303 2024-08-17T17:28:20.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:20.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:20.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:20.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:20.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:20.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:20.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:20.981 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:21.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.120 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:21.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:21.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:21.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:21.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:21 smithi142 ceph-mon[103614]: pgmap v966: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:28:21.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:21 smithi142 ceph-mon[106729]: pgmap v966: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:28:21.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:21.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:21.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:21.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:21 smithi152 ceph-mon[93456]: pgmap v966: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:28:21.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:21.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 103753 103737 95.3258 61.5 0.031254 0.0104857 2024-08-17T17:28:21.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:21.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:21.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.891 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:21.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:22.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:22.035 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:28:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:28:21] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:28:22.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:22.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:22.174 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:22.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:22.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:22.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:22.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:22.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:22.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:22.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 104434 104418 94.5611 42.5625 0.0131298 0.01057 2024-08-17T17:28:22.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.687 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:22.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:22.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:22.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:22.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:22.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:22.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:23.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:23.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:23.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:23.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:23.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:23.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:23.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:23 smithi142 ceph-mon[106729]: pgmap v967: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:28:23.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:23 smithi142 ceph-mon[103614]: pgmap v967: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:28:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.375 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:23.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:23.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:23.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:23 smithi152 ceph-mon[93456]: pgmap v967: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:28:23.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:23.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 105486 105470 94.1495 65.75 0.0159945 0.0106163 2024-08-17T17:28:23.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:23.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:23.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:23.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:23.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:24.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:24.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:24.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:28:24.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:24.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:24.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:24.293 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:24.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:24.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:24.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:24.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:24.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 106383 106367 93.6131 56.0625 0.00355052 0.0106732 2024-08-17T17:28:24.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:24.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:24.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:24.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:24.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:24.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:24.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:25.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:25.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:25.197 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:25.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:25.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:25.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:25 smithi142 ceph-mon[106729]: pgmap v968: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-08-17T17:28:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:25 smithi142 ceph-mon[103614]: pgmap v968: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-08-17T17:28:25.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:25.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:25.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:25 smithi152 ceph-mon[93456]: pgmap v968: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-08-17T17:28:25.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:25.635 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 107380 107364 93.1783 62.3125 0.021109 0.0107274 2024-08-17T17:28:25.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:25.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:25.755 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:25.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:25.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:25.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:25.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:25.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:26.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:26.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:26.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:26.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:26.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:26.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:26.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:26.583 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 108302 108286 92.6913 57.625 0.0113244 0.0107837 2024-08-17T17:28:26.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:26.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:26.797 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:26.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:26.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:27.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:27.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:27.115 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:27.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:27.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:27.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:27.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[103614]: pgmap v969: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/447990111' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[106729]: pgmap v969: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:27.431 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:27.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/447990111' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:27.432 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:27 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:27.432 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:28:27.432 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:28:27.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:27.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:27.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:27 smithi152 ceph-mon[93456]: pgmap v969: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:28:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/447990111' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:27 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:27.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.605 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:27.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:27.633 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 109135 109119 92.1423 52.0625 0.0112778 0.0108477 2024-08-17T17:28:27.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:27.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:27.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:27.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:27.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:28.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:28.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:28.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:28.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:28.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:28.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:28.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:28.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3669616602' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[103614]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3669616602' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:28.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:28.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:28:28.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:28.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:28 smithi142 ceph-mon[106729]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:28:28.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:28.424 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:28.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:28.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:28.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:28.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3669616602' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:28 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:28:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:28.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:28 smithi152 ceph-mon[93456]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:28:28.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 15 110143 110128 91.7545 63.0625 0.0244984 0.0108933 2024-08-17T17:28:28.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:28.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:28.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:28.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:28.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:28.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:28.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:29.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:29.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:29.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:29.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:29.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:29.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:29 smithi142 ceph-mon[106729]: pgmap v971: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T17:28:29.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:29 smithi142 ceph-mon[103614]: pgmap v971: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T17:28:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:29.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:29.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:29 smithi152 ceph-mon[93456]: pgmap v971: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T17:28:29.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:29.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:29.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:29.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 111147 111131 91.3721 62.6875 0.0233967 0.0109389 2024-08-17T17:28:29.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:29.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:29.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:29.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:29.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:30.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:30.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:30.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:30.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:30.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:30.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:30.238 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:28:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:28:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:30.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:30.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:30.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:30.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:28:30.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:30.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:30.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:30.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 112100 112084 90.9589 59.5625 0.0178426 0.0109893 2024-08-17T17:28:30.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:30.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:30.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:30.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:30.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:30.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:31.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:31.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:31.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:28:31.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:31.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:31.193 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:31 smithi142 ceph-mon[103614]: pgmap v972: 89 pgs: 89 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-08-17T17:28:31.194 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:31 smithi142 ceph-mon[106729]: pgmap v972: 89 pgs: 89 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-08-17T17:28:31.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:31.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:31.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:31.483 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:31 smithi152 ceph-mon[93456]: pgmap v972: 89 pgs: 89 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-08-17T17:28:31.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:31.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:31.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 112961 112945 90.4827 53.8125 0.0217723 0.0110472 2024-08-17T17:28:31.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:31.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:31.785 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:31.856 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:28:31.857 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:31.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:31.893 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:28:31.894 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:31.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:31.931 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:28:31.931 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:31.965 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:28:31.965 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:31.993 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:28:31.993 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:32.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:32.024 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:28:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:28:31] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:28:32.025 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:28:32.025 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:32.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:32.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:32.058 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:28:32.058 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:32.089 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:28:32.089 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:32.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:32.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:32.235 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:32.236 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:32.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:32.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:32.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:32.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2034062578' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:32.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2034062578' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:32.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:32.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:32.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:32.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2034062578' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:32.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:32.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 113870 113854 90.0565 56.8125 0.0166193 0.011099 2024-08-17T17:28:32.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:32.731 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:32.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:32.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:32.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:32.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:32.990 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:33.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.120 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:33.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:33.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:33.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:33.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:33.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:33.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:33.420 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:33 smithi142 ceph-mon[103614]: pgmap v973: 89 pgs: 89 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-08-17T17:28:33.421 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:33 smithi142 ceph-mon[106729]: pgmap v973: 89 pgs: 89 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-08-17T17:28:33.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:28:33.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:33.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:33 smithi152 ceph-mon[93456]: pgmap v973: 89 pgs: 89 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-08-17T17:28:33.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:33.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:28:33.634712+0000 min lat: 0.00101647 max lat: 0.452582 avg lat: 0.0111422 2024-08-17T17:28:33.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:28:33.634 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 114859 114843 89.7031 61.8125 0.00750777 0.0111422 2024-08-17T17:28:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:33.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:33.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:33.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:33.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:33.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:33.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:34.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:34.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:34.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.241 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:34.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:34.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:34.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:34.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:34.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:34.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:34.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:34.635 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 115789 115773 89.3123 58.125 0.0167985 0.0111905 2024-08-17T17:28:34.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:34.678 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:34.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:34.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:34.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:34.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:34.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:35.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:35.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:35.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:35.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:35.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:35.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:35.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:35.479 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:35 smithi142 ceph-mon[106729]: pgmap v974: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:28:35.480 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:35 smithi142 ceph-mon[103614]: pgmap v974: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:28:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:35.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:35.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:35 smithi152 ceph-mon[93456]: pgmap v974: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:28:35.579 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:35.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:35.636 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 116691 116675 88.9099 56.375 0.0137351 0.0112416 2024-08-17T17:28:35.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:35.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:35.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:35.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:35.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:36.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:36.081 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:36.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:36.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:28:36.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:36.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:36.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.636 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 117529 117513 88.4697 52.375 0.0201137 0.0112981 2024-08-17T17:28:36.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:36.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:36.772 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:36.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:36.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:37.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:37.082 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:37.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:37.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:37.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:37.275 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:28:37.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:37.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:37.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:37.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:37.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:37 smithi152 ceph-mon[93456]: pgmap v975: 89 pgs: 89 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2024-08-17T17:28:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2167772193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:37 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:37.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:37 smithi142 ceph-mon[106729]: pgmap v975: 89 pgs: 89 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2024-08-17T17:28:37.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2167772193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:37.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:37 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:37.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:37 smithi142 ceph-mon[103614]: pgmap v975: 89 pgs: 89 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2024-08-17T17:28:37.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2167772193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:37.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:37 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:37.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 118454 118438 88.1046 57.8125 0.0163247 0.0113448 2024-08-17T17:28:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:37.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:37.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:37.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:37.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:37.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:37.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:38.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:38.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:38.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:38.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:38.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:38.316 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:38 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:28:38.317 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:38 smithi142 ceph-mon[103614]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:28:38.317 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:38 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:28:38.317 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:38 smithi142 ceph-mon[106729]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:28:38.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:38.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:38.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:38.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:38.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:38.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:38 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:28:38.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:38 smithi152 ceph-mon[93456]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:28:38.595 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:38.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:38.636 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 119404 119388 87.7666 59.375 0.0128064 0.0113894 2024-08-17T17:28:38.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:38.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:38.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:38.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:38.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:39.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:28:39.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:39.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:39.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:39.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:39.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:39.273 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:39 smithi142 ceph-mon[103614]: pgmap v977: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2024-08-17T17:28:39.274 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:39 smithi142 ceph-mon[106729]: pgmap v977: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2024-08-17T17:28:39.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:39.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:39 smithi152 ceph-mon[93456]: pgmap v977: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2024-08-17T17:28:39.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:39.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.636 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 120175 120159 87.3064 48.1875 0.00511024 0.0114477 2024-08-17T17:28:39.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.846 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.869 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:39.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:40.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:40.107 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:40.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:40.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:40.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:40.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:40.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:40.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:40.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:28:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:28:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:40.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 120883 120867 86.8115 44.25 0.0172553 0.0115144 2024-08-17T17:28:40.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:40.665 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:40.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:40.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:40.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:40.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:41.024 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:41.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:41.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:41.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:41.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:41.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:41.220 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:28:41.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:41.319 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:41 smithi142 ceph-mon[103614]: pgmap v978: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2024-08-17T17:28:41.319 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/251213672' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:41.319 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:41 smithi142 ceph-mon[106729]: pgmap v978: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2024-08-17T17:28:41.319 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/251213672' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:41.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:41.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:41.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:41.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:41 smithi152 ceph-mon[93456]: pgmap v978: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2024-08-17T17:28:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/251213672' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:41.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:41.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:41.636 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 121780 121764 86.4621 56.0625 0.006599 0.0115608 2024-08-17T17:28:41.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:41.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:41.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:41.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:42.023 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:42.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:42.084 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:28:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:28:41] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:28:42.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:42.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:42.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:42.325 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:42.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:28:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:28:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/251213672' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[106729]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:28:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:42.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:42.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:28:42.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:28:42.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/251213672' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:42.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[103614]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:28:42.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:42.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:42.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:42.471 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:42.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:42 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:28:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:42 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:28:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/251213672' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:42 smithi152 ceph-mon[93456]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:28:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:42.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:42.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:42.633 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:42.633 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:42.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 122637 122621 86.0924 53.5625 0.0238865 0.0116097 2024-08-17T17:28:42.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:42.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:42.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:42.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:42.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:43.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:43.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:43.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:43.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:43.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:43.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:43.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:43 smithi142 ceph-mon[106729]: pgmap v980: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-08-17T17:28:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:43 smithi142 ceph-mon[103614]: pgmap v980: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-08-17T17:28:43.387 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:43.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:43.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:43.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:43.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:43 smithi152 ceph-mon[93456]: pgmap v980: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-08-17T17:28:43.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0049 2024-08-17T17:28:43.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 123416 2024-08-17T17:28:43.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:28:43.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:28:43.637 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 85.7009 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 39.2373 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 249.312 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 42.5625 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1371 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 627.797 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 3989 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 681 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0116645 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0115679 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 0.452582 2024-08-17T17:28:43.638 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00101647 2024-08-17T17:28:43.642 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:28:43.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:43.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:43.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:43.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:43.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:43.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:43.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:44.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:44.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:44.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:44.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:44.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:44.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:44.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:44.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:44.483 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:44.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:44.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:44.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:44.750 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:44.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:44.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:44.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:44.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:45.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:45.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:45.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:45.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:45.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:45.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:45.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:45.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:45.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:45.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:45.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:45 smithi152 ceph-mon[93456]: pgmap v981: 89 pgs: 89 active+clean; 7.4 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 49 MiB/s wr, 786 op/s 2024-08-17T17:28:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:45 smithi142 ceph-mon[103614]: pgmap v981: 89 pgs: 89 active+clean; 7.4 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 49 MiB/s wr, 786 op/s 2024-08-17T17:28:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:45 smithi142 ceph-mon[106729]: pgmap v981: 89 pgs: 89 active+clean; 7.4 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 49 MiB/s wr, 786 op/s 2024-08-17T17:28:45.653 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:45.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:45.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:45.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:45.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:45.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:46.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:46.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:46.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:46.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:46.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:46.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:46.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:46.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:46.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:46.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:46.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:46.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:46.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:46.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:46.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:46.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:47.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:47.027 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:47.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:47.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:47.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:47.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:47.286 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:47.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:47 smithi142 ceph-mon[106729]: pgmap v982: 89 pgs: 89 active+clean; 7.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 115 B/s rd, 54 MiB/s wr, 944 op/s 2024-08-17T17:28:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:47 smithi142 ceph-mon[103614]: pgmap v982: 89 pgs: 89 active+clean; 7.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 115 B/s rd, 54 MiB/s wr, 944 op/s 2024-08-17T17:28:47.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:47.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:47.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:47.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:47 smithi152 ceph-mon[93456]: pgmap v982: 89 pgs: 89 active+clean; 7.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 115 B/s rd, 54 MiB/s wr, 944 op/s 2024-08-17T17:28:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:47.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:47.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:28:47.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:47.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:47.816 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:47.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:47.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:47.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.021 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:48.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:48.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:48.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:28:48.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:48.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:48.275 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:48.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:48.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:48.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:48.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:48.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:48.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:48.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:48.938 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:49.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:49.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:49.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:49.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:49.224 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:28:49.224 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:49.271 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:28:49.271 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:49.310 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:28:49.310 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:49.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:49.345 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:28:49.346 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:49.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:49.375 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:28:49.376 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:49.405 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:49 smithi142 ceph-mon[103614]: pgmap v983: 89 pgs: 89 active+clean; 7.4 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 862 op/s 2024-08-17T17:28:49.405 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3652722884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:49.405 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:49 smithi142 ceph-mon[106729]: pgmap v983: 89 pgs: 89 active+clean; 7.4 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 862 op/s 2024-08-17T17:28:49.406 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3652722884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:49.420 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:28:49.421 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:49.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:49.468 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:28:49.468 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:49.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:49.505 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:28:49.505 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:49 smithi152 ceph-mon[93456]: pgmap v983: 89 pgs: 89 active+clean; 7.4 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 862 op/s 2024-08-17T17:28:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3652722884' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:49.589 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:49.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:49.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:49.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:49.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:49.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:50.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:50.042 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:50.143 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:50.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:50.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:50.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:50.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:50.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:50.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:50.543 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:28:50.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2847238305' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:50.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2847238305' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:50.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:50.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2847238305' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:28:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:28:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:50.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:50.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:50.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:50.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:50.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:50.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:50.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:50.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:50.974 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:51.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:51.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:51.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:51.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:51.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:51.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:51.252 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:51.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:28:51.281 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:51.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:51.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:51.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:51.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:51.500 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:51.542 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-mon[106729]: pgmap v984: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 846 op/s 2024-08-17T17:28:51.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2847238305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:51.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-mon[106729]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:28:51.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-mon[103614]: pgmap v984: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 846 op/s 2024-08-17T17:28:51.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2847238305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:51.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-mon[103614]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:28:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:51 smithi152 ceph-mon[93456]: pgmap v984: 89 pgs: 89 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 846 op/s 2024-08-17T17:28:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2847238305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:51 smithi152 ceph-mon[93456]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:28:51.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:51.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:51.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:51.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:51.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:51.831 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:28:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:28:51] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:28:51.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:51.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:52.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:52.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:52.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:52.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:52.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:52.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:52.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:52.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:28:52.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:52.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:52.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:52.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:52.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:52.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:52.768 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:52.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:52.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:52.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:52.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:53.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:53.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:53.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:53.183 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:53.285 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:53.350 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:53.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:53.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:53.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:53 smithi152 ceph-mon[93456]: pgmap v986: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 860 op/s 2024-08-17T17:28:53.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:53.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:53 smithi142 ceph-mon[106729]: pgmap v986: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 860 op/s 2024-08-17T17:28:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:53 smithi142 ceph-mon[103614]: pgmap v986: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 860 op/s 2024-08-17T17:28:53.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:53.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:53.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:53.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:53.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:53.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:54.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:54.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:54.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:54.250 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:54.250 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:54.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:54.325 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:54.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:54.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:54.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:54.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:54.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:54.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:54.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:54.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.878 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:133277 11.6 11.ab8712d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:54.875857+0000", 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.00037054500000000001, 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00039526699999999998, 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "queued for pg", 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 133277 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875857+0000", 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875857+0000", 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.880 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875857+0000", 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4999999999999998e-07 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875859+0000", 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1660000000000001e-06 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:54.881 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875865+0000", 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7039999999999999e-06 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875870+0000", 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.729e-06 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.882 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:133258 11.4 e297/283 11:29662cd0:::benchmark_data_smithi142_227468_object9839:head v 297'8318, mlcod=297'8315)", 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:54.857952+0000", 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.018275461999999999, 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018359940000000002, 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:54.883 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.857952+0000", 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.857952+0000", 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.857953+0000", 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0999999999999997e-07 2024-08-17T17:28:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.857959+0000", 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3890000000000002e-06 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.858184+0000", 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022499900000000001 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.858193+0000", 2024-08-17T17:28:54.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8279999999999992e-06 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.858342+0000", 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014935600000000001 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.858362+0000", 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0021e-05 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.886 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:133261 11.c e297/283 11:3b849017:::benchmark_data_smithi142_227468_object9842:head v 297'8350, mlcod=297'8346)", 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:54.861644+0000", 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.014583113, 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014718635000000001, 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861644+0000", 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861644+0000", 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861645+0000", 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.09e-07 2024-08-17T17:28:54.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861651+0000", 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0460000000000002e-06 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861666+0000", 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5353999999999999e-05 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:54.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861673+0000", 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5440000000000004e-06 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861891+0000", 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00021808600000000001 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861989+0000", 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7898000000000004e-05 2024-08-17T17:28:54.890 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:133262 11.4 e297/283 11:2e8734e5:::benchmark_data_smithi142_227468_object9843:head v 297'8319, mlcod=297'8315)", 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:54.861699+0000", 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.014527841, 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014712067000000001, 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.891 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861699+0000", 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861699+0000", 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:54.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861700+0000", 2024-08-17T17:28:54.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5199999999999998e-07 2024-08-17T17:28:54.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.893 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:54.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861706+0000", 2024-08-17T17:28:54.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4969999999999998e-06 2024-08-17T17:28:54.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:54.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861713+0000", 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5470000000000002e-06 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.861717+0000", 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2830000000000003e-06 2024-08-17T17:28:54.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.867027+0000", 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053100140000000001 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:54.896 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.867045+0000", 2024-08-17T17:28:54.897 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7473e-05 2024-08-17T17:28:54.897 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.897 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:54.897 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.897 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.897 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.898 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:133268 11.4 e297/283 11:2128ac3d:::benchmark_data_smithi142_227468_object9849:head v 297'8320, mlcod=297'8316)", 2024-08-17T17:28:54.898 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:54.866337+0000", 2024-08-17T17:28:54.898 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0098906399999999992, 2024-08-17T17:28:54.898 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010127137, 2024-08-17T17:28:54.898 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:54.898 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:54.899 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:28:54.899 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:54.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:54.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.866337+0000", 2024-08-17T17:28:54.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.866337+0000", 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.866338+0000", 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.003e-06 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.866342+0000", 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7419999999999997e-06 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:54.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.866357+0000", 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4837999999999999e-05 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.866364+0000", 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1389999999999998e-06 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.874375+0000", 2024-08-17T17:28:54.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080107440000000002 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.874396+0000", 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0593000000000001e-05 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:133275 11.2 11:45b4d509:::benchmark_data_smithi142_227468_object9856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2024-08-17T17:28:54.903 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:54.875796+0000", 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.00043088300000000001, 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00071578899999999999, 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 133275 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:28:54.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875796+0000", 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875796+0000", 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875798+0000", 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.841e-06 2024-08-17T17:28:54.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875805+0000", 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3100000000000003e-06 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875825+0000", 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.98e-05 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875834+0000", 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4589999999999998e-06 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.875924+0000", 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0022e-05 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.876038+0000", 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011404999999999999 2024-08-17T17:28:54.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:54.876138+0000", 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9691000000000005e-05 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.908 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:28:54.909 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:28:54.909 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 6 2024-08-17T17:28:54.909 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:28:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:54.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:55.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:55.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:55.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:55.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:55.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:55.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:55 smithi142 ceph-mon[106729]: pgmap v987: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 12 MiB/s wr, 866 op/s 2024-08-17T17:28:55.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:55 smithi142 ceph-mon[103614]: pgmap v987: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 12 MiB/s wr, 866 op/s 2024-08-17T17:28:55.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:55.461 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:55.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:55 smithi152 ceph-mon[93456]: pgmap v987: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 12 MiB/s wr, 866 op/s 2024-08-17T17:28:55.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:55.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:55.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:55.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:55.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:55.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:55.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:55.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:56.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:56.079 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:28:56.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:56.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:56.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:56.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:56.310 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2065144266' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:56.310 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2065144266' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:56.311 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:28:56.323 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:56.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:56.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:56.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:56.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2065144266' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:56.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:56.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:56.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:56.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:56.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:56.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:56.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:56.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:57.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:57.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:57.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:57.249 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:57.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:57.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:57.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:57.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:57 smithi152 ceph-mon[93456]: pgmap v988: 89 pgs: 89 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 1.4 MiB/s wr, 863 op/s 2024-08-17T17:28:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:57 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2065144266' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:57 smithi152 ceph-mon[93456]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:28:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:57.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[106729]: pgmap v988: 89 pgs: 89 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 1.4 MiB/s wr, 863 op/s 2024-08-17T17:28:57.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:57.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2065144266' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:57.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[106729]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:28:57.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:57.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:57.587 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[103614]: pgmap v988: 89 pgs: 89 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 1.4 MiB/s wr, 863 op/s 2024-08-17T17:28:57.587 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:57.587 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2065144266' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:57.588 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[103614]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:28:57.588 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:28:57.588 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:57.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:57.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:57.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:57.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:57.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:57.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:57.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:58.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:58.073 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:58.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:58.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:58.316 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:58.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:58.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:58.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:58.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:58.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:58.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:58.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:58.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:58.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:58.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:59.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:59.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:59.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:59.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:59.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:59.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:59.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:59.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:59.451 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:59.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:28:59 smithi152 ceph-mon[93456]: pgmap v990: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:28:59.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:59.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:59.591 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:28:59 smithi142 ceph-mon[103614]: pgmap v990: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:28:59.591 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:28:59 smithi142 ceph-mon[106729]: pgmap v990: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:28:59.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:59.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:59.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:59.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:28:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:59.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:59.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:28:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:00.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:00.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:00.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:00.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:00.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:00.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:29:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:29:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:29:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:00.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:00.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:00.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:00.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:00.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:00.891 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:00.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:01.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:01.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:01.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:01.185 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:29:01.185 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:01.239 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:29:01.239 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:01.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:01.290 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:29:01.295 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:29:01.295 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:01.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:01.352 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:29:01.352 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:01.390 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:29:01.390 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:01.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:01.428 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:29:01.428 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:01.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:01.457 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:29:01.457 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:01.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-mon[103614]: pgmap v991: 89 pgs: 89 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:29:01.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/259148645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:01.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1295489677' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:01.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-mon[106729]: pgmap v991: 89 pgs: 89 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:29:01.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/259148645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:01.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1295489677' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:01.495 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:29:01.495 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:01.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:01 smithi152 ceph-mon[93456]: pgmap v991: 89 pgs: 89 active+clean; 6.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:29:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/259148645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1295489677' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:01.578 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:01.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:01.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:01.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:01.853 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:01.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:01.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:01.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:02.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:02.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:29:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:29:01] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-08-17T17:29:02.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:02.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:02.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:02.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:02.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:02.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:02.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:02.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1295489677' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:29:02.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:02 smithi152 ceph-mon[93456]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:29:02.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:02.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1295489677' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:29:02.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:02 smithi142 ceph-mon[103614]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:29:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1295489677' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:29:02.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:02 smithi142 ceph-mon[106729]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:29:02.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:02.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:02.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:02.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:02.903 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:02.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:02.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:02.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:29:03.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:03.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:03.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:03.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:03.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:03.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:03.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:03.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:03.508 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:03 smithi142 ceph-mon[103614]: pgmap v993: 89 pgs: 89 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:29:03.508 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:03 smithi142 ceph-mon[106729]: pgmap v993: 89 pgs: 89 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:29:03.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:03 smithi152 ceph-mon[93456]: pgmap v993: 89 pgs: 89 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:29:03.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:03.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:03.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:03.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:03.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:03.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:03.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.923 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:03.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:04.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:04.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:04.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:04.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:04.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:04.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:04.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:04.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:04.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:04.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:04.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:04.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:04.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:04.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:04.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:04.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:04.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:04.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:04.876 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:04.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:05.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:05.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:05.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:05.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:05.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:05.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:05.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:05.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:05.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:05 smithi142 ceph-mon[103614]: pgmap v994: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:29:05.537 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:05 smithi142 ceph-mon[106729]: pgmap v994: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:29:05.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:05.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:05 smithi152 ceph-mon[93456]: pgmap v994: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:29:05.631 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:05.631 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:05.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:05.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:05.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:05.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:05.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:05.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:05.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:06.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:06.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:06.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:06.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:06.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:06.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:06.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:06.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.589 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:06.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:06.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:06.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:06.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:06.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:06.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:07.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:07.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:07.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:07.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:07.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:07.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:07.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:07.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:07.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:07.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:07 smithi152 ceph-mon[93456]: pgmap v995: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-08-17T17:29:07.591 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:07.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:29:07.594 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:07 smithi142 ceph-mon[106729]: pgmap v995: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-08-17T17:29:07.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:07 smithi142 ceph-mon[103614]: pgmap v995: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-08-17T17:29:07.598 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:07.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:07.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:07.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:07.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:07.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:07.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:08.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:08.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:08.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:08.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:08.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:08.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:08.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:08.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:08.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:08.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:08.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:08.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:08.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:08.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:08.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:09.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:09.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:09.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:09.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:09.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:09 smithi142 ceph-mon[106729]: pgmap v996: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:29:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:09 smithi142 ceph-mon[103614]: pgmap v996: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:29:09.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:09.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:09.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:09.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:09 smithi152 ceph-mon[93456]: pgmap v996: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:29:09.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:10.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:10.069 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:10.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:10.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:10.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:10.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:10.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:10.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:10.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:10.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:10.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:29:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:29:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:10.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:10.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:10.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:10.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:10.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:10.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:11.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:11.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:11.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:11.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:11.262 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:11.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:11.363 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:29:11.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:11.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:11.486 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:11 smithi142 ceph-mon[106729]: pgmap v997: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2024-08-17T17:29:11.486 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2334694273' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:29:11.487 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:11 smithi142 ceph-mon[103614]: pgmap v997: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2024-08-17T17:29:11.487 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2334694273' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:29:11.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:11.520 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:11.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:11.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:11.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:11.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.783 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:11 smithi152 ceph-mon[93456]: pgmap v997: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 859 op/s 2024-08-17T17:29:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2334694273' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:29:11.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:11.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:12.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:12.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:29:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:29:11] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:29:12.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:12.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:12.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:12.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:12.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:12.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:12.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2334694273' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:29:12.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[103614]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:29:12.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:12.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[103614]: pgmap v999: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-17T17:29:12.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:12.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:12.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:12.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:12.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:12.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:12.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:12.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2334694273' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:29:12.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[106729]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:29:12.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:12.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[106729]: pgmap v999: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-17T17:29:12.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:12.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:12.721 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:29:12.721 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:12.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:12.758 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:29:12.758 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:12.789 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:29:12.790 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2334694273' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:29:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:12 smithi152 ceph-mon[93456]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:29:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:12 smithi152 ceph-mon[93456]: pgmap v999: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-17T17:29:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:12.823 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:29:12.823 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:12.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:12.853 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:29:12.853 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:12.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:12.882 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:29:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.882 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:12.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:12.908 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:29:12.909 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:12.935 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:29:12.936 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:12.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:13.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:13.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:13.138 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:13.163 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:13.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:13.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:13.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:13.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:13.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:13.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:13.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:13.558 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/692238231' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/692238231' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:13.660 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:13.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:13.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/692238231' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:13.851 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:13.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:13.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:14.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:14.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:14.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:14.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:14.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:14.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:14.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:14.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:14.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:14 smithi142 ceph-mon[106729]: pgmap v1000: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-08-17T17:29:14.569 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:14 smithi142 ceph-mon[103614]: pgmap v1000: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-08-17T17:29:14.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:14.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:14.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:14.812 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:14.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:14 smithi152 ceph-mon[93456]: pgmap v1000: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-08-17T17:29:14.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:14.938 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:15.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:15.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:15.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:15.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:15.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:15.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:15.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:15.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:15.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:15.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:15.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:15.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:15.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:15.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:15.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:15.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:15.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:16.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:16.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:16.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:16.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:16.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:16.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:16.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:16.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:16.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:16.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:16.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:16.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:16.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:16.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:16.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:17.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:17.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:17.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:17.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:17.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:17 smithi152 ceph-mon[93456]: pgmap v1001: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-17T17:29:17.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:17.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:17.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:17.560 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:17.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:17 smithi142 ceph-mon[106729]: pgmap v1001: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-17T17:29:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:17 smithi142 ceph-mon[103614]: pgmap v1001: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-17T17:29:17.662 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:17.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:17.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:17.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:17.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:18.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:18.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:18.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:18.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:18.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:18.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:18.480 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:29:18.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:18.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:116253 10.8 10:1d55b2b8:::benchmark_data_smithi142_192453_object116252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:46.636710+0000", 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 331.928603485, 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0213902429999999, 2024-08-17T17:29:18.569 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 116253 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636710+0000", 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636710+0000", 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636712+0000", 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1890000000000001e-06 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.571 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636768+0000", 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5698000000000001e-05 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636782+0000", 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3881e-05 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636792+0000", 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5429999999999999e-06 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640545+0000", 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037532910000000002 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640553+0000", 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6960000000000005e-06 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.573 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.646220+0000", 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.00566713 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.646818+0000", 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000598268 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.657964+0000", 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011146201 2024-08-17T17:29:18.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.658101+0000", 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000136346 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:116255 10.8 10:1c6e316a:::benchmark_data_smithi142_192453_object116254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:29:18.575 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 331.92850690699998, 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.072248343, 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 116255 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.576 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636807+0000", 2024-08-17T17:29:18.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0800000000000004e-07 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636851+0000", 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3763000000000001e-05 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636860+0000", 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2280000000000004e-06 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.578 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.636867+0000", 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3380000000000001e-06 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640569+0000", 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037025130000000002 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:46.640574+0000", 2024-08-17T17:29:18.579 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1789999999999999e-06 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.647111+0000", 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.006536724 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.647199+0000", 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7706000000000004e-05 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.657395+0000", 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010196026 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.657453+0000", 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8026000000000003e-05 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.708930+0000", 2024-08-17T17:29:18.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.051476916999999997 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:23:47.709055+0000", 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000125315 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144807 10.8 10:12e5ad24:::benchmark_data_smithi142_192453_object19059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:29:18.582 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.832998+0000", 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 282.73231554099999, 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3307359860000001, 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144807 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.584 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.832998+0000", 2024-08-17T17:29:18.584 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.584 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.584 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.832998+0000", 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833000+0000", 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.99e-06 2024-08-17T17:29:18.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833006+0000", 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9730000000000002e-06 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833022+0000", 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5475999999999999e-05 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.833029+0000", 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.498e-06 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.851615+0000", 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018585476 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:29:18.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.851882+0000", 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026769800000000002 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862610+0000", 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010727571999999999 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862694+0000", 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3969999999999997e-05 2024-08-17T17:29:18.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862817+0000", 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012301799999999999 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.895911+0000", 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.033093769000000002 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.896238+0000", 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000326967 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.158715+0000", 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2624777950000001 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.158913+0000", 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019719 2024-08-17T17:29:18.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.163684+0000", 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0047715539999999999 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.163734+0000", 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0040000000000002e-05 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.591 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:144822 10.5 10:a9630a4c:::benchmark_data_smithi142_192453_object19074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:24:35.861787+0000", 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 282.70352699300003, 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3618416280000001, 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.592 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 144822 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861787+0000", 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861787+0000", 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861788+0000", 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.221e-06 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861793+0000", 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5029999999999997e-06 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861807+0000", 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3886000000000001e-05 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.861815+0000", 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8280000000000003e-06 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862564+0000", 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000748663 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862724+0000", 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015982799999999999 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.862853+0000", 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012978299999999999 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.895758+0000", 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032904920999999997 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:35.895772+0000", 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3959e-05 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.188975+0000", 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.293202934 2024-08-17T17:29:18.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.189529+0000", 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055367700000000003 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.223551+0000", 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034022377999999999 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:24:37.223628+0000", 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7046999999999996e-05 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:178984 10.5 10:a2f12403:::benchmark_data_smithi142_192453_object53236:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751405+0000", 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.813908349, 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0189305549999998, 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.599 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 178984 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751405+0000", 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751405+0000", 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751407+0000", 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3170000000000001e-06 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751411+0000", 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5619999999999997e-06 2024-08-17T17:29:18.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751422+0000", 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.095e-05 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751428+0000", 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9699999999999996e-06 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247757+0000", 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49632847200000002 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247782+0000", 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5482000000000001e-05 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774427+0000", 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.52664463699999997 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774438+0000", 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0951999999999999e-05 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770249+0000", 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99581095200000003 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770336+0000", 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7261000000000004e-05 2024-08-17T17:29:18.604 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179006 10.5 10:a5e93376:::benchmark_data_smithi142_192453_object53258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.813875878, 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0192025789999999, 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.605 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179006 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:29:18.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751438+0000", 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9400000000000001e-07 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751440+0000", 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9819999999999998e-06 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751445+0000", 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0710000000000002e-06 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751449+0000", 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.031e-06 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247903+0000", 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49645393399999999 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.247926+0000", 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2846000000000001e-05 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774536+0000", 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.52661009800000003 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774543+0000", 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1520000000000003e-06 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770583+0000", 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99603933700000002 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770640+0000", 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7734e-05 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179009 10.15 10:adfe3263:::benchmark_data_smithi142_192453_object53261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751454+0000", 2024-08-17T17:29:18.611 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.813859374, 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.103763888, 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179009 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.612 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751454+0000", 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751454+0000", 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751455+0000", 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0600000000000001e-07 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751456+0000", 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7460000000000001e-06 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751461+0000", 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8539999999999996e-06 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751465+0000", 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8240000000000001e-06 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249382+0000", 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49791667099999998 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249402+0000", 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9830000000000002e-05 2024-08-17T17:29:18.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855145+0000", 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.60574357300000004 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855218+0000", 2024-08-17T17:29:18.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3083999999999993e-05 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179011 10.13 10:cac77333:::benchmark_data_smithi142_192453_object53263:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.81384420000001, 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1257471589999999, 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.617 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179011 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:29:18.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751470+0000", 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9700000000000003e-07 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751472+0000", 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1390000000000002e-06 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751477+0000", 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751481+0000", 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0670000000000002e-06 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.218181+0000", 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.46670027200000003 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.218189+0000", 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4260000000000003e-06 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877127+0000", 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.65893728600000001 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877217+0000", 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9952999999999997e-05 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179016 10.5 10:a4006e9a:::benchmark_data_smithi142_192453_object53268:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751485+0000", 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.81382830499999, 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1085708730000001, 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.623 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179016 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751485+0000", 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751485+0000", 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751486+0000", 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.03e-07 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751487+0000", 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.64e-06 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751492+0000", 2024-08-17T17:29:18.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0080000000000002e-06 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751497+0000", 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1710000000000004e-06 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.248656+0000", 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49715965400000001 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.248673+0000", 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7258e-05 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774566+0000", 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.52589242199999997 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.774573+0000", 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0589999999999997e-06 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770664+0000", 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99609061799999998 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770773+0000", 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109517 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.770911+0000", 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013743100000000001 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779028+0000", 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0081173779999999997 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779042+0000", 2024-08-17T17:29:18.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.414e-05 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.859820+0000", 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.080778325999999998 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.859900+0000", 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9658000000000002e-05 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.860056+0000", 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015629000000000001 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.632 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179025 10.3 10:c56d5223:::benchmark_data_smithi142_192453_object53277:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.632 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751501+0000", 2024-08-17T17:29:18.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:18.633 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.81381250499999, 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.083066557, 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179025 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.634 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751501+0000", 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751501+0000", 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751502+0000", 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7399999999999999e-07 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751503+0000", 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6199999999999999e-06 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751508+0000", 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9169999999999997e-06 2024-08-17T17:29:18.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751512+0000", 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8920000000000002e-06 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.216846+0000", 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.46533451799999997 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.216856+0000", 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2410000000000001e-06 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.805891+0000", 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.58903523800000002 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.806054+0000", 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000163091 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.806177+0000", 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012318400000000001 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.811229+0000", 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0050514490000000004 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.819512+0000", 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0082837770000000008 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.834457+0000", 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014945007999999999 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.834516+0000", 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8554000000000002e-05 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.834568+0000", 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1793999999999999e-05 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179027 10.15 10:a8f69768:::benchmark_data_smithi142_192453_object53279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 236.813797601, 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1410192640000001, 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179027 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.642 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751516+0000", 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3999999999999998e-07 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751518+0000", 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3960000000000001e-06 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751523+0000", 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1220000000000003e-06 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:21.751529+0000", 2024-08-17T17:29:18.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7899999999999996e-06 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249462+0000", 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.49793338700000001 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.249473+0000", 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0618999999999999e-05 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855498+0000", 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.60602568700000004 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855628+0000", 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012980199999999999 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.855770+0000", 2024-08-17T17:29:18.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014220500000000001 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.890485+0000", 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034714936000000002 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.890523+0000", 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8160000000000001e-05 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.892352+0000", 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001828367 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.892390+0000", 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8386000000000001e-05 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.892535+0000", 2024-08-17T17:29:18.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000145167 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179051 10.18 10:1ac43054:::benchmark_data_smithi142_192453_object53303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.842100+0000", 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 235.72321337100001, 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9537706620000002, 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.650 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179051 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842100+0000", 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842100+0000", 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842102+0000", 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.807e-06 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842109+0000", 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3999999999999997e-06 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842125+0000", 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6248000000000001e-05 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842135+0000", 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.045e-05 2024-08-17T17:29:18.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842196+0000", 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0943999999999998e-05 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.842211+0000", 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.429e-05 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.746081+0000", 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.90387036200000004 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.746091+0000", 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0304e-05 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.765556+0000", 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0194644129999999 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.765695+0000", 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013904100000000001 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.765830+0000", 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013585800000000001 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795604+0000", 2024-08-17T17:29:18.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.029773871 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795734+0000", 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012952800000000001 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795758+0000", 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4521000000000001e-05 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795799+0000", 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0237000000000001e-05 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:24.795871+0000", 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2387999999999999e-05 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.658 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183179 10.18 10:1c594a47:::benchmark_data_smithi142_192453_object57431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.766992+0000", 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.79832174000001, 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1695684370000001, 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183179 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.659 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766992+0000", 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766992+0000", 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766993+0000", 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1349999999999999e-06 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.766998+0000", 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6430000000000003e-06 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767012+0000", 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4429e-05 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767020+0000", 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4440000000000002e-06 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767100+0000", 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0448e-05 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767261+0000", 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016068699999999999 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.767846+0000", 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058507100000000005 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793461+0000", 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025615427 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793468+0000", 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8199999999999999e-06 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785176+0000", 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99170805799999995 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785186+0000", 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.876e-06 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.931901+0000", 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1467151289999999 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.931968+0000", 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7092000000000001e-05 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936479+0000", 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045107200000000002 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936560+0000", 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1458000000000005e-05 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.666 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183181 10.19 10:9d925f31:::benchmark_data_smithi142_192453_object57433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.769548+0000", 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.79576593499999, 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.040106049, 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183181 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769548+0000", 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769548+0000", 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769549+0000", 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.291e-06 2024-08-17T17:29:18.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769554+0000", 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6190000000000002e-06 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769563+0000", 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5640000000000002e-06 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.769570+0000", 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4969999999999998e-06 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.780479+0000", 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010908800999999999 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.780842+0000", 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036355600000000001 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.781296+0000", 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045431300000000003 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795305+0000", 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014008270999999999 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795314+0000", 2024-08-17T17:29:18.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9050000000000003e-06 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.809578+0000", 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0142647600000001 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.809654+0000", 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5471999999999998e-05 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.672 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183184 10.8 10:16da935b:::benchmark_data_smithi142_192453_object57436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.79467504900001, 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1660231830000001, 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.673 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183184 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770639+0000", 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6899999999999997e-07 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770642+0000", 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1190000000000001e-06 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770651+0000", 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3580000000000003e-06 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770656+0000", 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7579999999999998e-06 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.771143+0000", 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000486279 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.771313+0000", 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170583 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.771913+0000", 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059953100000000004 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793423+0000", 2024-08-17T17:29:18.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.021510226 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.793434+0000", 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0453e-05 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785060+0000", 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99162608100000005 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.785075+0000", 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.502e-05 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.930179+0000", 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1451046819999999 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for missing object", 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.930300+0000", 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121138 2024-08-17T17:29:18.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936593+0000", 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0062929860000000004 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:29.936662+0000", 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8399999999999996e-05 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.680 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183185 10.19 10:9d6db525:::benchmark_data_smithi142_192453_object57437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.794643662, 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.07294892, 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183185 2024-08-17T17:29:18.681 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770670+0000", 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6899999999999998e-07 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770672+0000", 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8300000000000001e-06 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770678+0000", 2024-08-17T17:29:18.683 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3550000000000004e-06 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.770683+0000", 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0529999999999997e-06 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795338+0000", 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024655763000000001 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.684 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795344+0000", 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1379999999999998e-06 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.812566+0000", 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0172222440000001 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.812692+0000", 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012577399999999999 2024-08-17T17:29:18.685 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.813081+0000", 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000389436 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826624+0000", 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013542903 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.686 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843473+0000", 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016849122000000001 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843525+0000", 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1595999999999999e-05 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843619+0000", 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4337000000000002e-05 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.687 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.688 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.688 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.688 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.689 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183189 10.19 10:990f2256:::benchmark_data_smithi142_192453_object57441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:29:18.689 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.786460+0000", 2024-08-17T17:29:18.689 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.778853344, 2024-08-17T17:29:18.689 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.057317346, 2024-08-17T17:29:18.689 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183189 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786460+0000", 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.690 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786460+0000", 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786462+0000", 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.455e-06 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.691 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786468+0000", 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5549999999999999e-06 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786482+0000", 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3573e-05 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.786490+0000", 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2069999999999995e-06 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.692 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795363+0000", 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0088728950000000004 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.795368+0000", 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0409999999999997e-06 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.816415+0000", 2024-08-17T17:29:18.693 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0210472370000001 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.816550+0000", 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135088 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.817391+0000", 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00084071099999999995 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.694 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826630+0000", 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0092389719999999998 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843658+0000", 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017028002 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843701+0000", 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2864999999999997e-05 2024-08-17T17:29:18.695 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.696 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.696 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.696 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.843778+0000", 2024-08-17T17:29:18.696 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6744999999999998e-05 2024-08-17T17:29:18.696 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183201 10.19 10:9f2fe008:::benchmark_data_smithi142_192453_object57453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.804721+0000", 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.76059247699999, 2024-08-17T17:29:18.698 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0521881289999999, 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183201 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804721+0000", 2024-08-17T17:29:18.699 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804721+0000", 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804722+0000", 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1510000000000001e-06 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.700 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804727+0000", 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2579999999999997e-06 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804736+0000", 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2990000000000002e-06 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.804742+0000", 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.302e-06 2024-08-17T17:29:18.701 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.805312+0000", 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00056939800000000004 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.805322+0000", 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0081000000000001e-05 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.702 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.819432+0000", 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0141100670000001 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.819562+0000", 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013033599999999999 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.819930+0000", 2024-08-17T17:29:18.703 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000368045 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826635+0000", 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.006704659 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856588+0000", 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.029952880000000001 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.704 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856794+0000", 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000206164 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856909+0000", 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011548900000000001 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.705 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183209 10.19 10:9d6a895e:::benchmark_data_smithi142_192453_object57461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.812976+0000", 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.75233754300001, 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.044152341, 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183209 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.706 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812976+0000", 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812976+0000", 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.707 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812978+0000", 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.889e-06 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.812985+0000", 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8120000000000002e-06 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813002+0000", 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6755000000000001e-05 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.708 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813010+0000", 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813411+0000", 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00040115700000000002 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.813419+0000", 2024-08-17T17:29:18.709 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.799e-06 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.822029+0000", 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0086097350000001 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.822159+0000", 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013015300000000001 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.710 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.822521+0000", 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036132200000000002 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826640+0000", 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041192449999999997 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.856970+0000", 2024-08-17T17:29:18.711 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030330182000000001 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857027+0000", 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6888999999999997e-05 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857129+0000", 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000101544 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.712 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:183218 10.19 10:9950a22e:::benchmark_data_smithi142_192453_object57470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:27.823832+0000", 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 230.741481765, 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.03347671, 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:29:18.713 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 183218 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823832+0000", 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823832+0000", 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.714 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823833+0000", 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2750000000000001e-06 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823838+0000", 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2769999999999999e-06 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823851+0000", 2024-08-17T17:29:18.715 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3560999999999999e-05 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823858+0000", 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2490000000000004e-06 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823925+0000", 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6742000000000006e-05 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.716 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:27.823933+0000", 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6389999999999992e-06 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.824484+0000", 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0005513880000001 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.824622+0000", 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013785499999999999 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.717 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5", 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.825177+0000", 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055459499999999996 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.826740+0000", 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001563191 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857177+0000", 2024-08-17T17:29:18.718 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030437740000000001 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857227+0000", 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9391000000000002e-05 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:28.857309+0000", 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1806999999999998e-05 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.719 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.720 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:18.720 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:18.720 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:29:18.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:18.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:18.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[103614]: pgmap v1002: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:29:18.721 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4213032435' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:18.721 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.721 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:18.721 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:18.721 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[106729]: pgmap v1002: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:29:18.721 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4213032435' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:18.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:18 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:18.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:18.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:18 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:18 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:18 smithi152 ceph-mon[93456]: pgmap v1002: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:29:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4213032435' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:18 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:18.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:18.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:18.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:18.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:19.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:19.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:19.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.262 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:29:19.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:29:19.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:19.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:19.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:19.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:19.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:19.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:19 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:19 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:19 smithi142 ceph-mon[103614]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:19 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:19 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:19.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:19 smithi142 ceph-mon[106729]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:29:19.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:19.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.821 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:19 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:19 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:19 smithi152 ceph-mon[93456]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:29:19.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:19.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.006 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:20.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:20.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:20.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:20.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:29:20.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:20.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:20.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:20.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:20.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:20.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:20.473 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:29:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:29:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:20.475 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:20 smithi142 ceph-mon[103614]: pgmap v1004: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:29:20.523 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:20.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:20.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:20.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:20.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:20.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:20.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:20 smithi152 ceph-mon[93456]: pgmap v1004: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:29:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:20.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:20 smithi142 ceph-mon[106729]: pgmap v1004: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:29:20.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.996 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:20.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:21.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:21.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:21.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:21.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:21.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:21.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:21.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:21.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:21.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:21.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:21.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:21.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:21.777 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:21.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:21.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:29:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:29:21] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:29:21.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:21.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:21.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:22.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:22.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:22.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:22.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:22.342 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:22.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:22.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:22.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:22.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:22.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:22.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:22.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:22.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:22.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:23.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:23.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:23.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:29:23.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:23.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:23.217 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:23 smithi142 ceph-mon[106729]: pgmap v1005: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:29:23.218 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:23 smithi142 ceph-mon[103614]: pgmap v1005: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:29:23.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:23.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:23.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:23.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:23.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:23 smithi152 ceph-mon[93456]: pgmap v1005: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:29:23.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:23.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:23.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:23.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:23.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:23.889 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:23.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:24.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:24.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:24.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:24.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:24.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:24.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:24.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:24.431 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:24.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:24.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:24.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:24.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:24.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:24.732 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:24.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:24.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:24.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:25.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:25.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:25.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:25.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:25.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:25.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:25.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:25.467 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:25 smithi142 ceph-mon[103614]: pgmap v1006: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:29:25.467 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:25 smithi142 ceph-mon[106729]: pgmap v1006: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:29:25.508 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:25.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:25 smithi152 ceph-mon[93456]: pgmap v1006: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:29:25.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:25.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:25.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:25.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:25.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:25.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:25.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:26.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:26.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:26.276 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:26.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:26.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:26.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:26.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:26.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:26.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:26.967 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:29:26.970 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:29:27.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:27.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:27.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:27.287 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:27 smithi142 ceph-mon[106729]: pgmap v1007: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:29:27.287 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:27.287 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:27 smithi142 ceph-mon[103614]: pgmap v1007: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:29:27.288 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:27.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:27.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:27.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:27.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:27.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:27 smithi152 ceph-mon[93456]: pgmap v1007: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:29:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:27.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:27.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:27.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:27.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:27.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:27.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:28.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:28.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:28.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:28.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:28.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:28.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:28.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:28.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:28 smithi142 ceph-mon[103614]: from='client.56122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:28.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:28 smithi142 ceph-mon[106729]: from='client.56122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:28.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:28.455 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1007,"stamp":"2024-08-17T17:29:26.084885+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5835528165,"num_objects":89061,"num_object_clones":0,"num_object_copies":267183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89061,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":158077,"num_write_kb":7901072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2362634,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137606,"ondisk_log_size":137606,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22835344,"kb_used_data":17295056,"kb_used_omap":45,"kb_used_meta":5540050,"kb_avail":727125872,"statfs":{"total":767960285184,"available":744576892928,"internally_reserved":0,"allocated":17710137344,"data_stored":17697782079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47063,"internal_metadata":5673011241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,7,4,4,3,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-532348928,"num_objects":-8123,"num_object_clones":0,"num_object_copies":-24369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8123,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003465"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":426,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.490628+0000","last_change":"2024-08-17T17:26:32.212092+0000","last_active":"2024-08-17T17:29:18.490628+0000","last_peered":"2024-08-17T17:29:18.490628+0000","last_clean":"2024-08-17T17:29:18.490628+0000","last_became_active":"2024-08-17T17:26:32.211774+0000","last_became_peered":"2024-08-17T17:26:32.211774+0000","last_unstale":"2024-08-17T17:29:18.490628+0000","last_undegraded":"2024-08-17T17:29:18.490628+0000","last_fullsized":"2024-08-17T17:29:18.490628+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:45.518565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":246,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.369982+0000","last_change":"2024-08-17T17:26:32.210903+0000","last_active":"2024-08-17T17:29:11.369982+0000","last_peered":"2024-08-17T17:29:11.369982+0000","last_clean":"2024-08-17T17:29:11.369982+0000","last_became_active":"2024-08-17T17:26:32.209823+0000","last_became_peered":"2024-08-17T17:26:32.209823+0000","last_unstale":"2024-08-17T17:29:11.369982+0000","last_undegraded":"2024-08-17T17:29:11.369982+0000","last_fullsized":"2024-08-17T17:29:11.369982+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:54:17.717352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":194,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.380203+0000","last_change":"2024-08-17T17:26:32.205305+0000","last_active":"2024-08-17T17:29:11.380203+0000","last_peered":"2024-08-17T17:29:11.380203+0000","last_clean":"2024-08-17T17:29:11.380203+0000","last_became_active":"2024-08-17T17:26:32.205000+0000","last_became_peered":"2024-08-17T17:26:32.205000+0000","last_unstale":"2024-08-17T17:29:11.380203+0000","last_undegraded":"2024-08-17T17:29:11.380203+0000","last_fullsized":"2024-08-17T17:29:11.380203+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:31.144881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":229,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.369936+0000","last_change":"2024-08-17T17:26:32.213319+0000","last_active":"2024-08-17T17:29:11.369936+0000","last_peered":"2024-08-17T17:29:11.369936+0000","last_clean":"2024-08-17T17:29:11.369936+0000","last_became_active":"2024-08-17T17:26:32.213079+0000","last_became_peered":"2024-08-17T17:26:32.213079+0000","last_unstale":"2024-08-17T17:29:11.369936+0000","last_undegraded":"2024-08-17T17:29:11.369936+0000","last_fullsized":"2024-08-17T17:29:11.369936+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:43.552406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":308,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491131+0000","last_change":"2024-08-17T17:26:32.211539+0000","last_active":"2024-08-17T17:29:18.491131+0000","last_peered":"2024-08-17T17:29:18.491131+0000","last_clean":"2024-08-17T17:29:18.491131+0000","last_became_active":"2024-08-17T17:26:32.211267+0000","last_became_peered":"2024-08-17T17:26:32.211267+0000","last_unstale":"2024-08-17T17:29:18.491131+0000","last_undegraded":"2024-08-17T17:29:18.491131+0000","last_fullsized":"2024-08-17T17:29:18.491131+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:16.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.369891+0000","last_change":"2024-08-17T17:26:32.233887+0000","last_active":"2024-08-17T17:29:11.369891+0000","last_peered":"2024-08-17T17:29:11.369891+0000","last_clean":"2024-08-17T17:29:11.369891+0000","last_became_active":"2024-08-17T17:26:32.230472+0000","last_became_peered":"2024-08-17T17:26:32.230472+0000","last_unstale":"2024-08-17T17:29:11.369891+0000","last_undegraded":"2024-08-17T17:29:11.369891+0000","last_fullsized":"2024-08-17T17:29:11.369891+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:09.045150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":401,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491268+0000","last_change":"2024-08-17T17:26:32.207139+0000","last_active":"2024-08-17T17:29:18.491268+0000","last_peered":"2024-08-17T17:29:18.491268+0000","last_clean":"2024-08-17T17:29:18.491268+0000","last_became_active":"2024-08-17T17:26:32.206763+0000","last_became_peered":"2024-08-17T17:26:32.206763+0000","last_unstale":"2024-08-17T17:29:18.491268+0000","last_undegraded":"2024-08-17T17:29:18.491268+0000","last_fullsized":"2024-08-17T17:29:18.491268+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:51.396647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":321,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377186+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:29:11.377186+0000","last_peered":"2024-08-17T17:29:11.377186+0000","last_clean":"2024-08-17T17:29:11.377186+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:29:11.377186+0000","last_undegraded":"2024-08-17T17:29:11.377186+0000","last_fullsized":"2024-08-17T17:29:11.377186+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"301'10112","reported_seq":10138,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:25.052745+0000","last_change":"2024-08-17T17:27:05.299771+0000","last_active":"2024-08-17T17:29:25.052745+0000","last_peered":"2024-08-17T17:29:25.052745+0000","last_clean":"2024-08-17T17:29:25.052745+0000","last_became_active":"2024-08-17T17:27:05.299171+0000","last_became_peered":"2024-08-17T17:27:05.299171+0000","last_unstale":"2024-08-17T17:29:25.052745+0000","last_undegraded":"2024-08-17T17:29:25.052745+0000","last_fullsized":"2024-08-17T17:29:25.052745+0000","mapping_epoch":283,"log_start":"287'100","ondisk_log_start":"287'100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10012,"log_dups_size":0,"ondisk_log_size":10012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:25.524635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10112,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370327+0000","last_change":"2024-08-17T17:26:32.214168+0000","last_active":"2024-08-17T17:29:11.370327+0000","last_peered":"2024-08-17T17:29:11.370327+0000","last_clean":"2024-08-17T17:29:11.370327+0000","last_became_active":"2024-08-17T17:26:32.213844+0000","last_became_peered":"2024-08-17T17:26:32.213844+0000","last_unstale":"2024-08-17T17:29:11.370327+0000","last_undegraded":"2024-08-17T17:29:11.370327+0000","last_fullsized":"2024-08-17T17:29:11.370327+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:52.468566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":177,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377624+0000","last_change":"2024-08-17T17:26:32.218633+0000","last_active":"2024-08-17T17:29:11.377624+0000","last_peered":"2024-08-17T17:29:11.377624+0000","last_clean":"2024-08-17T17:29:11.377624+0000","last_became_active":"2024-08-17T17:26:32.218212+0000","last_became_peered":"2024-08-17T17:26:32.218212+0000","last_unstale":"2024-08-17T17:29:11.377624+0000","last_undegraded":"2024-08-17T17:29:11.377624+0000","last_fullsized":"2024-08-17T17:29:11.377624+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:55.971779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"301'9878","reported_seq":9926,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:23.855612+0000","last_change":"2024-08-17T17:27:05.818422+0000","last_active":"2024-08-17T17:29:23.855612+0000","last_peered":"2024-08-17T17:29:23.855612+0000","last_clean":"2024-08-17T17:29:23.855612+0000","last_became_active":"2024-08-17T17:27:05.818117+0000","last_became_peered":"2024-08-17T17:27:05.818117+0000","last_unstale":"2024-08-17T17:29:23.855612+0000","last_undegraded":"2024-08-17T17:29:23.855612+0000","last_fullsized":"2024-08-17T17:29:23.855612+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":9878,"log_dups_size":0,"ondisk_log_size":9878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:36.236107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365035520,"num_objects":5570,"num_object_clones":0,"num_object_copies":16710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9878,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":232,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370299+0000","last_change":"2024-08-17T17:26:32.220489+0000","last_active":"2024-08-17T17:29:11.370299+0000","last_peered":"2024-08-17T17:29:11.370299+0000","last_clean":"2024-08-17T17:29:11.370299+0000","last_became_active":"2024-08-17T17:26:32.220150+0000","last_became_peered":"2024-08-17T17:26:32.220150+0000","last_unstale":"2024-08-17T17:29:11.370299+0000","last_undegraded":"2024-08-17T17:29:11.370299+0000","last_fullsized":"2024-08-17T17:29:11.370299+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:09.954487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368065+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:29:11.368065+0000","last_peered":"2024-08-17T17:29:11.368065+0000","last_clean":"2024-08-17T17:29:11.368065+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:29:11.368065+0000","last_undegraded":"2024-08-17T17:29:11.368065+0000","last_fullsized":"2024-08-17T17:29:11.368065+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"301'9987","reported_seq":10013,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:25.299563+0000","last_change":"2024-08-17T17:27:05.297172+0000","last_active":"2024-08-17T17:29:25.299563+0000","last_peered":"2024-08-17T17:29:25.299563+0000","last_clean":"2024-08-17T17:29:25.299563+0000","last_became_active":"2024-08-17T17:27:05.296899+0000","last_became_peered":"2024-08-17T17:27:05.296899+0000","last_unstale":"2024-08-17T17:29:25.299563+0000","last_undegraded":"2024-08-17T17:29:25.299563+0000","last_fullsized":"2024-08-17T17:29:25.299563+0000","mapping_epoch":283,"log_start":"287'1600","ondisk_log_start":"287'1600","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8387,"log_dups_size":0,"ondisk_log_size":8387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:32.207215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358678528,"num_objects":5473,"num_object_clones":0,"num_object_copies":16419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9987,"num_write_kb":494720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":242,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370310+0000","last_change":"2024-08-17T17:26:32.216517+0000","last_active":"2024-08-17T17:29:11.370310+0000","last_peered":"2024-08-17T17:29:11.370310+0000","last_clean":"2024-08-17T17:29:11.370310+0000","last_became_active":"2024-08-17T17:26:32.216161+0000","last_became_peered":"2024-08-17T17:26:32.216161+0000","last_unstale":"2024-08-17T17:29:11.370310+0000","last_undegraded":"2024-08-17T17:29:11.370310+0000","last_fullsized":"2024-08-17T17:29:11.370310+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:39.712941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377425+0000","last_change":"2024-08-17T17:26:32.217706+0000","last_active":"2024-08-17T17:29:11.377425+0000","last_peered":"2024-08-17T17:29:11.377425+0000","last_clean":"2024-08-17T17:29:11.377425+0000","last_became_active":"2024-08-17T17:26:32.217107+0000","last_became_peered":"2024-08-17T17:26:32.217107+0000","last_unstale":"2024-08-17T17:29:11.377425+0000","last_undegraded":"2024-08-17T17:29:11.377425+0000","last_fullsized":"2024-08-17T17:29:11.377425+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:39.827920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"301'9932","reported_seq":9958,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:25.281376+0000","last_change":"2024-08-17T17:27:05.297778+0000","last_active":"2024-08-17T17:29:25.281376+0000","last_peered":"2024-08-17T17:29:25.281376+0000","last_clean":"2024-08-17T17:29:25.281376+0000","last_became_active":"2024-08-17T17:27:05.297367+0000","last_became_peered":"2024-08-17T17:27:05.297367+0000","last_unstale":"2024-08-17T17:29:25.281376+0000","last_undegraded":"2024-08-17T17:29:25.281376+0000","last_fullsized":"2024-08-17T17:29:25.281376+0000","mapping_epoch":283,"log_start":"287'1500","ondisk_log_start":"287'1500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8432,"log_dups_size":0,"ondisk_log_size":8432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.484840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360448000,"num_objects":5500,"num_object_clones":0,"num_object_copies":16500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9932,"num_write_kb":493824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":351,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491076+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:29:18.491076+0000","last_peered":"2024-08-17T17:29:18.491076+0000","last_clean":"2024-08-17T17:29:18.491076+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:29:18.491076+0000","last_undegraded":"2024-08-17T17:29:18.491076+0000","last_fullsized":"2024-08-17T17:29:18.491076+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.494344+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:29:18.494344+0000","last_peered":"2024-08-17T17:29:18.494344+0000","last_clean":"2024-08-17T17:29:18.494344+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:29:18.494344+0000","last_undegraded":"2024-08-17T17:29:18.494344+0000","last_fullsized":"2024-08-17T17:29:18.494344+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"301'9859","reported_seq":9884,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:22.945500+0000","last_change":"2024-08-17T17:27:05.297382+0000","last_active":"2024-08-17T17:29:22.945500+0000","last_peered":"2024-08-17T17:29:22.945500+0000","last_clean":"2024-08-17T17:29:22.945500+0000","last_became_active":"2024-08-17T17:27:05.297149+0000","last_became_peered":"2024-08-17T17:27:05.297149+0000","last_unstale":"2024-08-17T17:29:22.945500+0000","last_undegraded":"2024-08-17T17:29:22.945500+0000","last_fullsized":"2024-08-17T17:29:22.945500+0000","mapping_epoch":283,"log_start":"287'1200","ondisk_log_start":"287'1200","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8659,"log_dups_size":0,"ondisk_log_size":8659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:25.034122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370081792,"num_objects":5647,"num_object_clones":0,"num_object_copies":16941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":496192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":441,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.494523+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:29:18.494523+0000","last_peered":"2024-08-17T17:29:18.494523+0000","last_clean":"2024-08-17T17:29:18.494523+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:29:18.494523+0000","last_undegraded":"2024-08-17T17:29:18.494523+0000","last_fullsized":"2024-08-17T17:29:18.494523+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368576+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:29:11.368576+0000","last_peered":"2024-08-17T17:29:11.368576+0000","last_clean":"2024-08-17T17:29:11.368576+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:29:11.368576+0000","last_undegraded":"2024-08-17T17:29:11.368576+0000","last_fullsized":"2024-08-17T17:29:11.368576+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"301'9865","reported_seq":9913,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:23.848298+0000","last_change":"2024-08-17T17:27:05.818538+0000","last_active":"2024-08-17T17:29:23.848298+0000","last_peered":"2024-08-17T17:29:23.848298+0000","last_clean":"2024-08-17T17:29:23.848298+0000","last_became_active":"2024-08-17T17:27:05.818278+0000","last_became_peered":"2024-08-17T17:27:05.818278+0000","last_unstale":"2024-08-17T17:29:23.848298+0000","last_undegraded":"2024-08-17T17:29:23.848298+0000","last_fullsized":"2024-08-17T17:29:23.848298+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":9865,"log_dups_size":0,"ondisk_log_size":9865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:06.057185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361955328,"num_objects":5523,"num_object_clones":0,"num_object_copies":16569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9865,"num_write_kb":492416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":256,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377531+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:29:11.377531+0000","last_peered":"2024-08-17T17:29:11.377531+0000","last_clean":"2024-08-17T17:29:11.377531+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:29:11.377531+0000","last_undegraded":"2024-08-17T17:29:11.377531+0000","last_fullsized":"2024-08-17T17:29:11.377531+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370190+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:29:11.370190+0000","last_peered":"2024-08-17T17:29:11.370190+0000","last_clean":"2024-08-17T17:29:11.370190+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:29:11.370190+0000","last_undegraded":"2024-08-17T17:29:11.370190+0000","last_fullsized":"2024-08-17T17:29:11.370190+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"301'9843","reported_seq":9869,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:22.970561+0000","last_change":"2024-08-17T17:27:05.818745+0000","last_active":"2024-08-17T17:29:22.970561+0000","last_peered":"2024-08-17T17:29:22.970561+0000","last_clean":"2024-08-17T17:29:22.970561+0000","last_became_active":"2024-08-17T17:27:05.818533+0000","last_became_peered":"2024-08-17T17:27:05.818533+0000","last_unstale":"2024-08-17T17:29:22.970561+0000","last_undegraded":"2024-08-17T17:29:22.970561+0000","last_fullsized":"2024-08-17T17:29:22.970561+0000","mapping_epoch":283,"log_start":"287'1200","ondisk_log_start":"287'1200","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8643,"log_dups_size":0,"ondisk_log_size":8643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:41.413190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366936064,"num_objects":5599,"num_object_clones":0,"num_object_copies":16797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9843,"num_write_kb":494144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":273,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368255+0000","last_change":"2024-08-17T17:26:32.216646+0000","last_active":"2024-08-17T17:29:11.368255+0000","last_peered":"2024-08-17T17:29:11.368255+0000","last_clean":"2024-08-17T17:29:11.368255+0000","last_became_active":"2024-08-17T17:26:32.216155+0000","last_became_peered":"2024-08-17T17:26:32.216155+0000","last_unstale":"2024-08-17T17:29:11.368255+0000","last_undegraded":"2024-08-17T17:29:11.368255+0000","last_fullsized":"2024-08-17T17:29:11.368255+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:55.449331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368713+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:29:11.368713+0000","last_peered":"2024-08-17T17:29:11.368713+0000","last_clean":"2024-08-17T17:29:11.368713+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:29:11.368713+0000","last_undegraded":"2024-08-17T17:29:11.368713+0000","last_fullsized":"2024-08-17T17:29:11.368713+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"301'9781","reported_seq":9829,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.653973+0000","last_change":"2024-08-17T17:27:05.297991+0000","last_active":"2024-08-17T17:29:24.653973+0000","last_peered":"2024-08-17T17:29:24.653973+0000","last_clean":"2024-08-17T17:29:24.653973+0000","last_became_active":"2024-08-17T17:27:05.297666+0000","last_became_peered":"2024-08-17T17:27:05.297666+0000","last_unstale":"2024-08-17T17:29:24.653973+0000","last_undegraded":"2024-08-17T17:29:24.653973+0000","last_fullsized":"2024-08-17T17:29:24.653973+0000","mapping_epoch":283,"log_start":"287'1800","ondisk_log_start":"287'1800","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7981,"log_dups_size":0,"ondisk_log_size":7981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:35.344484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367722496,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9781,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":403,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491711+0000","last_change":"2024-08-17T17:26:32.212135+0000","last_active":"2024-08-17T17:29:18.491711+0000","last_peered":"2024-08-17T17:29:18.491711+0000","last_clean":"2024-08-17T17:29:18.491711+0000","last_became_active":"2024-08-17T17:26:32.211604+0000","last_became_peered":"2024-08-17T17:26:32.211604+0000","last_unstale":"2024-08-17T17:29:18.491711+0000","last_undegraded":"2024-08-17T17:29:18.491711+0000","last_fullsized":"2024-08-17T17:29:18.491711+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:49.886809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":347,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491344+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:29:18.491344+0000","last_peered":"2024-08-17T17:29:18.491344+0000","last_clean":"2024-08-17T17:29:18.491344+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:29:18.491344+0000","last_undegraded":"2024-08-17T17:29:18.491344+0000","last_fullsized":"2024-08-17T17:29:18.491344+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"301'9851","reported_seq":9899,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.660831+0000","last_change":"2024-08-17T17:27:05.297901+0000","last_active":"2024-08-17T17:29:24.660831+0000","last_peered":"2024-08-17T17:29:24.660831+0000","last_clean":"2024-08-17T17:29:24.660831+0000","last_became_active":"2024-08-17T17:27:05.297625+0000","last_became_peered":"2024-08-17T17:27:05.297625+0000","last_unstale":"2024-08-17T17:29:24.660831+0000","last_undegraded":"2024-08-17T17:29:24.660831+0000","last_fullsized":"2024-08-17T17:29:24.660831+0000","mapping_epoch":283,"log_start":"287'1900","ondisk_log_start":"287'1900","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:27:29.568335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361168896,"num_objects":5511,"num_object_clones":0,"num_object_copies":16533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9851,"num_write_kb":491584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":623,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370386+0000","last_change":"2024-08-17T17:26:33.204740+0000","last_active":"2024-08-17T17:29:11.370386+0000","last_peered":"2024-08-17T17:29:11.370386+0000","last_clean":"2024-08-17T17:29:11.370386+0000","last_became_active":"2024-08-17T17:26:33.204340+0000","last_became_peered":"2024-08-17T17:26:33.204340+0000","last_unstale":"2024-08-17T17:29:11.370386+0000","last_undegraded":"2024-08-17T17:29:11.370386+0000","last_fullsized":"2024-08-17T17:29:11.370386+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:43.802253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":208,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377261+0000","last_change":"2024-08-17T17:26:32.219571+0000","last_active":"2024-08-17T17:29:11.377261+0000","last_peered":"2024-08-17T17:29:11.377261+0000","last_clean":"2024-08-17T17:29:11.377261+0000","last_became_active":"2024-08-17T17:26:32.219094+0000","last_became_peered":"2024-08-17T17:26:32.219094+0000","last_unstale":"2024-08-17T17:29:11.377261+0000","last_undegraded":"2024-08-17T17:29:11.377261+0000","last_fullsized":"2024-08-17T17:29:11.377261+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:09.450772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":390,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.501862+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:29:18.501862+0000","last_peered":"2024-08-17T17:29:18.501862+0000","last_clean":"2024-08-17T17:29:18.501862+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:29:18.501862+0000","last_undegraded":"2024-08-17T17:29:18.501862+0000","last_fullsized":"2024-08-17T17:29:18.501862+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.379670+0000","last_change":"2024-08-17T17:26:32.215221+0000","last_active":"2024-08-17T17:29:11.379670+0000","last_peered":"2024-08-17T17:29:11.379670+0000","last_clean":"2024-08-17T17:29:11.379670+0000","last_became_active":"2024-08-17T17:26:32.214756+0000","last_became_peered":"2024-08-17T17:26:32.214756+0000","last_unstale":"2024-08-17T17:29:11.379670+0000","last_undegraded":"2024-08-17T17:29:11.379670+0000","last_fullsized":"2024-08-17T17:29:11.379670+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.545194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"301'10065","reported_seq":10113,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.691814+0000","last_change":"2024-08-17T17:27:05.819104+0000","last_active":"2024-08-17T17:29:24.691814+0000","last_peered":"2024-08-17T17:29:24.691814+0000","last_clean":"2024-08-17T17:29:24.691814+0000","last_became_active":"2024-08-17T17:27:05.818813+0000","last_became_peered":"2024-08-17T17:27:05.818813+0000","last_unstale":"2024-08-17T17:29:24.691814+0000","last_undegraded":"2024-08-17T17:29:24.691814+0000","last_fullsized":"2024-08-17T17:29:24.691814+0000","mapping_epoch":283,"log_start":"287'2100","ondisk_log_start":"287'2100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:28.659641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369426432,"num_objects":5637,"num_object_clones":0,"num_object_copies":16911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10065,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":297,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370147+0000","last_change":"2024-08-17T17:26:32.212907+0000","last_active":"2024-08-17T17:29:11.370147+0000","last_peered":"2024-08-17T17:29:11.370147+0000","last_clean":"2024-08-17T17:29:11.370147+0000","last_became_active":"2024-08-17T17:26:32.212549+0000","last_became_peered":"2024-08-17T17:26:32.212549+0000","last_unstale":"2024-08-17T17:29:11.370147+0000","last_undegraded":"2024-08-17T17:29:11.370147+0000","last_fullsized":"2024-08-17T17:29:11.370147+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:35.071333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":237,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.367945+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:29:11.367945+0000","last_peered":"2024-08-17T17:29:11.367945+0000","last_clean":"2024-08-17T17:29:11.367945+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:29:11.367945+0000","last_undegraded":"2024-08-17T17:29:11.367945+0000","last_fullsized":"2024-08-17T17:29:11.367945+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.499550+0000","last_change":"2024-08-17T17:24:37.073148+0000","last_active":"2024-08-17T17:29:18.499550+0000","last_peered":"2024-08-17T17:29:18.499550+0000","last_clean":"2024-08-17T17:29:18.499550+0000","last_became_active":"2024-08-17T17:24:37.072372+0000","last_became_peered":"2024-08-17T17:24:37.072372+0000","last_unstale":"2024-08-17T17:29:18.499550+0000","last_undegraded":"2024-08-17T17:29:18.499550+0000","last_fullsized":"2024-08-17T17:29:18.499550+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.659683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"301'9772","reported_seq":9820,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.683456+0000","last_change":"2024-08-17T17:27:05.298546+0000","last_active":"2024-08-17T17:29:24.683456+0000","last_peered":"2024-08-17T17:29:24.683456+0000","last_clean":"2024-08-17T17:29:24.683456+0000","last_became_active":"2024-08-17T17:27:05.297541+0000","last_became_peered":"2024-08-17T17:27:05.297541+0000","last_unstale":"2024-08-17T17:29:24.683456+0000","last_undegraded":"2024-08-17T17:29:24.683456+0000","last_fullsized":"2024-08-17T17:29:24.683456+0000","mapping_epoch":283,"log_start":"287'1800","ondisk_log_start":"287'1800","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7972,"log_dups_size":0,"ondisk_log_size":7972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:41.264251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364773376,"num_objects":5566,"num_object_clones":0,"num_object_copies":16698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9772,"num_write_kb":490816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":465,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.481019+0000","last_change":"2024-08-17T17:24:36.969933+0000","last_active":"2024-08-17T17:29:18.481019+0000","last_peered":"2024-08-17T17:29:18.481019+0000","last_clean":"2024-08-17T17:29:18.481019+0000","last_became_active":"2024-08-17T17:24:36.969657+0000","last_became_peered":"2024-08-17T17:24:36.969657+0000","last_unstale":"2024-08-17T17:29:18.481019+0000","last_undegraded":"2024-08-17T17:29:18.481019+0000","last_fullsized":"2024-08-17T17:29:18.481019+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:10.367450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":117,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480858+0000","last_change":"2024-08-17T17:26:50.368470+0000","last_active":"2024-08-17T17:29:18.480858+0000","last_peered":"2024-08-17T17:29:18.480858+0000","last_clean":"2024-08-17T17:29:18.480858+0000","last_became_active":"2024-08-17T17:26:50.367844+0000","last_became_peered":"2024-08-17T17:26:50.367844+0000","last_unstale":"2024-08-17T17:29:18.480858+0000","last_undegraded":"2024-08-17T17:29:18.480858+0000","last_fullsized":"2024-08-17T17:29:18.480858+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:02.244611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":482,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479597+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:29:18.479597+0000","last_peered":"2024-08-17T17:29:18.479597+0000","last_clean":"2024-08-17T17:29:18.479597+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:29:18.479597+0000","last_undegraded":"2024-08-17T17:29:18.479597+0000","last_fullsized":"2024-08-17T17:29:18.479597+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"301'9957","reported_seq":10005,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.660010+0000","last_change":"2024-08-17T17:27:05.819139+0000","last_active":"2024-08-17T17:29:24.660010+0000","last_peered":"2024-08-17T17:29:24.660010+0000","last_clean":"2024-08-17T17:29:24.660010+0000","last_became_active":"2024-08-17T17:27:05.818873+0000","last_became_peered":"2024-08-17T17:27:05.818873+0000","last_unstale":"2024-08-17T17:29:24.660010+0000","last_undegraded":"2024-08-17T17:29:24.660010+0000","last_fullsized":"2024-08-17T17:29:24.660010+0000","mapping_epoch":283,"log_start":"287'2000","ondisk_log_start":"287'2000","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7957,"log_dups_size":0,"ondisk_log_size":7957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:16.338176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370475008,"num_objects":5653,"num_object_clones":0,"num_object_copies":16959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9957,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.477661+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:29:18.477661+0000","last_peered":"2024-08-17T17:29:18.477661+0000","last_clean":"2024-08-17T17:29:18.477661+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:29:18.477661+0000","last_undegraded":"2024-08-17T17:29:18.477661+0000","last_fullsized":"2024-08-17T17:29:18.477661+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":276,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368443+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:29:11.368443+0000","last_peered":"2024-08-17T17:29:11.368443+0000","last_clean":"2024-08-17T17:29:11.368443+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:29:11.368443+0000","last_undegraded":"2024-08-17T17:29:11.368443+0000","last_fullsized":"2024-08-17T17:29:11.368443+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":182,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483405+0000","last_change":"2024-08-17T17:26:50.310453+0000","last_active":"2024-08-17T17:29:18.483405+0000","last_peered":"2024-08-17T17:29:18.483405+0000","last_clean":"2024-08-17T17:29:18.483405+0000","last_became_active":"2024-08-17T17:26:50.310180+0000","last_became_peered":"2024-08-17T17:26:50.310180+0000","last_unstale":"2024-08-17T17:29:18.483405+0000","last_undegraded":"2024-08-17T17:29:18.483405+0000","last_fullsized":"2024-08-17T17:29:18.483405+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:38.334936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"301'9621","reported_seq":9670,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:23.805305+0000","last_change":"2024-08-17T17:27:05.299496+0000","last_active":"2024-08-17T17:29:23.805305+0000","last_peered":"2024-08-17T17:29:23.805305+0000","last_clean":"2024-08-17T17:29:23.805305+0000","last_became_active":"2024-08-17T17:27:05.299195+0000","last_became_peered":"2024-08-17T17:27:05.299195+0000","last_unstale":"2024-08-17T17:29:23.805305+0000","last_undegraded":"2024-08-17T17:29:23.805305+0000","last_fullsized":"2024-08-17T17:29:23.805305+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":9621,"log_dups_size":0,"ondisk_log_size":9621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:06.961502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496600,"num_objects":5517,"num_object_clones":0,"num_object_copies":16551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5517,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9621,"num_write_kb":484353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":205,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377192+0000","last_change":"2024-08-17T17:25:46.633494+0000","last_active":"2024-08-17T17:29:11.377192+0000","last_peered":"2024-08-17T17:29:11.377192+0000","last_clean":"2024-08-17T17:29:11.377192+0000","last_became_active":"2024-08-17T17:25:46.633216+0000","last_became_peered":"2024-08-17T17:25:46.633216+0000","last_unstale":"2024-08-17T17:29:11.377192+0000","last_undegraded":"2024-08-17T17:29:11.377192+0000","last_fullsized":"2024-08-17T17:29:11.377192+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:14.957437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":462,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370022+0000","last_change":"2024-08-17T17:26:50.338185+0000","last_active":"2024-08-17T17:29:11.370022+0000","last_peered":"2024-08-17T17:29:11.370022+0000","last_clean":"2024-08-17T17:29:11.370022+0000","last_became_active":"2024-08-17T17:26:50.337992+0000","last_became_peered":"2024-08-17T17:26:50.337992+0000","last_unstale":"2024-08-17T17:29:11.370022+0000","last_undegraded":"2024-08-17T17:29:11.370022+0000","last_fullsized":"2024-08-17T17:29:11.370022+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:26.720733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":433,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.477875+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:29:18.477875+0000","last_peered":"2024-08-17T17:29:18.477875+0000","last_clean":"2024-08-17T17:29:18.477875+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:29:18.477875+0000","last_undegraded":"2024-08-17T17:29:18.477875+0000","last_fullsized":"2024-08-17T17:29:18.477875+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"301'9682","reported_seq":9729,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:21.229702+0000","last_change":"2024-08-17T17:27:05.296016+0000","last_active":"2024-08-17T17:29:21.229702+0000","last_peered":"2024-08-17T17:29:21.229702+0000","last_clean":"2024-08-17T17:29:21.229702+0000","last_became_active":"2024-08-17T17:27:05.295582+0000","last_became_peered":"2024-08-17T17:27:05.295582+0000","last_unstale":"2024-08-17T17:29:21.229702+0000","last_undegraded":"2024-08-17T17:29:21.229702+0000","last_fullsized":"2024-08-17T17:29:21.229702+0000","mapping_epoch":283,"log_start":"287'2100","ondisk_log_start":"287'2100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:39:20.962510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372375552,"num_objects":5682,"num_object_clones":0,"num_object_copies":17046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9682,"num_write_kb":491648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":291,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483235+0000","last_change":"2024-08-17T17:26:50.315231+0000","last_active":"2024-08-17T17:29:18.483235+0000","last_peered":"2024-08-17T17:29:18.483235+0000","last_clean":"2024-08-17T17:29:18.483235+0000","last_became_active":"2024-08-17T17:26:50.314945+0000","last_became_peered":"2024-08-17T17:26:50.314945+0000","last_unstale":"2024-08-17T17:29:18.483235+0000","last_undegraded":"2024-08-17T17:29:18.483235+0000","last_fullsized":"2024-08-17T17:29:18.483235+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:13.740267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":280,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370206+0000","last_change":"2024-08-17T17:26:32.215417+0000","last_active":"2024-08-17T17:29:11.370206+0000","last_peered":"2024-08-17T17:29:11.370206+0000","last_clean":"2024-08-17T17:29:11.370206+0000","last_became_active":"2024-08-17T17:26:32.215160+0000","last_became_peered":"2024-08-17T17:26:32.215160+0000","last_unstale":"2024-08-17T17:29:11.370206+0000","last_undegraded":"2024-08-17T17:29:11.370206+0000","last_fullsized":"2024-08-17T17:29:11.370206+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:17.370439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":343,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377187+0000","last_change":"2024-08-17T17:26:32.221201+0000","last_active":"2024-08-17T17:29:11.377187+0000","last_peered":"2024-08-17T17:29:11.377187+0000","last_clean":"2024-08-17T17:29:11.377187+0000","last_became_active":"2024-08-17T17:26:32.220913+0000","last_became_peered":"2024-08-17T17:26:32.220913+0000","last_unstale":"2024-08-17T17:29:11.377187+0000","last_undegraded":"2024-08-17T17:29:11.377187+0000","last_fullsized":"2024-08-17T17:29:11.377187+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:20.201903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"301'9968","reported_seq":9994,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:25.272935+0000","last_change":"2024-08-17T17:27:05.300239+0000","last_active":"2024-08-17T17:29:25.272935+0000","last_peered":"2024-08-17T17:29:25.272935+0000","last_clean":"2024-08-17T17:29:25.272935+0000","last_became_active":"2024-08-17T17:27:05.296428+0000","last_became_peered":"2024-08-17T17:27:05.296428+0000","last_unstale":"2024-08-17T17:29:25.272935+0000","last_undegraded":"2024-08-17T17:29:25.272935+0000","last_fullsized":"2024-08-17T17:29:25.272935+0000","mapping_epoch":283,"log_start":"287'1600","ondisk_log_start":"287'1600","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8368,"log_dups_size":0,"ondisk_log_size":8368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:37:11.949968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361365504,"num_objects":5514,"num_object_clones":0,"num_object_copies":16542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9968,"num_write_kb":495424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":452,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491418+0000","last_change":"2024-08-17T17:26:32.209014+0000","last_active":"2024-08-17T17:29:18.491418+0000","last_peered":"2024-08-17T17:29:18.491418+0000","last_clean":"2024-08-17T17:29:18.491418+0000","last_became_active":"2024-08-17T17:26:32.208539+0000","last_became_peered":"2024-08-17T17:26:32.208539+0000","last_unstale":"2024-08-17T17:29:18.491418+0000","last_undegraded":"2024-08-17T17:29:18.491418+0000","last_fullsized":"2024-08-17T17:29:18.491418+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:40:15.666378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":431,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.478215+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:29:18.478215+0000","last_peered":"2024-08-17T17:29:18.478215+0000","last_clean":"2024-08-17T17:29:18.478215+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:29:18.478215+0000","last_undegraded":"2024-08-17T17:29:18.478215+0000","last_fullsized":"2024-08-17T17:29:18.478215+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":442,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491423+0000","last_change":"2024-08-17T17:26:32.211876+0000","last_active":"2024-08-17T17:29:18.491423+0000","last_peered":"2024-08-17T17:29:18.491423+0000","last_clean":"2024-08-17T17:29:18.491423+0000","last_became_active":"2024-08-17T17:26:32.211212+0000","last_became_peered":"2024-08-17T17:26:32.211212+0000","last_unstale":"2024-08-17T17:29:18.491423+0000","last_undegraded":"2024-08-17T17:29:18.491423+0000","last_fullsized":"2024-08-17T17:29:18.491423+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:50.032892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"301'9626","reported_seq":9652,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:22.600120+0000","last_change":"2024-08-17T17:27:05.291199+0000","last_active":"2024-08-17T17:29:22.600120+0000","last_peered":"2024-08-17T17:29:22.600120+0000","last_clean":"2024-08-17T17:29:22.600120+0000","last_became_active":"2024-08-17T17:27:05.290910+0000","last_became_peered":"2024-08-17T17:27:05.290910+0000","last_unstale":"2024-08-17T17:29:22.600120+0000","last_undegraded":"2024-08-17T17:29:22.600120+0000","last_fullsized":"2024-08-17T17:29:22.600120+0000","mapping_epoch":283,"log_start":"287'1500","ondisk_log_start":"287'1500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:32.168692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9626,"num_write_kb":484544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":122,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.499451+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:29:18.499451+0000","last_peered":"2024-08-17T17:29:18.499451+0000","last_clean":"2024-08-17T17:29:18.499451+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:29:18.499451+0000","last_undegraded":"2024-08-17T17:29:18.499451+0000","last_fullsized":"2024-08-17T17:29:18.499451+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":285,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377802+0000","last_change":"2024-08-17T17:26:32.218111+0000","last_active":"2024-08-17T17:29:11.377802+0000","last_peered":"2024-08-17T17:29:11.377802+0000","last_clean":"2024-08-17T17:29:11.377802+0000","last_became_active":"2024-08-17T17:26:32.217705+0000","last_became_peered":"2024-08-17T17:26:32.217705+0000","last_unstale":"2024-08-17T17:29:11.377802+0000","last_undegraded":"2024-08-17T17:29:11.377802+0000","last_fullsized":"2024-08-17T17:29:11.377802+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:50.751085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368179+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:29:11.368179+0000","last_peered":"2024-08-17T17:29:11.368179+0000","last_clean":"2024-08-17T17:29:11.368179+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:29:11.368179+0000","last_undegraded":"2024-08-17T17:29:11.368179+0000","last_fullsized":"2024-08-17T17:29:11.368179+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368310+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:29:11.368310+0000","last_peered":"2024-08-17T17:29:11.368310+0000","last_clean":"2024-08-17T17:29:11.368310+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:29:11.368310+0000","last_undegraded":"2024-08-17T17:29:11.368310+0000","last_fullsized":"2024-08-17T17:29:11.368310+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368719+0000","last_change":"2024-08-17T17:26:32.215288+0000","last_active":"2024-08-17T17:29:11.368719+0000","last_peered":"2024-08-17T17:29:11.368719+0000","last_clean":"2024-08-17T17:29:11.368719+0000","last_became_active":"2024-08-17T17:26:32.214754+0000","last_became_peered":"2024-08-17T17:26:32.214754+0000","last_unstale":"2024-08-17T17:29:11.368719+0000","last_undegraded":"2024-08-17T17:29:11.368719+0000","last_fullsized":"2024-08-17T17:29:11.368719+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.684058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":432,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.477749+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:29:18.477749+0000","last_peered":"2024-08-17T17:29:18.477749+0000","last_clean":"2024-08-17T17:29:18.477749+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:29:18.477749+0000","last_undegraded":"2024-08-17T17:29:18.477749+0000","last_fullsized":"2024-08-17T17:29:18.477749+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":232,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370190+0000","last_change":"2024-08-17T17:26:32.207981+0000","last_active":"2024-08-17T17:29:11.370190+0000","last_peered":"2024-08-17T17:29:11.370190+0000","last_clean":"2024-08-17T17:29:11.370190+0000","last_became_active":"2024-08-17T17:26:32.207636+0000","last_became_peered":"2024-08-17T17:26:32.207636+0000","last_unstale":"2024-08-17T17:29:11.370190+0000","last_undegraded":"2024-08-17T17:29:11.370190+0000","last_fullsized":"2024-08-17T17:29:11.370190+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:32.412636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":195,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377395+0000","last_change":"2024-08-17T17:26:32.210741+0000","last_active":"2024-08-17T17:29:11.377395+0000","last_peered":"2024-08-17T17:29:11.377395+0000","last_clean":"2024-08-17T17:29:11.377395+0000","last_became_active":"2024-08-17T17:26:32.210417+0000","last_became_peered":"2024-08-17T17:26:32.210417+0000","last_unstale":"2024-08-17T17:29:11.377395+0000","last_undegraded":"2024-08-17T17:29:11.377395+0000","last_fullsized":"2024-08-17T17:29:11.377395+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:41.290842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":390,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.495108+0000","last_change":"2024-08-17T17:26:32.211602+0000","last_active":"2024-08-17T17:29:18.495108+0000","last_peered":"2024-08-17T17:29:18.495108+0000","last_clean":"2024-08-17T17:29:18.495108+0000","last_became_active":"2024-08-17T17:26:32.211288+0000","last_became_peered":"2024-08-17T17:26:32.211288+0000","last_unstale":"2024-08-17T17:29:18.495108+0000","last_undegraded":"2024-08-17T17:29:18.495108+0000","last_fullsized":"2024-08-17T17:29:18.495108+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:49.979367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":349,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377621+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:29:11.377621+0000","last_peered":"2024-08-17T17:29:11.377621+0000","last_clean":"2024-08-17T17:29:11.377621+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:29:11.377621+0000","last_undegraded":"2024-08-17T17:29:11.377621+0000","last_fullsized":"2024-08-17T17:29:11.377621+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":391,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491132+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:29:18.491132+0000","last_peered":"2024-08-17T17:29:18.491132+0000","last_clean":"2024-08-17T17:29:18.491132+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:29:18.491132+0000","last_undegraded":"2024-08-17T17:29:18.491132+0000","last_fullsized":"2024-08-17T17:29:18.491132+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.377799+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:29:11.377799+0000","last_peered":"2024-08-17T17:29:11.377799+0000","last_clean":"2024-08-17T17:29:11.377799+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:29:11.377799+0000","last_undegraded":"2024-08-17T17:29:11.377799+0000","last_fullsized":"2024-08-17T17:29:11.377799+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":422,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491514+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:29:18.491514+0000","last_peered":"2024-08-17T17:29:18.491514+0000","last_clean":"2024-08-17T17:29:18.491514+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:29:18.491514+0000","last_undegraded":"2024-08-17T17:29:18.491514+0000","last_fullsized":"2024-08-17T17:29:18.491514+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":229,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.370324+0000","last_change":"2024-08-17T17:26:32.215823+0000","last_active":"2024-08-17T17:29:11.370324+0000","last_peered":"2024-08-17T17:29:11.370324+0000","last_clean":"2024-08-17T17:29:11.370324+0000","last_became_active":"2024-08-17T17:26:32.214110+0000","last_became_peered":"2024-08-17T17:26:32.214110+0000","last_unstale":"2024-08-17T17:29:11.370324+0000","last_undegraded":"2024-08-17T17:29:11.370324+0000","last_fullsized":"2024-08-17T17:29:11.370324+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.088172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":430,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.477589+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:29:18.477589+0000","last_peered":"2024-08-17T17:29:18.477589+0000","last_clean":"2024-08-17T17:29:18.477589+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:29:18.477589+0000","last_undegraded":"2024-08-17T17:29:18.477589+0000","last_fullsized":"2024-08-17T17:29:18.477589+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":390,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.490818+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:29:18.490818+0000","last_peered":"2024-08-17T17:29:18.490818+0000","last_clean":"2024-08-17T17:29:18.490818+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:29:18.490818+0000","last_undegraded":"2024-08-17T17:29:18.490818+0000","last_fullsized":"2024-08-17T17:29:18.490818+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":274,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368363+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:29:11.368363+0000","last_peered":"2024-08-17T17:29:11.368363+0000","last_clean":"2024-08-17T17:29:11.368363+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:29:11.368363+0000","last_undegraded":"2024-08-17T17:29:11.368363+0000","last_fullsized":"2024-08-17T17:29:11.368363+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":454,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483330+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:29:18.483330+0000","last_peered":"2024-08-17T17:29:18.483330+0000","last_clean":"2024-08-17T17:29:18.483330+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:29:18.483330+0000","last_undegraded":"2024-08-17T17:29:18.483330+0000","last_fullsized":"2024-08-17T17:29:18.483330+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":128,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.494214+0000","last_change":"2024-08-17T17:26:32.214552+0000","last_active":"2024-08-17T17:29:18.494214+0000","last_peered":"2024-08-17T17:29:18.494214+0000","last_clean":"2024-08-17T17:29:18.494214+0000","last_became_active":"2024-08-17T17:26:32.214338+0000","last_became_peered":"2024-08-17T17:26:32.214338+0000","last_unstale":"2024-08-17T17:29:18.494214+0000","last_undegraded":"2024-08-17T17:29:18.494214+0000","last_fullsized":"2024-08-17T17:29:18.494214+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:21.212607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":282,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368230+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:29:11.368230+0000","last_peered":"2024-08-17T17:29:11.368230+0000","last_clean":"2024-08-17T17:29:11.368230+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:29:11.368230+0000","last_undegraded":"2024-08-17T17:29:11.368230+0000","last_fullsized":"2024-08-17T17:29:11.368230+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":155,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368661+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:29:11.368661+0000","last_peered":"2024-08-17T17:29:11.368661+0000","last_clean":"2024-08-17T17:29:11.368661+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:29:11.368661+0000","last_undegraded":"2024-08-17T17:29:11.368661+0000","last_fullsized":"2024-08-17T17:29:11.368661+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":292,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.368009+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:29:11.368009+0000","last_peered":"2024-08-17T17:29:11.368009+0000","last_clean":"2024-08-17T17:29:11.368009+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:29:11.368009+0000","last_undegraded":"2024-08-17T17:29:11.368009+0000","last_fullsized":"2024-08-17T17:29:11.368009+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":155,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.369976+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:29:11.369976+0000","last_peered":"2024-08-17T17:29:11.369976+0000","last_clean":"2024-08-17T17:29:11.369976+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:29:11.369976+0000","last_undegraded":"2024-08-17T17:29:11.369976+0000","last_fullsized":"2024-08-17T17:29:11.369976+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":435,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.495510+0000","last_change":"2024-08-17T17:24:36.972596+0000","last_active":"2024-08-17T17:29:18.495510+0000","last_peered":"2024-08-17T17:29:18.495510+0000","last_clean":"2024-08-17T17:29:18.495510+0000","last_became_active":"2024-08-17T17:24:36.971541+0000","last_became_peered":"2024-08-17T17:24:36.971541+0000","last_unstale":"2024-08-17T17:29:18.495510+0000","last_undegraded":"2024-08-17T17:29:18.495510+0000","last_fullsized":"2024-08-17T17:29:18.495510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:38.457003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":408,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.491172+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:29:18.491172+0000","last_peered":"2024-08-17T17:29:18.491172+0000","last_clean":"2024-08-17T17:29:18.491172+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:29:18.491172+0000","last_undegraded":"2024-08-17T17:29:18.491172+0000","last_fullsized":"2024-08-17T17:29:18.491172+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-17T17:29:11.367922+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:29:11.367922+0000","last_peered":"2024-08-17T17:29:11.367922+0000","last_clean":"2024-08-17T17:29:11.367922+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:29:11.367922+0000","last_undegraded":"2024-08-17T17:29:11.367922+0000","last_fullsized":"2024-08-17T17:29:11.367922+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":124,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.493184+0000","last_change":"2024-08-17T17:24:37.032048+0000","last_active":"2024-08-17T17:29:18.493184+0000","last_peered":"2024-08-17T17:29:18.493184+0000","last_clean":"2024-08-17T17:29:18.493184+0000","last_became_active":"2024-08-17T17:24:36.972161+0000","last_became_peered":"2024-08-17T17:24:36.972161+0000","last_unstale":"2024-08-17T17:29:18.493184+0000","last_undegraded":"2024-08-17T17:29:18.493184+0000","last_fullsized":"2024-08-17T17:29:18.493184+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:55.363820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5834932248,"num_objects":89035,"num_object_clones":0,"num_object_copies":267105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89035,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157799,"num_write_kb":7898625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17619628032,"data_stored":17619615816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137399,"ondisk_log_size":137399,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1162,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":270,"seq":1159641169956,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2498332,"kb_used_data":2115540,"kb_used_omap":12,"kb_used_meta":382707,"kb_avail":91246820,"statfs":{"total":95995035648,"available":93436743680,"internally_reserved":0,"allocated":2166312960,"data_stored":2164753703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12410,"internal_metadata":391892870},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463325,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3800280,"kb_used_data":2902416,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":89944872,"statfs":{"total":95995035648,"available":92103548928,"internally_reserved":0,"allocated":2972073984,"data_stored":2970518704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492000,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1811840,"kb_used_data":1420664,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":91933312,"statfs":{"total":95995035648,"available":94139711488,"internally_reserved":0,"allocated":1454759936,"data_stored":1453223440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986599,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2348180,"kb_used_data":1447244,"kb_used_omap":1,"kb_used_meta":900926,"kb_avail":91396972,"statfs":{"total":95995035648,"available":93590499328,"internally_reserved":0,"allocated":1481977856,"data_stored":1480425928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":922548568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225419,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3214012,"kb_used_data":2498548,"kb_used_omap":4,"kb_used_meta":715451,"kb_avail":90531140,"statfs":{"total":95995035648,"available":92703887360,"internally_reserved":0,"allocated":2558513152,"data_stored":2556980980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":732622272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075744,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2631368,"kb_used_data":2139072,"kb_used_omap":6,"kb_used_meta":492281,"kb_avail":91113784,"statfs":{"total":95995035648,"available":93300514816,"internally_reserved":0,"allocated":2190409728,"data_stored":2188866095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":504096187},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533636,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3793340,"kb_used_data":2938676,"kb_used_omap":5,"kb_used_meta":854586,"kb_avail":89951812,"statfs":{"total":95995035648,"available":92110655488,"internally_reserved":0,"allocated":3009204224,"data_stored":3007670850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":875096304},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089613,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2737992,"kb_used_data":1832896,"kb_used_omap":5,"kb_used_meta":905082,"kb_avail":91007160,"statfs":{"total":95995035648,"available":93191331840,"internally_reserved":0,"allocated":1876885504,"data_stored":1875342379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":926804027},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1858666496,"data_stored":1858666496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3001090048,"data_stored":3001090048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2183598080,"data_stored":2183594008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2553479168,"data_stored":2553479168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1467289600,"data_stored":1467285528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1444872192,"data_stored":1444872192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2949844992,"data_stored":2949840920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2160787456,"data_stored":2160787456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:29:28.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:29:28.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:28.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:28.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:28.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:28 smithi152 ceph-mon[93456]: from='client.56122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:28.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:28.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:28.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:28.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:28.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:28.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:28.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:29.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:29.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:29.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:29.163 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:29 smithi142 ceph-mon[103614]: pgmap v1008: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:29:29.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:29.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:29.346 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:29.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:29.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:29.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:29 smithi152 ceph-mon[93456]: pgmap v1008: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:29:29.510 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:29.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:29.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:29 smithi142 ceph-mon[106729]: pgmap v1008: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:29:29.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:29.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:29.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:29.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:29.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:30.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:30.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:30.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:30.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:30.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:30.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:30.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:30.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:29:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:29:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:30.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:30.603 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:30.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:30.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:30.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:30.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:30.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:31.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:31.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:31.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:31.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:31.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:31.317 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:31 smithi142 ceph-mon[106729]: pgmap v1009: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-08-17T17:29:31.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:31 smithi142 ceph-mon[103614]: pgmap v1009: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-08-17T17:29:31.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:31.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:31.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:31 smithi152 ceph-mon[93456]: pgmap v1009: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-08-17T17:29:31.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:31.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:31.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:31.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:31.755 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:29:31.755 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:31.792 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:29:31.792 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:31.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:31.828 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:29:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:29:31] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:29:31.828 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:29:31.828 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:31.860 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:29:31.860 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:31.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:31.889 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:29:31.889 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:31.921 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:29:31.921 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:31.955 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:29:31.955 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:31.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:32.000 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:29:32.000 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:32.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:32.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:32.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:32.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:32.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:32.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:32.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:32.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:32.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2737412431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:32.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:32.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2737412431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:32.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2737412431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:32.641 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:32.641 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:32.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:32.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:32.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:32.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:32.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:32.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:32.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:33.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:33.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:33.185 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:29:33.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:33.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.260 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:33.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:33.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:33.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:33.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:33.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:33 smithi152 ceph-mon[93456]: pgmap v1010: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:29:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:33 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3281078164' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:33 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:33.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:33 smithi142 ceph-mon[106729]: pgmap v1010: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:29:33.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:33 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3281078164' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:33.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:33 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:33.620 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:33 smithi142 ceph-mon[103614]: pgmap v1010: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-08-17T17:29:33.620 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:33 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3281078164' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:33.620 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:33 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:33.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:33.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:29:33.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:33.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:33.850 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:33.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:33.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:33.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:33.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:34.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.122 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:34.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:34.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:34.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:34.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:34 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:34.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:34 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:34.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:34 smithi142 ceph-mon[103614]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:29:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:34 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:34 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:34.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:34 smithi142 ceph-mon[106729]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:29:34.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:34.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:34.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:34.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:34 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:34 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:34.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:34 smithi152 ceph-mon[93456]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:29:34.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:34.658 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:34.760 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:34.806 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:34.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:34.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:35.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:35.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:35.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:35.190 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:35.228 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:35.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:35.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:35.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:35.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:35.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:35 smithi152 ceph-mon[93456]: pgmap v1012: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-17T17:29:35.603 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:35.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:35 smithi142 ceph-mon[103614]: pgmap v1012: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-17T17:29:35.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:35 smithi142 ceph-mon[106729]: pgmap v1012: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-17T17:29:35.644 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:35.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:35.771 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:35.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:35.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:35.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:36.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:36.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:36.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:36.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:36.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:36.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:36.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:36.432 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:36.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:36.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:36.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:29:36.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:36.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:36.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:36.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:36.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:37.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:37.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:37.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:37.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:37.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:37.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:37.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:37.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:37 smithi152 ceph-mon[93456]: pgmap v1013: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-08-17T17:29:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:37.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:37 smithi142 ceph-mon[106729]: pgmap v1013: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-08-17T17:29:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:37 smithi142 ceph-mon[103614]: pgmap v1013: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-08-17T17:29:37.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:37.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:37.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:37.883 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:37.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:37.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:38.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:38.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:38.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:38.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:38.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:38.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:38.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.478 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:38.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:38.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:38.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:38.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:38.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:38.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:38.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:38.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:39.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:39.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:39.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.164 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:39.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:39.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:39.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:39.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:39.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:39.508 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:39 smithi142 ceph-mon[103614]: pgmap v1014: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-08-17T17:29:39.508 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:39 smithi142 ceph-mon[106729]: pgmap v1014: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-08-17T17:29:39.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:39 smithi152 ceph-mon[93456]: pgmap v1014: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-08-17T17:29:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:39.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:39.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:39.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:39.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:39.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:39.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:39.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:40.049 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:40.050 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:40.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:40.141 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:40.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:40.243 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:40.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:40.324 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:29:40.326 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:29:40.390 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:40.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:40.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:40.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:40.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:29:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:29:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:40.607 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:40.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:40.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:40.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:40.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:40.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:40.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:40.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:41.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:41.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:41.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:41.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:41.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:41.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:41.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:41.398 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:41.446 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1015,"stamp":"2024-08-17T17:29:40.090932+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5097330661,"num_objects":77797,"num_object_clones":0,"num_object_copies":233391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77797,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":169341,"num_write_kb":7901072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2362634,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138170,"ondisk_log_size":138170,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20637368,"kb_used_data":15099504,"kb_used_omap":45,"kb_used_meta":5537298,"kb_avail":729323848,"statfs":{"total":767960285184,"available":746827620352,"internally_reserved":0,"allocated":15461892096,"data_stored":15449504511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47063,"internal_metadata":5670193193},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-527564800,"num_objects":-8050,"num_object_clones":0,"num_object_copies":-24150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8050,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002456"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":428,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192797+0000","last_change":"2024-08-17T17:26:32.212092+0000","last_active":"2024-08-17T17:29:33.192797+0000","last_peered":"2024-08-17T17:29:33.192797+0000","last_clean":"2024-08-17T17:29:33.192797+0000","last_became_active":"2024-08-17T17:26:32.211774+0000","last_became_peered":"2024-08-17T17:26:32.211774+0000","last_unstale":"2024-08-17T17:29:33.192797+0000","last_undegraded":"2024-08-17T17:29:33.192797+0000","last_fullsized":"2024-08-17T17:29:33.192797+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:45.518565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":247,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.475947+0000","last_change":"2024-08-17T17:26:32.210903+0000","last_active":"2024-08-17T17:29:18.475947+0000","last_peered":"2024-08-17T17:29:18.475947+0000","last_clean":"2024-08-17T17:29:18.475947+0000","last_became_active":"2024-08-17T17:26:32.209823+0000","last_became_peered":"2024-08-17T17:26:32.209823+0000","last_unstale":"2024-08-17T17:29:18.475947+0000","last_undegraded":"2024-08-17T17:29:18.475947+0000","last_fullsized":"2024-08-17T17:29:18.475947+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:54:17.717352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483518+0000","last_change":"2024-08-17T17:26:32.205305+0000","last_active":"2024-08-17T17:29:18.483518+0000","last_peered":"2024-08-17T17:29:18.483518+0000","last_clean":"2024-08-17T17:29:18.483518+0000","last_became_active":"2024-08-17T17:26:32.205000+0000","last_became_peered":"2024-08-17T17:26:32.205000+0000","last_unstale":"2024-08-17T17:29:18.483518+0000","last_undegraded":"2024-08-17T17:29:18.483518+0000","last_fullsized":"2024-08-17T17:29:18.483518+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:31.144881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":230,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474682+0000","last_change":"2024-08-17T17:26:32.213319+0000","last_active":"2024-08-17T17:29:18.474682+0000","last_peered":"2024-08-17T17:29:18.474682+0000","last_clean":"2024-08-17T17:29:18.474682+0000","last_became_active":"2024-08-17T17:26:32.213079+0000","last_became_peered":"2024-08-17T17:26:32.213079+0000","last_unstale":"2024-08-17T17:29:18.474682+0000","last_undegraded":"2024-08-17T17:29:18.474682+0000","last_fullsized":"2024-08-17T17:29:18.474682+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:43.552406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":310,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.194779+0000","last_change":"2024-08-17T17:26:32.211539+0000","last_active":"2024-08-17T17:29:33.194779+0000","last_peered":"2024-08-17T17:29:33.194779+0000","last_clean":"2024-08-17T17:29:33.194779+0000","last_became_active":"2024-08-17T17:26:32.211267+0000","last_became_peered":"2024-08-17T17:26:32.211267+0000","last_unstale":"2024-08-17T17:29:33.194779+0000","last_undegraded":"2024-08-17T17:29:33.194779+0000","last_fullsized":"2024-08-17T17:29:33.194779+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:16.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":216,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474826+0000","last_change":"2024-08-17T17:26:32.233887+0000","last_active":"2024-08-17T17:29:18.474826+0000","last_peered":"2024-08-17T17:29:18.474826+0000","last_clean":"2024-08-17T17:29:18.474826+0000","last_became_active":"2024-08-17T17:26:32.230472+0000","last_became_peered":"2024-08-17T17:26:32.230472+0000","last_unstale":"2024-08-17T17:29:18.474826+0000","last_undegraded":"2024-08-17T17:29:18.474826+0000","last_fullsized":"2024-08-17T17:29:18.474826+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:09.045150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":404,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.193078+0000","last_change":"2024-08-17T17:26:32.207139+0000","last_active":"2024-08-17T17:29:33.193078+0000","last_peered":"2024-08-17T17:29:33.193078+0000","last_clean":"2024-08-17T17:29:33.193078+0000","last_became_active":"2024-08-17T17:26:32.206763+0000","last_became_peered":"2024-08-17T17:26:32.206763+0000","last_unstale":"2024-08-17T17:29:33.193078+0000","last_undegraded":"2024-08-17T17:29:33.193078+0000","last_fullsized":"2024-08-17T17:29:33.193078+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:51.396647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":322,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479990+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:29:18.479990+0000","last_peered":"2024-08-17T17:29:18.479990+0000","last_clean":"2024-08-17T17:29:18.479990+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:29:18.479990+0000","last_undegraded":"2024-08-17T17:29:18.479990+0000","last_fullsized":"2024-08-17T17:29:18.479990+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"302'10873","reported_seq":10900,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:39.943170+0000","last_change":"2024-08-17T17:27:05.299771+0000","last_active":"2024-08-17T17:29:39.943170+0000","last_peered":"2024-08-17T17:29:39.943170+0000","last_clean":"2024-08-17T17:29:39.943170+0000","last_became_active":"2024-08-17T17:27:05.299171+0000","last_became_peered":"2024-08-17T17:27:05.299171+0000","last_unstale":"2024-08-17T17:29:39.943170+0000","last_undegraded":"2024-08-17T17:29:39.943170+0000","last_fullsized":"2024-08-17T17:29:39.943170+0000","mapping_epoch":283,"log_start":"287'800","ondisk_log_start":"287'800","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10073,"log_dups_size":0,"ondisk_log_size":10073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:25.524635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311623680,"num_objects":4755,"num_object_clones":0,"num_object_copies":14265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10873,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":278,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474840+0000","last_change":"2024-08-17T17:26:32.214168+0000","last_active":"2024-08-17T17:29:18.474840+0000","last_peered":"2024-08-17T17:29:18.474840+0000","last_clean":"2024-08-17T17:29:18.474840+0000","last_became_active":"2024-08-17T17:26:32.213844+0000","last_became_peered":"2024-08-17T17:26:32.213844+0000","last_unstale":"2024-08-17T17:29:18.474840+0000","last_undegraded":"2024-08-17T17:29:18.474840+0000","last_fullsized":"2024-08-17T17:29:18.474840+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:52.468566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":178,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483310+0000","last_change":"2024-08-17T17:26:32.218633+0000","last_active":"2024-08-17T17:29:18.483310+0000","last_peered":"2024-08-17T17:29:18.483310+0000","last_clean":"2024-08-17T17:29:18.483310+0000","last_became_active":"2024-08-17T17:26:32.218212+0000","last_became_peered":"2024-08-17T17:26:32.218212+0000","last_unstale":"2024-08-17T17:29:18.483310+0000","last_undegraded":"2024-08-17T17:29:18.483310+0000","last_fullsized":"2024-08-17T17:29:18.483310+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:55.971779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"302'10619","reported_seq":10669,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.862942+0000","last_change":"2024-08-17T17:27:05.818422+0000","last_active":"2024-08-17T17:29:38.862942+0000","last_peered":"2024-08-17T17:29:38.862942+0000","last_clean":"2024-08-17T17:29:38.862942+0000","last_became_active":"2024-08-17T17:27:05.818117+0000","last_became_peered":"2024-08-17T17:27:05.818117+0000","last_unstale":"2024-08-17T17:29:38.862942+0000","last_undegraded":"2024-08-17T17:29:38.862942+0000","last_fullsized":"2024-08-17T17:29:38.862942+0000","mapping_epoch":283,"log_start":"287'600","ondisk_log_start":"287'600","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10019,"log_dups_size":0,"ondisk_log_size":10019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:36.236107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316473344,"num_objects":4829,"num_object_clones":0,"num_object_copies":14487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10619,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":233,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474948+0000","last_change":"2024-08-17T17:26:32.220489+0000","last_active":"2024-08-17T17:29:18.474948+0000","last_peered":"2024-08-17T17:29:18.474948+0000","last_clean":"2024-08-17T17:29:18.474948+0000","last_became_active":"2024-08-17T17:26:32.220150+0000","last_became_peered":"2024-08-17T17:26:32.220150+0000","last_unstale":"2024-08-17T17:29:18.474948+0000","last_undegraded":"2024-08-17T17:29:18.474948+0000","last_fullsized":"2024-08-17T17:29:18.474948+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:09.954487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":273,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479457+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:29:18.479457+0000","last_peered":"2024-08-17T17:29:18.479457+0000","last_clean":"2024-08-17T17:29:18.479457+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:29:18.479457+0000","last_undegraded":"2024-08-17T17:29:18.479457+0000","last_fullsized":"2024-08-17T17:29:18.479457+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"302'10458","reported_seq":10485,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.293200+0000","last_change":"2024-08-17T17:27:05.297172+0000","last_active":"2024-08-17T17:29:35.293200+0000","last_peered":"2024-08-17T17:29:35.293200+0000","last_clean":"2024-08-17T17:29:35.293200+0000","last_became_active":"2024-08-17T17:27:05.296899+0000","last_became_peered":"2024-08-17T17:27:05.296899+0000","last_unstale":"2024-08-17T17:29:35.293200+0000","last_undegraded":"2024-08-17T17:29:35.293200+0000","last_fullsized":"2024-08-17T17:29:35.293200+0000","mapping_epoch":283,"log_start":"287'2100","ondisk_log_start":"287'2100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8358,"log_dups_size":0,"ondisk_log_size":8358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:32.207215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811072,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10458,"num_write_kb":494720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":243,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.475072+0000","last_change":"2024-08-17T17:26:32.216517+0000","last_active":"2024-08-17T17:29:18.475072+0000","last_peered":"2024-08-17T17:29:18.475072+0000","last_clean":"2024-08-17T17:29:18.475072+0000","last_became_active":"2024-08-17T17:26:32.216161+0000","last_became_peered":"2024-08-17T17:26:32.216161+0000","last_unstale":"2024-08-17T17:29:18.475072+0000","last_undegraded":"2024-08-17T17:29:18.475072+0000","last_fullsized":"2024-08-17T17:29:18.475072+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:39.712941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":273,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479797+0000","last_change":"2024-08-17T17:26:32.217706+0000","last_active":"2024-08-17T17:29:18.479797+0000","last_peered":"2024-08-17T17:29:18.479797+0000","last_clean":"2024-08-17T17:29:18.479797+0000","last_became_active":"2024-08-17T17:26:32.217107+0000","last_became_peered":"2024-08-17T17:26:32.217107+0000","last_unstale":"2024-08-17T17:29:18.479797+0000","last_undegraded":"2024-08-17T17:29:18.479797+0000","last_fullsized":"2024-08-17T17:29:18.479797+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:39.827920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"302'10414","reported_seq":10441,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.303139+0000","last_change":"2024-08-17T17:27:05.297778+0000","last_active":"2024-08-17T17:29:35.303139+0000","last_peered":"2024-08-17T17:29:35.303139+0000","last_clean":"2024-08-17T17:29:35.303139+0000","last_became_active":"2024-08-17T17:27:05.297367+0000","last_became_peered":"2024-08-17T17:27:05.297367+0000","last_unstale":"2024-08-17T17:29:35.303139+0000","last_undegraded":"2024-08-17T17:29:35.303139+0000","last_fullsized":"2024-08-17T17:29:35.303139+0000","mapping_epoch":283,"log_start":"287'2000","ondisk_log_start":"287'2000","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8414,"log_dups_size":0,"ondisk_log_size":8414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.484840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328859648,"num_objects":5018,"num_object_clones":0,"num_object_copies":15054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10414,"num_write_kb":493824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":354,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192981+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:29:33.192981+0000","last_peered":"2024-08-17T17:29:33.192981+0000","last_clean":"2024-08-17T17:29:33.192981+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:29:33.192981+0000","last_undegraded":"2024-08-17T17:29:33.192981+0000","last_fullsized":"2024-08-17T17:29:33.192981+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.202016+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:29:33.202016+0000","last_peered":"2024-08-17T17:29:33.202016+0000","last_clean":"2024-08-17T17:29:33.202016+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:29:33.202016+0000","last_undegraded":"2024-08-17T17:29:33.202016+0000","last_fullsized":"2024-08-17T17:29:33.202016+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"302'10648","reported_seq":10675,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.950272+0000","last_change":"2024-08-17T17:27:05.297382+0000","last_active":"2024-08-17T17:29:37.950272+0000","last_peered":"2024-08-17T17:29:37.950272+0000","last_clean":"2024-08-17T17:29:37.950272+0000","last_became_active":"2024-08-17T17:27:05.297149+0000","last_became_peered":"2024-08-17T17:27:05.297149+0000","last_unstale":"2024-08-17T17:29:37.950272+0000","last_undegraded":"2024-08-17T17:29:37.950272+0000","last_fullsized":"2024-08-17T17:29:37.950272+0000","mapping_epoch":283,"log_start":"287'2000","ondisk_log_start":"287'2000","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8648,"log_dups_size":0,"ondisk_log_size":8648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:25.034122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10648,"num_write_kb":496192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":443,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.195057+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:29:33.195057+0000","last_peered":"2024-08-17T17:29:33.195057+0000","last_clean":"2024-08-17T17:29:33.195057+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:29:33.195057+0000","last_undegraded":"2024-08-17T17:29:33.195057+0000","last_fullsized":"2024-08-17T17:29:33.195057+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.481826+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:29:18.481826+0000","last_peered":"2024-08-17T17:29:18.481826+0000","last_clean":"2024-08-17T17:29:18.481826+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:29:18.481826+0000","last_undegraded":"2024-08-17T17:29:18.481826+0000","last_fullsized":"2024-08-17T17:29:18.481826+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"302'10592","reported_seq":10640,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.865628+0000","last_change":"2024-08-17T17:27:05.818538+0000","last_active":"2024-08-17T17:29:38.865628+0000","last_peered":"2024-08-17T17:29:38.865628+0000","last_clean":"2024-08-17T17:29:38.865628+0000","last_became_active":"2024-08-17T17:27:05.818278+0000","last_became_peered":"2024-08-17T17:27:05.818278+0000","last_unstale":"2024-08-17T17:29:38.865628+0000","last_undegraded":"2024-08-17T17:29:38.865628+0000","last_fullsized":"2024-08-17T17:29:38.865628+0000","mapping_epoch":283,"log_start":"287'500","ondisk_log_start":"287'500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:06.057185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314310656,"num_objects":4796,"num_object_clones":0,"num_object_copies":14388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10592,"num_write_kb":492416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":257,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479942+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:29:18.479942+0000","last_peered":"2024-08-17T17:29:18.479942+0000","last_clean":"2024-08-17T17:29:18.479942+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:29:18.479942+0000","last_undegraded":"2024-08-17T17:29:18.479942+0000","last_fullsized":"2024-08-17T17:29:18.479942+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":273,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480316+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:29:18.480316+0000","last_peered":"2024-08-17T17:29:18.480316+0000","last_clean":"2024-08-17T17:29:18.480316+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:29:18.480316+0000","last_undegraded":"2024-08-17T17:29:18.480316+0000","last_fullsized":"2024-08-17T17:29:18.480316+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"302'10624","reported_seq":10651,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.974625+0000","last_change":"2024-08-17T17:27:05.818745+0000","last_active":"2024-08-17T17:29:37.974625+0000","last_peered":"2024-08-17T17:29:37.974625+0000","last_clean":"2024-08-17T17:29:37.974625+0000","last_became_active":"2024-08-17T17:27:05.818533+0000","last_became_peered":"2024-08-17T17:27:05.818533+0000","last_unstale":"2024-08-17T17:29:37.974625+0000","last_undegraded":"2024-08-17T17:29:37.974625+0000","last_fullsized":"2024-08-17T17:29:37.974625+0000","mapping_epoch":283,"log_start":"287'2000","ondisk_log_start":"287'2000","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8624,"log_dups_size":0,"ondisk_log_size":8624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:41.413190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315752448,"num_objects":4818,"num_object_clones":0,"num_object_copies":14454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10624,"num_write_kb":494144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479541+0000","last_change":"2024-08-17T17:26:32.216646+0000","last_active":"2024-08-17T17:29:18.479541+0000","last_peered":"2024-08-17T17:29:18.479541+0000","last_clean":"2024-08-17T17:29:18.479541+0000","last_became_active":"2024-08-17T17:26:32.216155+0000","last_became_peered":"2024-08-17T17:26:32.216155+0000","last_unstale":"2024-08-17T17:29:18.479541+0000","last_undegraded":"2024-08-17T17:29:18.479541+0000","last_fullsized":"2024-08-17T17:29:18.479541+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:55.449331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480285+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:29:18.480285+0000","last_peered":"2024-08-17T17:29:18.480285+0000","last_clean":"2024-08-17T17:29:18.480285+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:29:18.480285+0000","last_undegraded":"2024-08-17T17:29:18.480285+0000","last_fullsized":"2024-08-17T17:29:18.480285+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"302'10531","reported_seq":10580,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:39.682814+0000","last_change":"2024-08-17T17:27:05.297991+0000","last_active":"2024-08-17T17:29:39.682814+0000","last_peered":"2024-08-17T17:29:39.682814+0000","last_clean":"2024-08-17T17:29:39.682814+0000","last_became_active":"2024-08-17T17:27:05.297666+0000","last_became_peered":"2024-08-17T17:27:05.297666+0000","last_unstale":"2024-08-17T17:29:39.682814+0000","last_undegraded":"2024-08-17T17:29:39.682814+0000","last_fullsized":"2024-08-17T17:29:39.682814+0000","mapping_epoch":283,"log_start":"288'2600","ondisk_log_start":"288'2600","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7931,"log_dups_size":0,"ondisk_log_size":7931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:35.344484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318570496,"num_objects":4861,"num_object_clones":0,"num_object_copies":14583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10531,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.193087+0000","last_change":"2024-08-17T17:26:32.212135+0000","last_active":"2024-08-17T17:29:33.193087+0000","last_peered":"2024-08-17T17:29:33.193087+0000","last_clean":"2024-08-17T17:29:33.193087+0000","last_became_active":"2024-08-17T17:26:32.211604+0000","last_became_peered":"2024-08-17T17:26:32.211604+0000","last_unstale":"2024-08-17T17:29:33.193087+0000","last_undegraded":"2024-08-17T17:29:33.193087+0000","last_fullsized":"2024-08-17T17:29:33.193087+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:49.886809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":350,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.193151+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:29:33.193151+0000","last_peered":"2024-08-17T17:29:33.193151+0000","last_clean":"2024-08-17T17:29:33.193151+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:29:33.193151+0000","last_undegraded":"2024-08-17T17:29:33.193151+0000","last_fullsized":"2024-08-17T17:29:33.193151+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"302'10631","reported_seq":10681,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:39.709201+0000","last_change":"2024-08-17T17:27:05.297901+0000","last_active":"2024-08-17T17:29:39.709201+0000","last_peered":"2024-08-17T17:29:39.709201+0000","last_clean":"2024-08-17T17:29:39.709201+0000","last_became_active":"2024-08-17T17:27:05.297625+0000","last_became_peered":"2024-08-17T17:27:05.297625+0000","last_unstale":"2024-08-17T17:29:39.709201+0000","last_undegraded":"2024-08-17T17:29:39.709201+0000","last_fullsized":"2024-08-17T17:29:39.709201+0000","mapping_epoch":283,"log_start":"288'2700","ondisk_log_start":"288'2700","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7931,"log_dups_size":0,"ondisk_log_size":7931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:27:29.568335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310050816,"num_objects":4731,"num_object_clones":0,"num_object_copies":14193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10631,"num_write_kb":491584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":624,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474682+0000","last_change":"2024-08-17T17:26:33.204740+0000","last_active":"2024-08-17T17:29:18.474682+0000","last_peered":"2024-08-17T17:29:18.474682+0000","last_clean":"2024-08-17T17:29:18.474682+0000","last_became_active":"2024-08-17T17:26:33.204340+0000","last_became_peered":"2024-08-17T17:26:33.204340+0000","last_unstale":"2024-08-17T17:29:18.474682+0000","last_undegraded":"2024-08-17T17:29:18.474682+0000","last_fullsized":"2024-08-17T17:29:18.474682+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:43.802253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":209,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483325+0000","last_change":"2024-08-17T17:26:32.219571+0000","last_active":"2024-08-17T17:29:18.483325+0000","last_peered":"2024-08-17T17:29:18.483325+0000","last_clean":"2024-08-17T17:29:18.483325+0000","last_became_active":"2024-08-17T17:26:32.219094+0000","last_became_peered":"2024-08-17T17:26:32.219094+0000","last_unstale":"2024-08-17T17:29:18.483325+0000","last_undegraded":"2024-08-17T17:29:18.483325+0000","last_fullsized":"2024-08-17T17:29:18.483325+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:09.450772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":392,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.193733+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:29:33.193733+0000","last_peered":"2024-08-17T17:29:33.193733+0000","last_clean":"2024-08-17T17:29:33.193733+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:29:33.193733+0000","last_undegraded":"2024-08-17T17:29:33.193733+0000","last_fullsized":"2024-08-17T17:29:33.193733+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480322+0000","last_change":"2024-08-17T17:26:32.215221+0000","last_active":"2024-08-17T17:29:18.480322+0000","last_peered":"2024-08-17T17:29:18.480322+0000","last_clean":"2024-08-17T17:29:18.480322+0000","last_became_active":"2024-08-17T17:26:32.214756+0000","last_became_peered":"2024-08-17T17:26:32.214756+0000","last_unstale":"2024-08-17T17:29:18.480322+0000","last_undegraded":"2024-08-17T17:29:18.480322+0000","last_fullsized":"2024-08-17T17:29:18.480322+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.545194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"302'10834","reported_seq":10884,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:39.624347+0000","last_change":"2024-08-17T17:27:05.819104+0000","last_active":"2024-08-17T17:29:39.624347+0000","last_peered":"2024-08-17T17:29:39.624347+0000","last_clean":"2024-08-17T17:29:39.624347+0000","last_became_active":"2024-08-17T17:27:05.818813+0000","last_became_peered":"2024-08-17T17:27:05.818813+0000","last_unstale":"2024-08-17T17:29:39.624347+0000","last_undegraded":"2024-08-17T17:29:39.624347+0000","last_fullsized":"2024-08-17T17:29:39.624347+0000","mapping_epoch":283,"log_start":"288'2900","ondisk_log_start":"288'2900","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7934,"log_dups_size":0,"ondisk_log_size":7934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:28.659641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319029248,"num_objects":4868,"num_object_clones":0,"num_object_copies":14604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10834,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":298,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474713+0000","last_change":"2024-08-17T17:26:32.212907+0000","last_active":"2024-08-17T17:29:18.474713+0000","last_peered":"2024-08-17T17:29:18.474713+0000","last_clean":"2024-08-17T17:29:18.474713+0000","last_became_active":"2024-08-17T17:26:32.212549+0000","last_became_peered":"2024-08-17T17:26:32.212549+0000","last_unstale":"2024-08-17T17:29:18.474713+0000","last_undegraded":"2024-08-17T17:29:18.474713+0000","last_fullsized":"2024-08-17T17:29:18.474713+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:35.071333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":238,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479334+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:29:18.479334+0000","last_peered":"2024-08-17T17:29:18.479334+0000","last_clean":"2024-08-17T17:29:18.479334+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:29:18.479334+0000","last_undegraded":"2024-08-17T17:29:18.479334+0000","last_fullsized":"2024-08-17T17:29:18.479334+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.203526+0000","last_change":"2024-08-17T17:24:37.073148+0000","last_active":"2024-08-17T17:29:33.203526+0000","last_peered":"2024-08-17T17:29:33.203526+0000","last_clean":"2024-08-17T17:29:33.203526+0000","last_became_active":"2024-08-17T17:24:37.072372+0000","last_became_peered":"2024-08-17T17:24:37.072372+0000","last_unstale":"2024-08-17T17:29:33.203526+0000","last_undegraded":"2024-08-17T17:29:33.203526+0000","last_fullsized":"2024-08-17T17:29:33.203526+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.659683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"302'10507","reported_seq":10557,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:39.703430+0000","last_change":"2024-08-17T17:27:05.298546+0000","last_active":"2024-08-17T17:29:39.703430+0000","last_peered":"2024-08-17T17:29:39.703430+0000","last_clean":"2024-08-17T17:29:39.703430+0000","last_became_active":"2024-08-17T17:27:05.297541+0000","last_became_peered":"2024-08-17T17:27:05.297541+0000","last_unstale":"2024-08-17T17:29:39.703430+0000","last_undegraded":"2024-08-17T17:29:39.703430+0000","last_fullsized":"2024-08-17T17:29:39.703430+0000","mapping_epoch":283,"log_start":"288'2600","ondisk_log_start":"288'2600","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:41.264251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316604416,"num_objects":4831,"num_object_clones":0,"num_object_copies":14493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10507,"num_write_kb":490816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":468,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.190594+0000","last_change":"2024-08-17T17:24:36.969933+0000","last_active":"2024-08-17T17:29:33.190594+0000","last_peered":"2024-08-17T17:29:33.190594+0000","last_clean":"2024-08-17T17:29:33.190594+0000","last_became_active":"2024-08-17T17:24:36.969657+0000","last_became_peered":"2024-08-17T17:24:36.969657+0000","last_unstale":"2024-08-17T17:29:33.190594+0000","last_undegraded":"2024-08-17T17:29:33.190594+0000","last_fullsized":"2024-08-17T17:29:33.190594+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:10.367450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":119,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.190722+0000","last_change":"2024-08-17T17:26:50.368470+0000","last_active":"2024-08-17T17:29:33.190722+0000","last_peered":"2024-08-17T17:29:33.190722+0000","last_clean":"2024-08-17T17:29:33.190722+0000","last_became_active":"2024-08-17T17:26:50.367844+0000","last_became_peered":"2024-08-17T17:26:50.367844+0000","last_unstale":"2024-08-17T17:29:33.190722+0000","last_undegraded":"2024-08-17T17:29:33.190722+0000","last_fullsized":"2024-08-17T17:29:33.190722+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:02.244611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":485,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.202983+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:29:33.202983+0000","last_peered":"2024-08-17T17:29:33.202983+0000","last_clean":"2024-08-17T17:29:33.202983+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:29:33.202983+0000","last_undegraded":"2024-08-17T17:29:33.202983+0000","last_fullsized":"2024-08-17T17:29:33.202983+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"302'10727","reported_seq":10777,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:39.706049+0000","last_change":"2024-08-17T17:27:05.819139+0000","last_active":"2024-08-17T17:29:39.706049+0000","last_peered":"2024-08-17T17:29:39.706049+0000","last_clean":"2024-08-17T17:29:39.706049+0000","last_became_active":"2024-08-17T17:27:05.818873+0000","last_became_peered":"2024-08-17T17:27:05.818873+0000","last_unstale":"2024-08-17T17:29:39.706049+0000","last_undegraded":"2024-08-17T17:29:39.706049+0000","last_fullsized":"2024-08-17T17:29:39.706049+0000","mapping_epoch":283,"log_start":"288'2800","ondisk_log_start":"288'2800","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:16.338176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320012288,"num_objects":4883,"num_object_clones":0,"num_object_copies":14649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10727,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":276,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.194725+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:29:33.194725+0000","last_peered":"2024-08-17T17:29:33.194725+0000","last_clean":"2024-08-17T17:29:33.194725+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:29:33.194725+0000","last_undegraded":"2024-08-17T17:29:33.194725+0000","last_fullsized":"2024-08-17T17:29:33.194725+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":277,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479959+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:29:18.479959+0000","last_peered":"2024-08-17T17:29:18.479959+0000","last_clean":"2024-08-17T17:29:18.479959+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:29:18.479959+0000","last_undegraded":"2024-08-17T17:29:18.479959+0000","last_fullsized":"2024-08-17T17:29:18.479959+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192198+0000","last_change":"2024-08-17T17:26:50.310453+0000","last_active":"2024-08-17T17:29:33.192198+0000","last_peered":"2024-08-17T17:29:33.192198+0000","last_clean":"2024-08-17T17:29:33.192198+0000","last_became_active":"2024-08-17T17:26:50.310180+0000","last_became_peered":"2024-08-17T17:26:50.310180+0000","last_unstale":"2024-08-17T17:29:33.192198+0000","last_undegraded":"2024-08-17T17:29:33.192198+0000","last_fullsized":"2024-08-17T17:29:33.192198+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:38.334936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"302'10358","reported_seq":10409,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.887927+0000","last_change":"2024-08-17T17:27:05.299496+0000","last_active":"2024-08-17T17:29:38.887927+0000","last_peered":"2024-08-17T17:29:38.887927+0000","last_clean":"2024-08-17T17:29:38.887927+0000","last_became_active":"2024-08-17T17:27:05.299195+0000","last_became_peered":"2024-08-17T17:27:05.299195+0000","last_unstale":"2024-08-17T17:29:38.887927+0000","last_undegraded":"2024-08-17T17:29:38.887927+0000","last_fullsized":"2024-08-17T17:29:38.887927+0000","mapping_epoch":283,"log_start":"287'300","ondisk_log_start":"287'300","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:06.961502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313196568,"num_objects":4780,"num_object_clones":0,"num_object_copies":14340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4780,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10358,"num_write_kb":484353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":206,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483446+0000","last_change":"2024-08-17T17:25:46.633494+0000","last_active":"2024-08-17T17:29:18.483446+0000","last_peered":"2024-08-17T17:29:18.483446+0000","last_clean":"2024-08-17T17:29:18.483446+0000","last_became_active":"2024-08-17T17:25:46.633216+0000","last_became_peered":"2024-08-17T17:25:46.633216+0000","last_unstale":"2024-08-17T17:29:18.483446+0000","last_undegraded":"2024-08-17T17:29:18.483446+0000","last_fullsized":"2024-08-17T17:29:18.483446+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:14.957437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":463,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474905+0000","last_change":"2024-08-17T17:26:50.338185+0000","last_active":"2024-08-17T17:29:18.474905+0000","last_peered":"2024-08-17T17:29:18.474905+0000","last_clean":"2024-08-17T17:29:18.474905+0000","last_became_active":"2024-08-17T17:26:50.337992+0000","last_became_peered":"2024-08-17T17:26:50.337992+0000","last_unstale":"2024-08-17T17:29:18.474905+0000","last_undegraded":"2024-08-17T17:29:18.474905+0000","last_fullsized":"2024-08-17T17:29:18.474905+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:26.720733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":435,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.194947+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:29:33.194947+0000","last_peered":"2024-08-17T17:29:33.194947+0000","last_clean":"2024-08-17T17:29:33.194947+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:29:33.194947+0000","last_undegraded":"2024-08-17T17:29:33.194947+0000","last_fullsized":"2024-08-17T17:29:33.194947+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"302'10450","reported_seq":10500,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:36.228313+0000","last_change":"2024-08-17T17:27:05.296016+0000","last_active":"2024-08-17T17:29:36.228313+0000","last_peered":"2024-08-17T17:29:36.228313+0000","last_clean":"2024-08-17T17:29:36.228313+0000","last_became_active":"2024-08-17T17:27:05.295582+0000","last_became_peered":"2024-08-17T17:27:05.295582+0000","last_unstale":"2024-08-17T17:29:36.228313+0000","last_undegraded":"2024-08-17T17:29:36.228313+0000","last_fullsized":"2024-08-17T17:29:36.228313+0000","mapping_epoch":283,"log_start":"288'2900","ondisk_log_start":"288'2900","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:39:20.962510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322043904,"num_objects":4914,"num_object_clones":0,"num_object_copies":14742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10450,"num_write_kb":491648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192015+0000","last_change":"2024-08-17T17:26:50.315231+0000","last_active":"2024-08-17T17:29:33.192015+0000","last_peered":"2024-08-17T17:29:33.192015+0000","last_clean":"2024-08-17T17:29:33.192015+0000","last_became_active":"2024-08-17T17:26:50.314945+0000","last_became_peered":"2024-08-17T17:26:50.314945+0000","last_unstale":"2024-08-17T17:29:33.192015+0000","last_undegraded":"2024-08-17T17:29:33.192015+0000","last_fullsized":"2024-08-17T17:29:33.192015+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:13.740267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":281,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474892+0000","last_change":"2024-08-17T17:26:32.215417+0000","last_active":"2024-08-17T17:29:18.474892+0000","last_peered":"2024-08-17T17:29:18.474892+0000","last_clean":"2024-08-17T17:29:18.474892+0000","last_became_active":"2024-08-17T17:26:32.215160+0000","last_became_peered":"2024-08-17T17:26:32.215160+0000","last_unstale":"2024-08-17T17:29:18.474892+0000","last_undegraded":"2024-08-17T17:29:18.474892+0000","last_fullsized":"2024-08-17T17:29:18.474892+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:17.370439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":344,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483362+0000","last_change":"2024-08-17T17:26:32.221201+0000","last_active":"2024-08-17T17:29:18.483362+0000","last_peered":"2024-08-17T17:29:18.483362+0000","last_clean":"2024-08-17T17:29:18.483362+0000","last_became_active":"2024-08-17T17:26:32.220913+0000","last_became_peered":"2024-08-17T17:26:32.220913+0000","last_unstale":"2024-08-17T17:29:18.483362+0000","last_undegraded":"2024-08-17T17:29:18.483362+0000","last_fullsized":"2024-08-17T17:29:18.483362+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:20.201903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"302'10459","reported_seq":10486,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.232226+0000","last_change":"2024-08-17T17:27:05.300239+0000","last_active":"2024-08-17T17:29:35.232226+0000","last_peered":"2024-08-17T17:29:35.232226+0000","last_clean":"2024-08-17T17:29:35.232226+0000","last_became_active":"2024-08-17T17:27:05.296428+0000","last_became_peered":"2024-08-17T17:27:05.296428+0000","last_unstale":"2024-08-17T17:29:35.232226+0000","last_undegraded":"2024-08-17T17:29:35.232226+0000","last_fullsized":"2024-08-17T17:29:35.232226+0000","mapping_epoch":283,"log_start":"287'2100","ondisk_log_start":"287'2100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8359,"log_dups_size":0,"ondisk_log_size":8359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:37:11.949968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329187328,"num_objects":5023,"num_object_clones":0,"num_object_copies":15069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10459,"num_write_kb":495424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":454,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.193193+0000","last_change":"2024-08-17T17:26:32.209014+0000","last_active":"2024-08-17T17:29:33.193193+0000","last_peered":"2024-08-17T17:29:33.193193+0000","last_clean":"2024-08-17T17:29:33.193193+0000","last_became_active":"2024-08-17T17:26:32.208539+0000","last_became_peered":"2024-08-17T17:26:32.208539+0000","last_unstale":"2024-08-17T17:29:33.193193+0000","last_undegraded":"2024-08-17T17:29:33.193193+0000","last_fullsized":"2024-08-17T17:29:33.193193+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:40:15.666378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":434,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192131+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:29:33.192131+0000","last_peered":"2024-08-17T17:29:33.192131+0000","last_clean":"2024-08-17T17:29:33.192131+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:29:33.192131+0000","last_undegraded":"2024-08-17T17:29:33.192131+0000","last_fullsized":"2024-08-17T17:29:33.192131+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":444,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192744+0000","last_change":"2024-08-17T17:26:32.211876+0000","last_active":"2024-08-17T17:29:33.192744+0000","last_peered":"2024-08-17T17:29:33.192744+0000","last_clean":"2024-08-17T17:29:33.192744+0000","last_became_active":"2024-08-17T17:26:32.211212+0000","last_became_peered":"2024-08-17T17:26:32.211212+0000","last_unstale":"2024-08-17T17:29:33.192744+0000","last_undegraded":"2024-08-17T17:29:33.192744+0000","last_fullsized":"2024-08-17T17:29:33.192744+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:50.032892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"302'10338","reported_seq":10365,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.580954+0000","last_change":"2024-08-17T17:27:05.291199+0000","last_active":"2024-08-17T17:29:37.580954+0000","last_peered":"2024-08-17T17:29:37.580954+0000","last_clean":"2024-08-17T17:29:37.580954+0000","last_became_active":"2024-08-17T17:27:05.290910+0000","last_became_peered":"2024-08-17T17:27:05.290910+0000","last_unstale":"2024-08-17T17:29:37.580954+0000","last_undegraded":"2024-08-17T17:29:37.580954+0000","last_fullsized":"2024-08-17T17:29:37.580954+0000","mapping_epoch":283,"log_start":"287'2200","ondisk_log_start":"287'2200","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8138,"log_dups_size":0,"ondisk_log_size":8138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:32.168692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314834944,"num_objects":4804,"num_object_clones":0,"num_object_copies":14412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10338,"num_write_kb":484544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":124,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.203129+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:29:33.203129+0000","last_peered":"2024-08-17T17:29:33.203129+0000","last_clean":"2024-08-17T17:29:33.203129+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:29:33.203129+0000","last_undegraded":"2024-08-17T17:29:33.203129+0000","last_fullsized":"2024-08-17T17:29:33.203129+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":286,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480392+0000","last_change":"2024-08-17T17:26:32.218111+0000","last_active":"2024-08-17T17:29:18.480392+0000","last_peered":"2024-08-17T17:29:18.480392+0000","last_clean":"2024-08-17T17:29:18.480392+0000","last_became_active":"2024-08-17T17:26:32.217705+0000","last_became_peered":"2024-08-17T17:26:32.217705+0000","last_unstale":"2024-08-17T17:29:18.480392+0000","last_undegraded":"2024-08-17T17:29:18.480392+0000","last_fullsized":"2024-08-17T17:29:18.480392+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:50.751085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479519+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:29:18.479519+0000","last_peered":"2024-08-17T17:29:18.479519+0000","last_clean":"2024-08-17T17:29:18.479519+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:29:18.479519+0000","last_undegraded":"2024-08-17T17:29:18.479519+0000","last_fullsized":"2024-08-17T17:29:18.479519+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479895+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:29:18.479895+0000","last_peered":"2024-08-17T17:29:18.479895+0000","last_clean":"2024-08-17T17:29:18.479895+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:29:18.479895+0000","last_undegraded":"2024-08-17T17:29:18.479895+0000","last_fullsized":"2024-08-17T17:29:18.479895+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":279,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.481923+0000","last_change":"2024-08-17T17:26:32.215288+0000","last_active":"2024-08-17T17:29:18.481923+0000","last_peered":"2024-08-17T17:29:18.481923+0000","last_clean":"2024-08-17T17:29:18.481923+0000","last_became_active":"2024-08-17T17:26:32.214754+0000","last_became_peered":"2024-08-17T17:26:32.214754+0000","last_unstale":"2024-08-17T17:29:18.481923+0000","last_undegraded":"2024-08-17T17:29:18.481923+0000","last_fullsized":"2024-08-17T17:29:18.481923+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.684058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":434,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.194811+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:29:33.194811+0000","last_peered":"2024-08-17T17:29:33.194811+0000","last_clean":"2024-08-17T17:29:33.194811+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:29:33.194811+0000","last_undegraded":"2024-08-17T17:29:33.194811+0000","last_fullsized":"2024-08-17T17:29:33.194811+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":233,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.474806+0000","last_change":"2024-08-17T17:26:32.207981+0000","last_active":"2024-08-17T17:29:18.474806+0000","last_peered":"2024-08-17T17:29:18.474806+0000","last_clean":"2024-08-17T17:29:18.474806+0000","last_became_active":"2024-08-17T17:26:32.207636+0000","last_became_peered":"2024-08-17T17:26:32.207636+0000","last_unstale":"2024-08-17T17:29:18.474806+0000","last_undegraded":"2024-08-17T17:29:18.474806+0000","last_fullsized":"2024-08-17T17:29:18.474806+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:32.412636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":196,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483492+0000","last_change":"2024-08-17T17:26:32.210741+0000","last_active":"2024-08-17T17:29:18.483492+0000","last_peered":"2024-08-17T17:29:18.483492+0000","last_clean":"2024-08-17T17:29:18.483492+0000","last_became_active":"2024-08-17T17:26:32.210417+0000","last_became_peered":"2024-08-17T17:26:32.210417+0000","last_unstale":"2024-08-17T17:29:18.483492+0000","last_undegraded":"2024-08-17T17:29:18.483492+0000","last_fullsized":"2024-08-17T17:29:18.483492+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:41.290842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":392,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.195793+0000","last_change":"2024-08-17T17:26:32.211602+0000","last_active":"2024-08-17T17:29:33.195793+0000","last_peered":"2024-08-17T17:29:33.195793+0000","last_clean":"2024-08-17T17:29:33.195793+0000","last_became_active":"2024-08-17T17:26:32.211288+0000","last_became_peered":"2024-08-17T17:26:32.211288+0000","last_unstale":"2024-08-17T17:29:33.195793+0000","last_undegraded":"2024-08-17T17:29:33.195793+0000","last_fullsized":"2024-08-17T17:29:33.195793+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:49.979367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":350,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.483658+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:29:18.483658+0000","last_peered":"2024-08-17T17:29:18.483658+0000","last_clean":"2024-08-17T17:29:18.483658+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:29:18.483658+0000","last_undegraded":"2024-08-17T17:29:18.483658+0000","last_fullsized":"2024-08-17T17:29:18.483658+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192661+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:29:33.192661+0000","last_peered":"2024-08-17T17:29:33.192661+0000","last_clean":"2024-08-17T17:29:33.192661+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:29:33.192661+0000","last_undegraded":"2024-08-17T17:29:33.192661+0000","last_fullsized":"2024-08-17T17:29:33.192661+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":273,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480349+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:29:18.480349+0000","last_peered":"2024-08-17T17:29:18.480349+0000","last_clean":"2024-08-17T17:29:18.480349+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:29:18.480349+0000","last_undegraded":"2024-08-17T17:29:18.480349+0000","last_fullsized":"2024-08-17T17:29:18.480349+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":425,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192865+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:29:33.192865+0000","last_peered":"2024-08-17T17:29:33.192865+0000","last_clean":"2024-08-17T17:29:33.192865+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:29:33.192865+0000","last_undegraded":"2024-08-17T17:29:33.192865+0000","last_fullsized":"2024-08-17T17:29:33.192865+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":230,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.475021+0000","last_change":"2024-08-17T17:26:32.215823+0000","last_active":"2024-08-17T17:29:18.475021+0000","last_peered":"2024-08-17T17:29:18.475021+0000","last_clean":"2024-08-17T17:29:18.475021+0000","last_became_active":"2024-08-17T17:26:32.214110+0000","last_became_peered":"2024-08-17T17:26:32.214110+0000","last_unstale":"2024-08-17T17:29:18.475021+0000","last_undegraded":"2024-08-17T17:29:18.475021+0000","last_fullsized":"2024-08-17T17:29:18.475021+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.088172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":433,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.191710+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:29:33.191710+0000","last_peered":"2024-08-17T17:29:33.191710+0000","last_clean":"2024-08-17T17:29:33.191710+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:29:33.191710+0000","last_undegraded":"2024-08-17T17:29:33.191710+0000","last_fullsized":"2024-08-17T17:29:33.191710+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":392,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.194439+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:29:33.194439+0000","last_peered":"2024-08-17T17:29:33.194439+0000","last_clean":"2024-08-17T17:29:33.194439+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:29:33.194439+0000","last_undegraded":"2024-08-17T17:29:33.194439+0000","last_fullsized":"2024-08-17T17:29:33.194439+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479633+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:29:18.479633+0000","last_peered":"2024-08-17T17:29:18.479633+0000","last_clean":"2024-08-17T17:29:18.479633+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:29:18.479633+0000","last_undegraded":"2024-08-17T17:29:18.479633+0000","last_fullsized":"2024-08-17T17:29:18.479633+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":456,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.192109+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:29:33.192109+0000","last_peered":"2024-08-17T17:29:33.192109+0000","last_clean":"2024-08-17T17:29:33.192109+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:29:33.192109+0000","last_undegraded":"2024-08-17T17:29:33.192109+0000","last_fullsized":"2024-08-17T17:29:33.192109+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":131,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.201928+0000","last_change":"2024-08-17T17:26:32.214552+0000","last_active":"2024-08-17T17:29:33.201928+0000","last_peered":"2024-08-17T17:29:33.201928+0000","last_clean":"2024-08-17T17:29:33.201928+0000","last_became_active":"2024-08-17T17:26:32.214338+0000","last_became_peered":"2024-08-17T17:26:32.214338+0000","last_unstale":"2024-08-17T17:29:33.201928+0000","last_undegraded":"2024-08-17T17:29:33.201928+0000","last_fullsized":"2024-08-17T17:29:33.201928+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:21.212607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":283,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479827+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:29:18.479827+0000","last_peered":"2024-08-17T17:29:18.479827+0000","last_clean":"2024-08-17T17:29:18.479827+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:29:18.479827+0000","last_undegraded":"2024-08-17T17:29:18.479827+0000","last_fullsized":"2024-08-17T17:29:18.479827+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":156,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.481856+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:29:18.481856+0000","last_peered":"2024-08-17T17:29:18.481856+0000","last_clean":"2024-08-17T17:29:18.481856+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:29:18.481856+0000","last_undegraded":"2024-08-17T17:29:18.481856+0000","last_fullsized":"2024-08-17T17:29:18.481856+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479524+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:29:18.479524+0000","last_peered":"2024-08-17T17:29:18.479524+0000","last_clean":"2024-08-17T17:29:18.479524+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:29:18.479524+0000","last_undegraded":"2024-08-17T17:29:18.479524+0000","last_fullsized":"2024-08-17T17:29:18.479524+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":156,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.480167+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:29:18.480167+0000","last_peered":"2024-08-17T17:29:18.480167+0000","last_clean":"2024-08-17T17:29:18.480167+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:29:18.480167+0000","last_undegraded":"2024-08-17T17:29:18.480167+0000","last_fullsized":"2024-08-17T17:29:18.480167+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":438,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.196189+0000","last_change":"2024-08-17T17:24:36.972596+0000","last_active":"2024-08-17T17:29:33.196189+0000","last_peered":"2024-08-17T17:29:33.196189+0000","last_clean":"2024-08-17T17:29:33.196189+0000","last_became_active":"2024-08-17T17:24:36.971541+0000","last_became_peered":"2024-08-17T17:24:36.971541+0000","last_unstale":"2024-08-17T17:29:33.196189+0000","last_undegraded":"2024-08-17T17:29:33.196189+0000","last_fullsized":"2024-08-17T17:29:33.196189+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:38.457003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":410,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.194868+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:29:33.194868+0000","last_peered":"2024-08-17T17:29:33.194868+0000","last_clean":"2024-08-17T17:29:33.194868+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:29:33.194868+0000","last_undegraded":"2024-08-17T17:29:33.194868+0000","last_fullsized":"2024-08-17T17:29:33.194868+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-08-17T17:29:18.479324+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:29:18.479324+0000","last_peered":"2024-08-17T17:29:18.479324+0000","last_clean":"2024-08-17T17:29:18.479324+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:29:18.479324+0000","last_undegraded":"2024-08-17T17:29:18.479324+0000","last_fullsized":"2024-08-17T17:29:18.479324+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":126,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.202153+0000","last_change":"2024-08-17T17:24:37.032048+0000","last_active":"2024-08-17T17:29:33.202153+0000","last_peered":"2024-08-17T17:29:33.202153+0000","last_clean":"2024-08-17T17:29:33.202153+0000","last_became_active":"2024-08-17T17:24:36.972161+0000","last_became_peered":"2024-08-17T17:24:36.972161+0000","last_unstale":"2024-08-17T17:29:33.202153+0000","last_undegraded":"2024-08-17T17:29:33.202153+0000","last_fullsized":"2024-08-17T17:29:33.202153+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:55.363820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5096734744,"num_objects":77771,"num_object_clones":0,"num_object_copies":233313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77771,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169063,"num_write_kb":7898625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15344414720,"data_stored":15344402504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137963,"ondisk_log_size":137963,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1162,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":270,"seq":1159641169958,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307180,"kb_used_data":1928608,"kb_used_omap":12,"kb_used_meta":378547,"kb_avail":91437972,"statfs":{"total":95995035648,"available":93632483328,"internally_reserved":0,"allocated":1974894592,"data_stored":1973331295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12410,"internal_metadata":387633030},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463328,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3401124,"kb_used_data":2511580,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90344028,"statfs":{"total":95995035648,"available":92512284672,"internally_reserved":0,"allocated":2571857920,"data_stored":2570298600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492003,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1616588,"kb_used_data":1229572,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92128564,"statfs":{"total":95995035648,"available":94339649536,"internally_reserved":0,"allocated":1259081728,"data_stored":1257541192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986602,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2145440,"kb_used_data":1252824,"kb_used_omap":1,"kb_used_meta":892606,"kb_avail":91599712,"statfs":{"total":95995035648,"available":93798105088,"internally_reserved":0,"allocated":1282891776,"data_stored":1281335808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914028888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225422,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2897608,"kb_used_data":2181696,"kb_used_omap":4,"kb_used_meta":715451,"kb_avail":90847544,"statfs":{"total":95995035648,"available":93027885056,"internally_reserved":0,"allocated":2234056704,"data_stored":2232520492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":732622272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075747,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2363540,"kb_used_data":1853196,"kb_used_omap":6,"kb_used_meta":510329,"kb_avail":91381612,"statfs":{"total":95995035648,"available":93574770688,"internally_reserved":0,"allocated":1897672704,"data_stored":1896125031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":522577339},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533639,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3414924,"kb_used_data":2552000,"kb_used_omap":5,"kb_used_meta":862906,"kb_avail":90330228,"statfs":{"total":95995035648,"available":92498153472,"internally_reserved":0,"allocated":2613248000,"data_stored":2611710586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":883615984},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089616,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490964,"kb_used_data":1590028,"kb_used_omap":5,"kb_used_meta":900922,"kb_avail":91254188,"statfs":{"total":95995035648,"available":93444288512,"internally_reserved":0,"allocated":1628188672,"data_stored":1626641507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":922544187},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1609236480,"data_stored":1609236480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2601189376,"data_stored":2601189376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1888686080,"data_stored":1888682008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2203975680,"data_stored":2203975680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1270091776,"data_stored":1270087704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1249443840,"data_stored":1249443840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2553942016,"data_stored":2553937944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1967849472,"data_stored":1967849472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:29:41.500 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:29:41.500 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:41.500 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-08-17T17:29:41.500 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:29:41.500 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:29:41.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-08-17T17:29:41.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:41.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:41.530 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:41 smithi152 ceph-mon[93456]: pgmap v1015: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:29:41.530 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:41 smithi152 ceph-mon[93456]: from='client.46553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:41.531 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:41 smithi142 ceph-mon[106729]: pgmap v1015: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:29:41.531 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:41 smithi142 ceph-mon[106729]: from='client.46553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:41.531 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:41 smithi142 ceph-mon[103614]: pgmap v1015: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:29:41.531 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:41 smithi142 ceph-mon[103614]: from='client.46553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:41.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:41.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:41.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:41.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:41.855 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:29:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:29:41] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-08-17T17:29:41.891 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:41.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:29:41.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:41.932 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:42.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:42.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:42.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:42.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:42.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:42.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:42.357 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:42.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:42.452 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:42.453 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:42.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:42.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:42.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:42.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:42.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:42.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:42.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:42.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:42.917 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:43.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:43.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.146 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:43.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:43.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:43.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:43.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[106729]: pgmap v1016: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[103614]: pgmap v1016: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:43.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:43 smithi152 ceph-mon[93456]: pgmap v1016: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:43.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:43.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:43.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:43.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:43.597 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:43.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:43.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:43.839 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:43.877 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:43.908 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:29:43.908 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "" 2024-08-17T17:29:43.908 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:29:43.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:44.032 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:44.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:44.161 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:44.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:44.296 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:44.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:44.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:44.463 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:44.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:44.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:44.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:44.717 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:44.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:44.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:44.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:45.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:45.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:45.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:45.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:45.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:45.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:45.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:45.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:45.465 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:29:45.465 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:45.482 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:29:45.483 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:45.483 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:45 smithi142 ceph-mon[103614]: pgmap v1017: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:45.483 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:45 smithi142 ceph-mon[106729]: pgmap v1017: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:45.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:45.517 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:29:45.517 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:45.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:45.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:45.567 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:29:45.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:45 smithi152 ceph-mon[93456]: pgmap v1017: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:45.599 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:29:45.599 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:45.632 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:29:45.632 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:45.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:45.668 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:29:45.668 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:45.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.710 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:29:45.710 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:45.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:45.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:45.917 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:45.948 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:46.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:46.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:46.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:46.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:46.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:46.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:46.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:46.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:46.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3110594589' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:46.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:46.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:46.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:46.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3110594589' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3110594589' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:46.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:46.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:46.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:46.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:46.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:47.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:47.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.142 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:47.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:47.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:47.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:47.375 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:47.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:47.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:47.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:47.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:47.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:47 smithi152 ceph-mon[93456]: pgmap v1018: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-08-17T17:29:47.589 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:47 smithi142 ceph-mon[106729]: pgmap v1018: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-08-17T17:29:47.590 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:47 smithi142 ceph-mon[103614]: pgmap v1018: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-08-17T17:29:47.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:47.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:47.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:47.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:47.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:47.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:48.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:48.078 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:48.180 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:48.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:48.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:48.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:48.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:48.476 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:48.547 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242765 10.f e273/222 10:f5d57385:::benchmark_data_smithi142_192453_object117017:head v 273'15354, mlcod=273'15348)", 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.42327718300001, 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28726092600000003, 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.548 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.549 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069107+0000", 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8599999999999997e-07 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069115+0000", 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7100000000000007e-06 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.550 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069126+0000", 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1321000000000001e-05 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069133+0000", 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4790000000000001e-06 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.070023+0000", 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00089035999999999998 2024-08-17T17:29:48.551 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.070042+0000", 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8760999999999999e-05 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.356328+0000", 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28628575499999998 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.552 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.356367+0000", 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9753999999999999e-05 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242762 10.7 e273/257 10:e5d91e03:::benchmark_data_smithi142_192453_object117014:head v 273'13155, mlcod=273'13153)", 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.423105174, 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28567881899999997, 2024-08-17T17:29:48.553 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.554 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069279+0000", 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5199999999999997e-07 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069286+0000", 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8059999999999997e-06 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.555 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069295+0000", 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9800000000000004e-06 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069300+0000", 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4949999999999999e-06 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.556 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.071296+0000", 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0019955060000000002 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.071313+0000", 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7452000000000002e-05 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.354873+0000", 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28356008999999999 2024-08-17T17:29:48.557 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.354957+0000", 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3937999999999995e-05 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242764 10.13 e273/257 10:cbd587ba:::benchmark_data_smithi142_192453_object117016:head v 273'12990, mlcod=273'12988)", 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.075981+0000", 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.41640276800001, 2024-08-17T17:29:48.558 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27613257200000002, 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075981+0000", 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.559 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075981+0000", 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075982+0000", 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1480000000000001e-06 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075990+0000", 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0209999999999999e-06 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.560 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076003+0000", 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3266e-05 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076011+0000", 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076089+0000", 2024-08-17T17:29:48.561 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7329000000000005e-05 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076108+0000", 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9185e-05 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27596227299999998 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.562 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352113+0000", 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3398e-05 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242769 10.7 e273/257 10:e215c346:::benchmark_data_smithi142_192453_object117021:head v 273'13156, mlcod=273'13153)", 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.41440261299999, 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.281981127, 2024-08-17T17:29:48.563 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:29:48.564 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077982+0000", 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1100000000000005e-07 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077991+0000", 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1940000000000004e-06 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.565 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078003+0000", 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2264e-05 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078010+0000", 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0110000000000003e-06 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.566 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078277+0000", 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000267135 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078299+0000", 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1202000000000001e-05 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359932+0000", 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28163336 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.567 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359962+0000", 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0150000000000001e-05 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242772 10.14 e273/257 10:2c787a87:::benchmark_data_smithi142_192453_object117024:head v 273'13212, mlcod=273'13210)", 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.41123279300001, 2024-08-17T17:29:48.568 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.278431659, 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:29:48.569 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081152+0000", 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0559999999999999e-06 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081159+0000", 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3180000000000001e-06 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.570 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352055+0000", 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27089613600000001 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4620999999999999e-05 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352219+0000", 2024-08-17T17:29:48.571 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149436 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352240+0000", 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0767e-05 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359528+0000", 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072875090000000002 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359582+0000", 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4815999999999998e-05 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242786 10.5 e273/272 10:a01a2aa8:::benchmark_data_smithi142_192453_object117038:head v 273'13406, mlcod=238'8016)", 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.094527+0000", 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 187.397856797, 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27114051900000002, 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.573 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094527+0000", 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094527+0000", 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094528+0000", 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5820000000000001e-06 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094539+0000", 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0192e-05 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352055+0000", 2024-08-17T17:29:48.575 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25751674200000002 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4917e-05 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352261+0000", 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000190373 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.576 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352283+0000", 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2113e-05 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.365645+0000", 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013362384999999999 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.365667+0000", 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2215e-05 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.577 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:29101 11.9 11:91525119:::benchmark_data_smithi142_227468_object29100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.090031+0000", 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 144.40235214099999, 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26436782800000003, 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.578 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 29101 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090031+0000", 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090031+0000", 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090032+0000", 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1899999999999998e-07 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090050+0000", 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7844999999999999e-05 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090057+0000", 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.759e-06 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090062+0000", 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.028e-06 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090106+0000", 2024-08-17T17:29:48.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4416000000000003e-05 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090225+0000", 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011862900000000001 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090358+0000", 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013351300000000001 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.096138+0000", 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0057801750000000002 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.098490+0000", 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023520619999999998 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.354287+0000", 2024-08-17T17:29:48.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25579645099999998 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.354349+0000", 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2259000000000001e-05 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.354399+0000", 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0272000000000003e-05 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.584 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:29103 11.9 11:950c9362:::benchmark_data_smithi142_227468_object29102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.091474+0000", 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 144.400909197, 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.269225565, 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.585 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 29103 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091474+0000", 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091474+0000", 2024-08-17T17:29:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091476+0000", 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1850000000000001e-06 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091497+0000", 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1075000000000001e-05 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091508+0000", 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0883999999999999e-05 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091514+0000", 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6660000000000002e-06 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092564+0000", 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001050095 2024-08-17T17:29:48.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092680+0000", 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116146 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092826+0000", 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014555199999999999 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.100629+0000", 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078026399999999996 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.100675+0000", 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6579999999999998e-05 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.359858+0000", 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25918253899999999 2024-08-17T17:29:48.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.359938+0000", 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0249999999999999e-05 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.360700+0000", 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000761953 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.591 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:29108 11.b 11:d9747203:::benchmark_data_smithi142_227468_object29107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.092813+0000", 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 144.399571103, 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26441371600000002, 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.592 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 29108 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092813+0000", 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092813+0000", 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092817+0000", 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.279e-06 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092840+0000", 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3176e-05 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092850+0000", 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6630000000000005e-06 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092855+0000", 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7389999999999996e-06 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092963+0000", 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108111 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.093058+0000", 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4685999999999994e-05 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.093217+0000", 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000158796 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.095035+0000", 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001817978 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.095732+0000", 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00069662499999999998 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357091+0000", 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26135961000000002 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357182+0000", 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1092000000000001e-05 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357226+0000", 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3961000000000001e-05 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104285 11.9 11:956711b1:::benchmark_data_smithi142_227468_object104284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 86.238064378000004, 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27987187299999999, 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.599 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104285 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254320+0000", 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.226e-06 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254340+0000", 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9253999999999999e-05 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254348+0000", 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4440000000000008e-06 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254353+0000", 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3040000000000004e-06 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.257745+0000", 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003391336 2024-08-17T17:29:48.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.257867+0000", 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012196099999999999 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.258003+0000", 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013589099999999999 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.262664+0000", 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046612679999999997 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.267561+0000", 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048971850000000001 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534082+0000", 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26652041300000001 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534142+0000", 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0158999999999998e-05 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534191+0000", 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9431999999999997e-05 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104299 11.b 11:dd1aa66c:::benchmark_data_smithi142_227468_object104298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 86.229257602000004, 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27979115300000001, 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104299 2024-08-17T17:29:48.606 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263127+0000", 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7000000000000004e-07 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263147+0000", 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0576000000000001e-05 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263158+0000", 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0173999999999999e-05 2024-08-17T17:29:48.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263164+0000", 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7499999999999997e-06 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266586+0000", 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034218400000000002 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266722+0000", 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013612100000000001 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266854+0000", 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132001 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.274109+0000", 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072550760000000001 2024-08-17T17:29:48.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.275555+0000", 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014456219999999999 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542810+0000", 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26725485300000001 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542864+0000", 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3872999999999997e-05 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542917+0000", 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3497e-05 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104303 11.9 11:982685b7:::benchmark_data_smithi142_227468_object104302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:29:48.612 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 86.224158901999999, 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29623361999999998, 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104303 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4900000000000005e-07 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268246+0000", 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0990000000000001e-05 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268255+0000", 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7369999999999995e-06 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268261+0000", 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6749999999999999e-06 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268310+0000", 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9447999999999998e-05 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268420+0000", 2024-08-17T17:29:48.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010979 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268546+0000", 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012620700000000001 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272696+0000", 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041496550000000004 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.617 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277035+0000", 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043387249999999999 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564360+0000", 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28732556599999998 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564416+0000", 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5747e-05 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564458+0000", 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2330999999999998e-05 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104304 11.b 11:dcea405a:::benchmark_data_smithi142_227468_object104303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.269864+0000", 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 86.222519590000005, 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27317428999999999, 2024-08-17T17:29:48.619 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104304 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269864+0000", 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269864+0000", 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269866+0000", 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.474e-06 2024-08-17T17:29:48.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269892+0000", 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6038e-05 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269900+0000", 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.028e-06 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269905+0000", 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3839999999999996e-06 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.270162+0000", 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025740099999999998 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.270259+0000", 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7133000000000007e-05 2024-08-17T17:29:48.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.270418+0000", 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015801300000000001 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277853+0000", 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0074356509999999997 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277864+0000", 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0876999999999999e-05 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542956+0000", 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26509197200000001 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542996+0000", 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0268999999999997e-05 2024-08-17T17:29:48.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.543038+0000", 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2049999999999999e-05 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104305 11.9 11:9307d714:::benchmark_data_smithi142_227468_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:29:48.626 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 86.219684036999993, 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29189848299999999, 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104305 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272701+0000", 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1990000000000001e-06 2024-08-17T17:29:48.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272724+0000", 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3428999999999999e-05 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272736+0000", 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2177e-05 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272743+0000", 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8449999999999997e-06 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273130+0000", 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00038687699999999999 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273244+0000", 2024-08-17T17:29:48.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011396900000000001 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273376+0000", 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000131812 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.281412+0000", 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080358289999999995 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.282800+0000", 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00138784 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564526+0000", 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.281726537 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564568+0000", 2024-08-17T17:29:48.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1570999999999999e-05 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564598+0000", 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0397999999999999e-05 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104325 11.9 11:9715431e:::benchmark_data_smithi142_227468_object104324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:29:48.633 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 86.201356762000003, 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27710011899999998, 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104325 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.634 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291028+0000", 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2929999999999999e-06 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291051+0000", 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2566999999999999e-05 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291064+0000", 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3695e-05 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291071+0000", 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0219999999999999e-06 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291129+0000", 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7970000000000002e-05 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291263+0000", 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291405+0000", 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014159700000000001 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.295160+0000", 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037547869999999999 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.295314+0000", 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015397800000000001 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.567971+0000", 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27265716699999998 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.568083+0000", 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011186799999999999 2024-08-17T17:29:48.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.568127+0000", 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4175000000000003e-05 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162100 11.0 11:02a7f4f2:::benchmark_data_smithi142_227468_object38681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 19.004389161999999, 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.30841822800000002, 2024-08-17T17:29:48.640 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162100 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:29:48.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487996+0000", 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2100000000000001e-06 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.488000+0000", 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6040000000000003e-06 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499381+0000", 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011380988999999999 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499402+0000", 2024-08-17T17:29:48.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0930000000000001e-05 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499544+0000", 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014133400000000001 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499675+0000", 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013168000000000001 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499788+0000", 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112625 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511105+0000", 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011317159 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511158+0000", 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3236000000000003e-05 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796105+0000", 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28494646400000001 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796225+0000", 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120295 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796413+0000", 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018770200000000001 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162107 11.0 11:0142b711:::benchmark_data_smithi142_227468_object38688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 18.992927103, 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29756906700000002, 2024-08-17T17:29:48.647 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162107 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499458+0000", 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8780000000000001e-06 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499465+0000", 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8619999999999997e-06 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499477+0000", 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2011000000000001e-05 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499483+0000", 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4009999999999996e-06 2024-08-17T17:29:48.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.504931+0000", 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054487809999999998 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.505217+0000", 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00028554699999999998 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.505550+0000", 2024-08-17T17:29:48.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00033330399999999998 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511178+0000", 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0056279959999999997 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.512133+0000", 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00095433599999999996 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796542+0000", 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28440920800000002 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796893+0000", 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035122400000000002 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797026+0000", 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132519 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162129 11.0 11:078c6a55:::benchmark_data_smithi142_227468_object38710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:29:48.655 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.524455+0000", 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 18.967928386000001, 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27313411999999998, 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162129 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524455+0000", 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524455+0000", 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524456+0000", 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.13e-06 2024-08-17T17:29:48.657 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524461+0000", 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5299999999999998e-06 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524472+0000", 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1440000000000001e-05 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.658 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524478+0000", 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2680000000000002e-06 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524533+0000", 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5126e-05 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524688+0000", 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015477000000000001 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524817+0000", 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012918200000000001 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.528632+0000", 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038150319999999999 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.528729+0000", 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7169000000000004e-05 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797195+0000", 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26846631999999998 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797373+0000", 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017769800000000001 2024-08-17T17:29:48.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797589+0000", 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000216455 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162133 11.b 11:dac48ceb:::benchmark_data_smithi142_227468_object38714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:29:48.662 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 18.962813577999999, 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29427250700000002, 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162133 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529571+0000", 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.308e-06 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529576+0000", 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0980000000000001e-06 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529588+0000", 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1579999999999999e-05 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.665 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529595+0000", 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.613e-06 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529736+0000", 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014141699999999999 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529840+0000", 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103724 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529960+0000", 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120653 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537643+0000", 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0076821010000000002 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537677+0000", 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4820999999999997e-05 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823705+0000", 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.286027592 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823799+0000", 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3799000000000001e-05 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823843+0000", 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3801000000000003e-05 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162136 11.b 11:d77466cb:::benchmark_data_smithi142_227468_object38717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 18.960894235000001, 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29716111499999998, 2024-08-17T17:29:48.669 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162136 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:29:48.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531491+0000", 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2160000000000001e-06 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531562+0000", 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1204000000000005e-05 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531574+0000", 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1950000000000001e-05 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:29:48.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531581+0000", 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8909999999999998e-06 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534124+0000", 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025436669999999999 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534233+0000", 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010820899999999999 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534413+0000", 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018054799999999999 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537671+0000", 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032577740000000002 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:29:48.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537731+0000", 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0050000000000003e-05 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828485+0000", 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29075420899999999 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828589+0000", 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103916 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:29:48.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828651+0000", 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1481000000000003e-05 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:29:48.676 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:29:48.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:48.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:48.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:48.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:48.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:48.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:48.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:48.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:49.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:49.124 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:49.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:49.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:49.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:49.258 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:29:49.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:49.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:49.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:49.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:49 smithi152 ceph-mon[93456]: pgmap v1019: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-08-17T17:29:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2930974942' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:49 smithi142 ceph-mon[106729]: pgmap v1019: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-08-17T17:29:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2930974942' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:49 smithi142 ceph-mon[103614]: pgmap v1019: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-08-17T17:29:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2930974942' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:49.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:49.639 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:29:49.639 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:49.640 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:29:49.640 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.942334 2024-08-17T17:29:49.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 0.942333506989741 2024-08-17T17:29:49.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:49.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:49.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:49.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:49.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:29:49.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:49.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:50.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:50.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:50.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:50.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:50.218 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:29:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:29:50.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:50.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.230 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:50.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:50.255 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:50.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:50.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:50.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:50.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:50 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:50.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:50 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:29:50.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:50.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:50 smithi152 ceph-mon[93456]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:29:50.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:50.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:29:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[106729]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:29:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:29:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:50 smithi142 ceph-mon[103614]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:29:50.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:50.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:50.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:50.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:50.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:50.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:50.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:51.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:51.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:51.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:51.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:51.240 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:51.256 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:51.311 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:51.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:51 smithi152 ceph-mon[93456]: pgmap v1021: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-08-17T17:29:51.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:51.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:51.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:51 smithi142 ceph-mon[103614]: pgmap v1021: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-08-17T17:29:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:51 smithi142 ceph-mon[106729]: pgmap v1021: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-08-17T17:29:51.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:51.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:51.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:51.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:51.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:51.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:51.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:51.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:51.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:52.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:52.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:52.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:52.039 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:29:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:29:51] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-08-17T17:29:52.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:52.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:52.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:52.311 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:52.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:52.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:52.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:52.463 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:29:52.463 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:29:52.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:180392 11.e 11:752e3453:::benchmark_data_smithi142_227468_object56973:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:29:52.454340+0000", 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0074514550000000001, 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074818439999999996, 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 180392 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.464 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.454340+0000", 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.454340+0000", 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.454338+0000", 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.454344+0000", 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.835e-06 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:29:52.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.454357+0000", 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3203999999999999e-05 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.454363+0000", 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3020000000000002e-06 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:29:52.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.456409+0000", 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020463529999999999 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.456514+0000", 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010487 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.456617+0000", 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102632 2024-08-17T17:29:52.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:52.460882+0000", 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004265357 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 1 2024-08-17T17:29:52.469 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:29:52.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:52.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:52.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:52.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:52.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:52.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:52.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:52.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:52.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:53.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:53.103 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:53.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:29:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:53.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:29:53.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:53.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:53.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:53 smithi142 ceph-mon[103614]: pgmap v1022: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-08-17T17:29:53.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:53 smithi142 ceph-mon[106729]: pgmap v1022: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-08-17T17:29:53.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:53.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:53.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:53.560 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:53.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:53.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:53 smithi152 ceph-mon[93456]: pgmap v1022: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-08-17T17:29:53.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:53.720 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:53.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:53.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:53.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:53.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:54.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:54.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:54.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:54.282 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:54.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:54.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:54.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:54.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:54.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:54.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:54.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:54.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:54.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:54.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:55.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:55.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:55.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:55.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:55.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:55.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:55.306 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:29:55.306 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:29:55.306 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:55.306 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:55.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:29:55.323 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:55.373 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:55.386 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:29:55.386 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:55.414 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:55.428 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:29:55.428 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:55.465 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:55 smithi142 ceph-mon[103614]: pgmap v1023: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:29:55.465 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1337190863' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:55.465 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:55 smithi142 ceph-mon[106729]: pgmap v1023: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:29:55.465 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1337190863' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:55.467 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:29:55.467 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:55.502 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:29:55.502 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:55.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:55.533 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:29:55.533 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:55.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:55.567 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:55 smithi152 ceph-mon[93456]: pgmap v1023: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-17T17:29:55.567 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1337190863' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:55.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:55.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:55.770 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:55.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:55.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:55.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:55.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:55.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:55.988 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:55.988 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:56.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:56.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:56.309 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.6 primary-affinity to 0.942334 (8617562) 2024-08-17T17:29:56.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:56.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2023004011' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741} : dispatch 2024-08-17T17:29:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741} : dispatch 2024-08-17T17:29:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2023004011' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741} : dispatch 2024-08-17T17:29:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741} : dispatch 2024-08-17T17:29:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2023004011' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741} : dispatch 2024-08-17T17:29:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741} : dispatch 2024-08-17T17:29:56.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:56.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:56.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:56.816 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:56.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:56.917 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:29:56.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:56.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:57.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:57.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:57.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:57.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:57.315 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:57 smithi152 ceph-mon[93456]: pgmap v1024: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:57.315 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:57 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741}]': finished 2024-08-17T17:29:57.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:57 smithi152 ceph-mon[93456]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:29:57.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:57.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:57.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:57.343 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:57.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:57.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:57.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:57.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:57.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[106729]: pgmap v1024: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741}]': finished 2024-08-17T17:29:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[106729]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:29:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[103614]: pgmap v1024: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2024-08-17T17:29:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.942333506989741}]': finished 2024-08-17T17:29:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[103614]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:29:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:29:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:57.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:57.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:57.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:57.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:57.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:57.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:58.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:58.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:58.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:58.183 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:58.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:58.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:58.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:58.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:58.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:58.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:58.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:58.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:58.668 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:58.690 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:58.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:58.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:58.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:58.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:59.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:59.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:59.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:59.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:59.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:59.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:59.377 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:59.479 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:59.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:29:59 smithi152 ceph-mon[93456]: pgmap v1026: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:29:59.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:59.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:29:59 smithi142 ceph-mon[106729]: pgmap v1026: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:29:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:29:59 smithi142 ceph-mon[103614]: pgmap v1026: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:29:59.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:59.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:59.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:29:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:59.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:59.915 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:59.915 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:59.927 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:29:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:00.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:00.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:00.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:00.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:00.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:00.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:00.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:00.354 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:00 smithi152 ceph-mon[93456]: overall HEALTH_OK 2024-08-17T17:30:00.355 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:30:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:30:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:00.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:00.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:00.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:00.527 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:00 smithi142 ceph-mon[103614]: overall HEALTH_OK 2024-08-17T17:30:00.528 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:00 smithi142 ceph-mon[106729]: overall HEALTH_OK 2024-08-17T17:30:00.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:00.608 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:00.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:00.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:00.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:30:00.847 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:00.904 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:00.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:01.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:01.055 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:01.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:01.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:01.247 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:01.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:01.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:01.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:01.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:01.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:01 smithi152 ceph-mon[93456]: pgmap v1027: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:30:01.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:01.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:01 smithi142 ceph-mon[103614]: pgmap v1027: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:30:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:01 smithi142 ceph-mon[106729]: pgmap v1027: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:30:01.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:01.687 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:01.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:01.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:01.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:01.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:02.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:02.032 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:30:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:30:01] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:30:02.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:02.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:02.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:02.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:02.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:02.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:02.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:02.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:02.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:02.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:02.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.895 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:02.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:03.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:03.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:03.166 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:03.166 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:03.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:03.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:03.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:03.336 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:30:03.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:03.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:03.498 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:03 smithi152 ceph-mon[93456]: pgmap v1028: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:30:03.499 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1427089742' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:03.499 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:03 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:03.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:03.590 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:03.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:03 smithi142 ceph-mon[106729]: pgmap v1028: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:30:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1427089742' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:03.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:03 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:03 smithi142 ceph-mon[103614]: pgmap v1028: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:30:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1427089742' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:03.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:03 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:03.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:03.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:03.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:03.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:03.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:30:03.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:04.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:04.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:04.130 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:04.164 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:04.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.299 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:04.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:04.340 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:04.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:04.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:04.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:04.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:04.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:04 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:04.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:04.543 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:04 smithi142 ceph-mon[106729]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:30:04.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:04 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:04.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:04.543 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:04 smithi142 ceph-mon[103614]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:30:04.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:04.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:04.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:04.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:04 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:04 smithi152 ceph-mon[93456]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:30:04.848 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:04.950 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:05.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:05.096 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:30:05.096 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:05.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:05.134 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:30:05.134 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:05.167 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:30:05.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:30:05.167 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:30:05.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:30:05.168 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:30:05.169 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:05.211 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:30:05.211 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:05.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:05.243 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:30:05.243 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:05.257 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:05.279 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:30:05.279 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:05.312 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:30:05.313 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:30:05.342 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:30:05.343 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:05.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:05.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:05.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:05.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:05 smithi142 ceph-mon[106729]: pgmap v1030: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:30:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1496113337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:05 smithi142 ceph-mon[103614]: pgmap v1030: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:30:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1496113337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:05.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:05.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:05.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:05.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:05.785 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:05 smithi152 ceph-mon[93456]: pgmap v1030: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T17:30:05.785 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1496113337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:05.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:05.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:05.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:05.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:06.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:06.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:06.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:06.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:30:06.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:06.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:06.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.416 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:06.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:06.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:06.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:06.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:06.639 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:06.643 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:06.692 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:06.736 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:06 smithi152 ceph-mon[93456]: pgmap v1031: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:30:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:06.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:06.791 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:06 smithi142 ceph-mon[103614]: pgmap v1031: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:30:06.791 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:06 smithi142 ceph-mon[106729]: pgmap v1031: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:30:06.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:06.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:06.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.019 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:07.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:07.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:07.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:07.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:07.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:07.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:07.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.526 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.559 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.670 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:30:07.670 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:30:07.670 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:30:07.670 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:30:07.670 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:30:07.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:07.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:07.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:07.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:08.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:08.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:08.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:08.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:08.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:08.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:08.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:08.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:08.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:30:08.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:08.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:08.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:08.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:08.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:08.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:08.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:08.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:08.916 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:09.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:09.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:09.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:09.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:09.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:09 smithi152 ceph-mon[93456]: pgmap v1032: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2024-08-17T17:30:09.416 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:09.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:09.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:09.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:09 smithi142 ceph-mon[106729]: pgmap v1032: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2024-08-17T17:30:09.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:09 smithi142 ceph-mon[103614]: pgmap v1032: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2024-08-17T17:30:09.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:09.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:09.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:09.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:10.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:10.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:10.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:10.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:10.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:10.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:10.370 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:10.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:10.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:10.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:10.536 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:30:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:30:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:10.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:10.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:10.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:10.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:10.816 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:10.845 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:30:10.845 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:30:10.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:30:10.845 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36257 10.b 10:d2639b69:::benchmark_data_smithi142_192453_object36256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.708290+0000", 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.13442293000003, 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34889161800000001, 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36257 2024-08-17T17:30:10.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708290+0000", 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708290+0000", 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708289+0000", 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708411+0000", 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121872 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708426+0000", 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4791e-05 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708431+0000", 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6729999999999997e-06 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708475+0000", 2024-08-17T17:30:10.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3872999999999998e-05 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708607+0000", 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013139200000000001 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.708775+0000", 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000168395 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.710678+0000", 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001902429 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.713887+0000", 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032090140000000001 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.057097+0000", 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34321046199999999 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.057142+0000", 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5195999999999999e-05 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.057182+0000", 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9473e-05 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.852 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36269 10.b 10:d55284ef:::benchmark_data_smithi142_192453_object36268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.714949+0000", 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.12776377599999, 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34567767500000002, 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.853 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36269 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.714949+0000", 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.714949+0000", 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.714948+0000", 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.714990+0000", 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1795999999999998e-05 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715006+0000", 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5858e-05 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715012+0000", 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3629999999999999e-06 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715050+0000", 2024-08-17T17:30:10.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7487000000000002e-05 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715226+0000", 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017670000000000001 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715385+0000", 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015889699999999999 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.718725+0000", 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003340127 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721059+0000", 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023336860000000002 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.060547+0000", 2024-08-17T17:30:10.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33948799400000002 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.060595+0000", 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8321e-05 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.060627+0000", 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1393000000000003e-05 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.859 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36270 10.b 10:de0ed546:::benchmark_data_smithi142_192453_object36269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.715180+0000", 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.12753325099999, 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.349202664, 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36270 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715180+0000", 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715180+0000", 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715179+0000", 2024-08-17T17:30:10.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715213+0000", 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3636000000000001e-05 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715222+0000", 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4380000000000001e-06 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715228+0000", 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3909999999999997e-06 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.717376+0000", 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021487670000000002 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.717499+0000", 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012231500000000001 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.717641+0000", 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014229999999999999 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.718877+0000", 2024-08-17T17:30:10.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012362429999999999 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721120+0000", 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022422560000000002 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064249+0000", 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34312944000000001 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064303+0000", 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4070999999999998e-05 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064382+0000", 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9388000000000006e-05 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36271 10.0 10:0312d0df:::benchmark_data_smithi142_192453_object36270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.715254+0000", 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.12745941200001, 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34336528700000002, 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36271 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715254+0000", 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715254+0000", 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715253+0000", 2024-08-17T17:30:10.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715283+0000", 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0187000000000001e-05 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715369+0000", 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5864000000000003e-05 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715374+0000", 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4790000000000004e-06 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715511+0000", 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013656399999999999 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715613+0000", 2024-08-17T17:30:10.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010164 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.715722+0000", 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109491 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.717912+0000", 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021901400000000001 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.717926+0000", 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3997e-05 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.058521+0000", 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34059506499999997 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.058578+0000", 2024-08-17T17:30:10.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6323999999999998e-05 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.058619+0000", 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1174000000000003e-05 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36283 10.b 10:db36fe59:::benchmark_data_smithi142_192453_object36282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.874 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.721205+0000", 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.121507506, 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34327511799999999, 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36283 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721205+0000", 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721205+0000", 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721205+0000", 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721258+0000", 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2911000000000002e-05 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721271+0000", 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3035e-05 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721276+0000", 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3040000000000004e-06 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721333+0000", 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6950000000000002e-05 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721434+0000", 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010075 2024-08-17T17:30:10.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.721546+0000", 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112173 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723074+0000", 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001528574 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727209+0000", 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041343999999999999 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064420+0000", 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33721088300000002 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064453+0000", 2024-08-17T17:30:10.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3309999999999998e-05 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064481+0000", 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7623e-05 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36287 10.b 10:db7f0104:::benchmark_data_smithi142_192453_object36286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.723891+0000", 2024-08-17T17:30:10.881 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.11882203699997, 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34075685900000002, 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36287 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723891+0000", 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723891+0000", 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723890+0000", 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:10.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723972+0000", 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2563000000000002e-05 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723986+0000", 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3543999999999999e-05 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.723991+0000", 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9810000000000003e-06 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.724028+0000", 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7413999999999998e-05 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.724140+0000", 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011215 2024-08-17T17:30:10.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.724277+0000", 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000136868 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727216+0000", 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029387910000000001 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727258+0000", 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1990000000000003e-05 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064584+0000", 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33732631600000001 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064620+0000", 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5615999999999999e-05 2024-08-17T17:30:10.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064648+0000", 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8070000000000001e-05 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36292 10.b 10:d6ae8c9b:::benchmark_data_smithi142_192453_object36291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.725054+0000", 2024-08-17T17:30:10.888 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.117659382, 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33988036999999999, 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36292 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725054+0000", 2024-08-17T17:30:10.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725054+0000", 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725053+0000", 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725103+0000", 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0294000000000003e-05 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725112+0000", 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1290000000000002e-06 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725117+0000", 2024-08-17T17:30:10.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6580000000000001e-06 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725691+0000", 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00057405800000000001 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725798+0000", 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107302 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.725928+0000", 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000129574 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727320+0000", 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013917910000000001 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727774+0000", 2024-08-17T17:30:10.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045397100000000001 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064842+0000", 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33706801199999997 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064900+0000", 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8841000000000001e-05 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.064934+0000", 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3575000000000003e-05 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36296 10.b 10:dae3dcc3:::benchmark_data_smithi142_192453_object36295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.727660+0000", 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.115053198, 2024-08-17T17:30:10.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.337728325, 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36296 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727660+0000", 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727660+0000", 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727659+0000", 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727722+0000", 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3460999999999994e-05 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727731+0000", 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9779999999999994e-06 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727735+0000", 2024-08-17T17:30:10.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2479999999999998e-06 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727792+0000", 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6910999999999998e-05 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727880+0000", 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7834e-05 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727976+0000", 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5415999999999996e-05 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.729667+0000", 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001691112 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.731583+0000", 2024-08-17T17:30:10.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019162999999999999 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.065317+0000", 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.333734207 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.065358+0000", 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1310000000000003e-05 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.065388+0000", 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9689e-05 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36298 10.b 10:dc50d990:::benchmark_data_smithi142_192453_object36297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.727861+0000", 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.11485241299999, 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33762239799999999, 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36298 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727861+0000", 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727861+0000", 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727860+0000", 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727928+0000", 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7704000000000001e-05 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727934+0000", 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5150000000000003e-06 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.727940+0000", 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7520000000000002e-06 2024-08-17T17:30:10.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.728893+0000", 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00095324299999999999 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.728974+0000", 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1124999999999993e-05 2024-08-17T17:30:10.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:30:10.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.729056+0000", 2024-08-17T17:30:10.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1860000000000006e-05 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.729792+0000", 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00073588400000000002 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.731690+0000", 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001898521 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.065419+0000", 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33372874400000002 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.065454+0000", 2024-08-17T17:30:10.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4691999999999999e-05 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.065483+0000", 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9119e-05 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:36303 10.0 10:0d4491ad:::benchmark_data_smithi142_192453_object36302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2024-08-17T17:30:10.910 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:22:43.733063+0000", 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 447.10964975600001, 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.326809456, 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 36303 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733063+0000", 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733063+0000", 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733063+0000", 2024-08-17T17:30:10.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733130+0000", 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7476000000000002e-05 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733140+0000", 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8120000000000002e-06 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733145+0000", 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8199999999999996e-06 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733183+0000", 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8319999999999999e-05 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733265+0000", 2024-08-17T17:30:10.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1882e-05 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.733357+0000", 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2410999999999993e-05 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.734508+0000", 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001150388 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:43.734706+0000", 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019789300000000001 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.059803+0000", 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32509719399999998 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.059833+0000", 2024-08-17T17:30:10.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.057e-05 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:22:44.059873+0000", 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9347999999999997e-05 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:144823 10.a 10:5df35cf4:::benchmark_data_smithi142_192453_object19075:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:35.863934+0000", 2024-08-17T17:30:10.917 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 334.97877943100002, 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.254725527, 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144823 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.863934+0000", 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.863934+0000", 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.863932+0000", 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.863939+0000", 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6389999999999992e-06 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.863959+0000", 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9845999999999999e-05 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.863967+0000", 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4109999999999999e-06 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864019+0000", 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2627999999999998e-05 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864185+0000", 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000165415 2024-08-17T17:30:10.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864351+0000", 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000166207 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.892904+0000", 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028553255999999999 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.892912+0000", 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4950000000000002e-06 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.107970+0000", 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2150578240000001 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.108048+0000", 2024-08-17T17:30:10.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8363999999999997e-05 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.118615+0000", 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010566921 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.118659+0000", 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4299999999999999e-05 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.924 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:144824 10.a 10:5e6aec16:::benchmark_data_smithi142_192453_object19076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:35.864399+0000", 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 334.97831362900001, 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2637438109999999, 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.925 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144824 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864399+0000", 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864399+0000", 2024-08-17T17:30:10.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864399+0000", 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864402+0000", 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.596e-06 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864410+0000", 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9410000000000007e-06 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.864415+0000", 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0420000000000002e-06 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.869195+0000", 2024-08-17T17:30:10.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047799590000000003 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.869353+0000", 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015727699999999999 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.869503+0000", 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015087799999999999 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.892927+0000", 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023423385000000001 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.892931+0000", 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.721e-06 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.108243+0000", 2024-08-17T17:30:10.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.215312309 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.108327+0000", 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3761e-05 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.128090+0000", 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019763228000000001 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.128143+0000", 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3347999999999998e-05 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179022 10.17 10:ee302f00:::benchmark_data_smithi142_192453_object53274:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.751385+0000", 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 289.09132763000002, 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.070123556, 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.932 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179022 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.751385+0000", 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.751385+0000", 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.751384+0000", 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.751390+0000", 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2439999999999998e-06 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.751407+0000", 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6932999999999999e-05 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.751413+0000", 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9329999999999998e-06 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974468+0000", 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22305472000000001 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974475+0000", 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2060000000000001e-06 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.821310+0000", 2024-08-17T17:30:10.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84683540000000002 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.821509+0000", 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000198481 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179034 10.0 10:00ef1ee1:::benchmark_data_smithi142_192453_object53286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:30:10.937 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.974507+0000", 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 288.86820583600002, 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96553735600000001, 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179034 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974507+0000", 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974507+0000", 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974505+0000", 2024-08-17T17:30:10.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974517+0000", 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.145e-05 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974534+0000", 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6986e-05 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.974544+0000", 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0437e-05 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.986930+0000", 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012385854 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:30:10.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.986944+0000", 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3922e-05 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.929164+0000", 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94221997599999996 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.929250+0000", 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5916999999999997e-05 2024-08-17T17:30:10.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.929577+0000", 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032773399999999999 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.935847+0000", 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062697120000000002 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.935898+0000", 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0543e-05 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.939958+0000", 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040603460000000003 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.940004+0000", 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5747000000000001e-05 2024-08-17T17:30:10.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.940045+0000", 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0824000000000001e-05 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:234142 10.d 10:b411e20a:::benchmark_data_smithi142_192453_object108394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:28.258622+0000", 2024-08-17T17:30:10.945 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 222.584090514, 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32222192300000002, 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 234142 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.946 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.258622+0000", 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.258622+0000", 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.258622+0000", 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.258625+0000", 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2849999999999999e-06 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.258631+0000", 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.237e-06 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.258634+0000", 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3909999999999998e-06 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.263254+0000", 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046198309999999996 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.263367+0000", 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113479 2024-08-17T17:30:10.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.263481+0000", 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113719 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.575803+0000", 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.312321821 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580526+0000", 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047235580000000001 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580595+0000", 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8318999999999994e-05 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580656+0000", 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1611000000000004e-05 2024-08-17T17:30:10.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580844+0000", 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018806500000000001 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:234149 10.a 10:55ed54be:::benchmark_data_smithi142_192453_object108401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:28.259015+0000", 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 222.58369776999999, 2024-08-17T17:30:10.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32146035899999997, 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 234149 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259015+0000", 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259015+0000", 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259014+0000", 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259019+0000", 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1630000000000004e-06 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259028+0000", 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0089999999999996e-06 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259033+0000", 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8620000000000002e-06 2024-08-17T17:30:10.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.267027+0000", 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079937370000000008 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.267110+0000", 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2721999999999998e-05 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.267217+0000", 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010712100000000001 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.576124+0000", 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30890738899999998 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.576155+0000", 2024-08-17T17:30:10.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0199999999999999e-05 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.576252+0000", 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7273000000000002e-05 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.576285+0000", 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3281999999999997e-05 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580476+0000", 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041905220000000003 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:234151 10.d 10:b1dfc6fc:::benchmark_data_smithi142_192453_object108403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:28.259141+0000", 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 222.58357242299999, 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32195168200000002, 2024-08-17T17:30:10.959 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 234151 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259141+0000", 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259141+0000", 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259140+0000", 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259144+0000", 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9809999999999997e-06 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259152+0000", 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1999999999999994e-06 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.259156+0000", 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2710000000000003e-06 2024-08-17T17:30:10.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.267424+0000", 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082673469999999995 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.267547+0000", 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123602 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:30:10.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.267677+0000", 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012935099999999999 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.575810+0000", 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30813375799999998 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580563+0000", 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004752437 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580925+0000", 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036255800000000002 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580968+0000", 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2936999999999999e-05 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.581092+0000", 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123831 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:234158 10.d 10:b988e0ef:::benchmark_data_smithi142_192453_object108410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:26:28.271594+0000", 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 222.57111879499999, 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32490969800000002, 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.966 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 234158 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.271594+0000", 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.271594+0000", 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.271593+0000", 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.271598+0000", 2024-08-17T17:30:10.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3419999999999998e-06 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.271608+0000", 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5259999999999999e-06 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.271612+0000", 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4830000000000001e-06 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.575840+0000", 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30422816800000002 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.575998+0000", 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000157696 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:30:10.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.576149+0000", 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015051299999999999 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.580889+0000", 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047406460000000003 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.591497+0000", 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010607472999999999 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.596231+0000", 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047342410000000001 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.596314+0000", 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3529999999999995e-05 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:26:28.596504+0000", 2024-08-17T17:30:10.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018946999999999999 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:162104 11.3 11:c2291d75:::benchmark_data_smithi142_227468_object38685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:29:29.493668+0000", 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 41.349044663999997, 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.317932406, 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:10.973 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 162104 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493668+0000", 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493668+0000", 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493667+0000", 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493676+0000", 2024-08-17T17:30:10.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8259999999999999e-06 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493687+0000", 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1032e-05 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493691+0000", 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.454e-06 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493729+0000", 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7824999999999998e-05 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493865+0000", 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013523699999999999 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493986+0000", 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121933 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.502329+0000", 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083420590000000006 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.502339+0000", 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0956e-05 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.811458+0000", 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911825700000001 2024-08-17T17:30:10.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.811561+0000", 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103016 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.811601+0000", 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9977e-05 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.979 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:162106 11.3 11:cd377b44:::benchmark_data_smithi142_227468_object38687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 41.349013962999997, 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32571883000000001, 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 162106 2024-08-17T17:30:10.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493700+0000", 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7239999999999999e-06 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:10.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493705+0000", 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2899999999999996e-06 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493707+0000", 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1909999999999999e-06 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.498299+0000", 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045918419999999996 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.498402+0000", 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010297399999999999 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.498504+0000", 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102322 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.502521+0000", 2024-08-17T17:30:10.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040166159999999998 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.502874+0000", 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000353467 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.819325+0000", 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31645051899999999 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.819379+0000", 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4036999999999998e-05 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.819418+0000", 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9140000000000001e-05 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.986 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:10.987 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:10.987 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:30:10.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:11.015 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:11.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:11.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:11.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:11.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:11.291 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:11.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:11.393 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.422 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:11.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:11.464 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:11 smithi142 ceph-mon[106729]: pgmap v1033: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:30:11.464 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:11 smithi142 ceph-mon[103614]: pgmap v1033: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:30:11.474 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:30:11.482 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:30:11.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:11.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:11.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:11 smithi152 ceph-mon[93456]: pgmap v1033: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-17T17:30:11.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:11.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:11.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:11.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:11.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:30:11.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:11.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:11.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:12.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:12.056 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:30:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:30:11] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:30:12.101 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:12.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:12.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:12.243 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:12.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:12.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:12.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:12.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:12.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:12.447 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:30:12.527 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1033,"stamp":"2024-08-17T17:30:10.105256+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3526039525,"num_objects":53821,"num_object_clones":0,"num_object_copies":161463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53821,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":193317,"num_write_kb":7901072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2362634,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138346,"ondisk_log_size":138346,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16126036,"kb_used_data":10491216,"kb_used_omap":45,"kb_used_meta":5634450,"kb_avail":733835180,"statfs":{"total":767960285184,"available":751447224320,"internally_reserved":0,"allocated":10743005184,"data_stored":10730520703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47063,"internal_metadata":5769676841},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,6,5,11,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-527761408,"num_objects":-8053,"num_object_clones":0,"num_object_copies":-24159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8053,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005529"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":434,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336561+0000","last_change":"2024-08-17T17:26:32.212092+0000","last_active":"2024-08-17T17:30:03.336561+0000","last_peered":"2024-08-17T17:30:03.336561+0000","last_clean":"2024-08-17T17:30:03.336561+0000","last_became_active":"2024-08-17T17:26:32.211774+0000","last_became_peered":"2024-08-17T17:26:32.211774+0000","last_unstale":"2024-08-17T17:30:03.336561+0000","last_undegraded":"2024-08-17T17:30:03.336561+0000","last_fullsized":"2024-08-17T17:30:03.336561+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:45.518565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":250,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.306635+0000","last_change":"2024-08-17T17:26:32.210903+0000","last_active":"2024-08-17T17:29:56.306635+0000","last_peered":"2024-08-17T17:29:56.306635+0000","last_clean":"2024-08-17T17:29:56.306635+0000","last_became_active":"2024-08-17T17:26:32.209823+0000","last_became_peered":"2024-08-17T17:26:32.209823+0000","last_unstale":"2024-08-17T17:29:56.306635+0000","last_undegraded":"2024-08-17T17:29:56.306635+0000","last_fullsized":"2024-08-17T17:29:56.306635+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:54:17.717352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":198,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324655+0000","last_change":"2024-08-17T17:26:32.205305+0000","last_active":"2024-08-17T17:29:56.324655+0000","last_peered":"2024-08-17T17:29:56.324655+0000","last_clean":"2024-08-17T17:29:56.324655+0000","last_became_active":"2024-08-17T17:26:32.205000+0000","last_became_peered":"2024-08-17T17:26:32.205000+0000","last_unstale":"2024-08-17T17:29:56.324655+0000","last_undegraded":"2024-08-17T17:29:56.324655+0000","last_fullsized":"2024-08-17T17:29:56.324655+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:31.144881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":233,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.306771+0000","last_change":"2024-08-17T17:26:32.213319+0000","last_active":"2024-08-17T17:29:56.306771+0000","last_peered":"2024-08-17T17:29:56.306771+0000","last_clean":"2024-08-17T17:29:56.306771+0000","last_became_active":"2024-08-17T17:26:32.213079+0000","last_became_peered":"2024-08-17T17:26:32.213079+0000","last_unstale":"2024-08-17T17:29:56.306771+0000","last_undegraded":"2024-08-17T17:29:56.306771+0000","last_fullsized":"2024-08-17T17:29:56.306771+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:43.552406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":316,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336973+0000","last_change":"2024-08-17T17:26:32.211539+0000","last_active":"2024-08-17T17:30:03.336973+0000","last_peered":"2024-08-17T17:30:03.336973+0000","last_clean":"2024-08-17T17:30:03.336973+0000","last_became_active":"2024-08-17T17:26:32.211267+0000","last_became_peered":"2024-08-17T17:26:32.211267+0000","last_unstale":"2024-08-17T17:30:03.336973+0000","last_undegraded":"2024-08-17T17:30:03.336973+0000","last_fullsized":"2024-08-17T17:30:03.336973+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:16.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":219,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.306629+0000","last_change":"2024-08-17T17:26:32.233887+0000","last_active":"2024-08-17T17:29:56.306629+0000","last_peered":"2024-08-17T17:29:56.306629+0000","last_clean":"2024-08-17T17:29:56.306629+0000","last_became_active":"2024-08-17T17:26:32.230472+0000","last_became_peered":"2024-08-17T17:26:32.230472+0000","last_unstale":"2024-08-17T17:29:56.306629+0000","last_undegraded":"2024-08-17T17:29:56.306629+0000","last_fullsized":"2024-08-17T17:29:56.306629+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:09.045150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":410,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.337317+0000","last_change":"2024-08-17T17:26:32.207139+0000","last_active":"2024-08-17T17:30:03.337317+0000","last_peered":"2024-08-17T17:30:03.337317+0000","last_clean":"2024-08-17T17:30:03.337317+0000","last_became_active":"2024-08-17T17:26:32.206763+0000","last_became_peered":"2024-08-17T17:26:32.206763+0000","last_unstale":"2024-08-17T17:30:03.337317+0000","last_undegraded":"2024-08-17T17:30:03.337317+0000","last_fullsized":"2024-08-17T17:30:03.337317+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:51.396647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":325,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.309360+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:29:56.309360+0000","last_peered":"2024-08-17T17:29:56.309360+0000","last_clean":"2024-08-17T17:29:56.309360+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:29:56.309360+0000","last_undegraded":"2024-08-17T17:29:56.309360+0000","last_fullsized":"2024-08-17T17:29:56.309360+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"305'12341","reported_seq":12369,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:10.062299+0000","last_change":"2024-08-17T17:27:05.299771+0000","last_active":"2024-08-17T17:30:10.062299+0000","last_peered":"2024-08-17T17:30:10.062299+0000","last_clean":"2024-08-17T17:30:10.062299+0000","last_became_active":"2024-08-17T17:27:05.299171+0000","last_became_peered":"2024-08-17T17:27:05.299171+0000","last_unstale":"2024-08-17T17:30:10.062299+0000","last_undegraded":"2024-08-17T17:30:10.062299+0000","last_fullsized":"2024-08-17T17:30:10.062299+0000","mapping_epoch":283,"log_start":"287'2300","ondisk_log_start":"287'2300","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10041,"log_dups_size":0,"ondisk_log_size":10041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:25.524635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12341,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":281,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307077+0000","last_change":"2024-08-17T17:26:32.214168+0000","last_active":"2024-08-17T17:29:56.307077+0000","last_peered":"2024-08-17T17:29:56.307077+0000","last_clean":"2024-08-17T17:29:56.307077+0000","last_became_active":"2024-08-17T17:26:32.213844+0000","last_became_peered":"2024-08-17T17:26:32.213844+0000","last_unstale":"2024-08-17T17:29:56.307077+0000","last_undegraded":"2024-08-17T17:29:56.307077+0000","last_fullsized":"2024-08-17T17:29:56.307077+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:52.468566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324428+0000","last_change":"2024-08-17T17:26:32.218633+0000","last_active":"2024-08-17T17:29:56.324428+0000","last_peered":"2024-08-17T17:29:56.324428+0000","last_clean":"2024-08-17T17:29:56.324428+0000","last_became_active":"2024-08-17T17:26:32.218212+0000","last_became_peered":"2024-08-17T17:26:32.218212+0000","last_unstale":"2024-08-17T17:29:56.324428+0000","last_undegraded":"2024-08-17T17:29:56.324428+0000","last_fullsized":"2024-08-17T17:29:56.324428+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:55.971779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"305'12142","reported_seq":12198,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.027277+0000","last_change":"2024-08-17T17:27:05.818422+0000","last_active":"2024-08-17T17:30:09.027277+0000","last_peered":"2024-08-17T17:30:09.027277+0000","last_clean":"2024-08-17T17:30:09.027277+0000","last_became_active":"2024-08-17T17:27:05.818117+0000","last_became_peered":"2024-08-17T17:27:05.818117+0000","last_unstale":"2024-08-17T17:30:09.027277+0000","last_undegraded":"2024-08-17T17:30:09.027277+0000","last_fullsized":"2024-08-17T17:30:09.027277+0000","mapping_epoch":283,"log_start":"287'2100","ondisk_log_start":"287'2100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10042,"log_dups_size":0,"ondisk_log_size":10042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:36:36.236107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12142,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":236,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307346+0000","last_change":"2024-08-17T17:26:32.220489+0000","last_active":"2024-08-17T17:29:56.307346+0000","last_peered":"2024-08-17T17:29:56.307346+0000","last_clean":"2024-08-17T17:29:56.307346+0000","last_became_active":"2024-08-17T17:26:32.220150+0000","last_became_peered":"2024-08-17T17:26:32.220150+0000","last_unstale":"2024-08-17T17:29:56.307346+0000","last_undegraded":"2024-08-17T17:29:56.307346+0000","last_fullsized":"2024-08-17T17:29:56.307346+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:09.954487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":276,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325083+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:29:56.325083+0000","last_peered":"2024-08-17T17:29:56.325083+0000","last_clean":"2024-08-17T17:29:56.325083+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:29:56.325083+0000","last_undegraded":"2024-08-17T17:29:56.325083+0000","last_fullsized":"2024-08-17T17:29:56.325083+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"305'11933","reported_seq":11963,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:05.310799+0000","last_change":"2024-08-17T17:27:05.297172+0000","last_active":"2024-08-17T17:30:05.310799+0000","last_peered":"2024-08-17T17:30:05.310799+0000","last_clean":"2024-08-17T17:30:05.310799+0000","last_became_active":"2024-08-17T17:27:05.296899+0000","last_became_peered":"2024-08-17T17:27:05.296899+0000","last_unstale":"2024-08-17T17:30:05.310799+0000","last_undegraded":"2024-08-17T17:30:05.310799+0000","last_fullsized":"2024-08-17T17:30:05.310799+0000","mapping_epoch":283,"log_start":"289'3500","ondisk_log_start":"289'3500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8433,"log_dups_size":0,"ondisk_log_size":8433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:32.207215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231145472,"num_objects":3527,"num_object_clones":0,"num_object_copies":10581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11933,"num_write_kb":494720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":246,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307298+0000","last_change":"2024-08-17T17:26:32.216517+0000","last_active":"2024-08-17T17:29:56.307298+0000","last_peered":"2024-08-17T17:29:56.307298+0000","last_clean":"2024-08-17T17:29:56.307298+0000","last_became_active":"2024-08-17T17:26:32.216161+0000","last_became_peered":"2024-08-17T17:26:32.216161+0000","last_unstale":"2024-08-17T17:29:56.307298+0000","last_undegraded":"2024-08-17T17:29:56.307298+0000","last_fullsized":"2024-08-17T17:29:56.307298+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:39.712941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":276,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.308935+0000","last_change":"2024-08-17T17:26:32.217706+0000","last_active":"2024-08-17T17:29:56.308935+0000","last_peered":"2024-08-17T17:29:56.308935+0000","last_clean":"2024-08-17T17:29:56.308935+0000","last_became_active":"2024-08-17T17:26:32.217107+0000","last_became_peered":"2024-08-17T17:26:32.217107+0000","last_unstale":"2024-08-17T17:29:56.308935+0000","last_undegraded":"2024-08-17T17:29:56.308935+0000","last_fullsized":"2024-08-17T17:29:56.308935+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:39.827920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"305'11895","reported_seq":11924,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:05.296894+0000","last_change":"2024-08-17T17:27:05.297778+0000","last_active":"2024-08-17T17:30:05.296894+0000","last_peered":"2024-08-17T17:30:05.296894+0000","last_clean":"2024-08-17T17:30:05.296894+0000","last_became_active":"2024-08-17T17:27:05.297367+0000","last_became_peered":"2024-08-17T17:27:05.297367+0000","last_unstale":"2024-08-17T17:30:05.296894+0000","last_undegraded":"2024-08-17T17:30:05.296894+0000","last_fullsized":"2024-08-17T17:30:05.296894+0000","mapping_epoch":283,"log_start":"289'3500","ondisk_log_start":"289'3500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8395,"log_dups_size":0,"ondisk_log_size":8395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.484840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231800832,"num_objects":3537,"num_object_clones":0,"num_object_copies":10611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11895,"num_write_kb":493824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":360,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336689+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:30:03.336689+0000","last_peered":"2024-08-17T17:30:03.336689+0000","last_clean":"2024-08-17T17:30:03.336689+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:30:03.336689+0000","last_undegraded":"2024-08-17T17:30:03.336689+0000","last_fullsized":"2024-08-17T17:30:03.336689+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":126,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.348967+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:30:03.348967+0000","last_peered":"2024-08-17T17:30:03.348967+0000","last_clean":"2024-08-17T17:30:03.348967+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:30:03.348967+0000","last_undegraded":"2024-08-17T17:30:03.348967+0000","last_fullsized":"2024-08-17T17:30:03.348967+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"305'12151","reported_seq":12181,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.018601+0000","last_change":"2024-08-17T17:27:05.297382+0000","last_active":"2024-08-17T17:30:08.018601+0000","last_peered":"2024-08-17T17:30:08.018601+0000","last_clean":"2024-08-17T17:30:08.018601+0000","last_became_active":"2024-08-17T17:27:05.297149+0000","last_became_peered":"2024-08-17T17:27:05.297149+0000","last_unstale":"2024-08-17T17:30:08.018601+0000","last_undegraded":"2024-08-17T17:30:08.018601+0000","last_fullsized":"2024-08-17T17:30:08.018601+0000","mapping_epoch":283,"log_start":"289'3500","ondisk_log_start":"289'3500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8651,"log_dups_size":0,"ondisk_log_size":8651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:25.034122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219873280,"num_objects":3355,"num_object_clones":0,"num_object_copies":10065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12151,"num_write_kb":496192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":449,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336586+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:30:03.336586+0000","last_peered":"2024-08-17T17:30:03.336586+0000","last_clean":"2024-08-17T17:30:03.336586+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:30:03.336586+0000","last_undegraded":"2024-08-17T17:30:03.336586+0000","last_fullsized":"2024-08-17T17:30:03.336586+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325516+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:29:56.325516+0000","last_peered":"2024-08-17T17:29:56.325516+0000","last_clean":"2024-08-17T17:29:56.325516+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:29:56.325516+0000","last_undegraded":"2024-08-17T17:29:56.325516+0000","last_fullsized":"2024-08-17T17:29:56.325516+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"305'12085","reported_seq":12141,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.992007+0000","last_change":"2024-08-17T17:27:05.818538+0000","last_active":"2024-08-17T17:30:08.992007+0000","last_peered":"2024-08-17T17:30:08.992007+0000","last_clean":"2024-08-17T17:30:08.992007+0000","last_became_active":"2024-08-17T17:27:05.818278+0000","last_became_peered":"2024-08-17T17:27:05.818278+0000","last_unstale":"2024-08-17T17:30:08.992007+0000","last_undegraded":"2024-08-17T17:30:08.992007+0000","last_fullsized":"2024-08-17T17:30:08.992007+0000","mapping_epoch":283,"log_start":"287'2000","ondisk_log_start":"287'2000","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10085,"log_dups_size":0,"ondisk_log_size":10085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:06.057185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216465408,"num_objects":3303,"num_object_clones":0,"num_object_copies":9909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12085,"num_write_kb":492416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":260,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.309211+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:29:56.309211+0000","last_peered":"2024-08-17T17:29:56.309211+0000","last_clean":"2024-08-17T17:29:56.309211+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:29:56.309211+0000","last_undegraded":"2024-08-17T17:29:56.309211+0000","last_fullsized":"2024-08-17T17:29:56.309211+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":276,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.327908+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:29:56.327908+0000","last_peered":"2024-08-17T17:29:56.327908+0000","last_clean":"2024-08-17T17:29:56.327908+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:29:56.327908+0000","last_undegraded":"2024-08-17T17:29:56.327908+0000","last_fullsized":"2024-08-17T17:29:56.327908+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"305'12113","reported_seq":12143,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.010258+0000","last_change":"2024-08-17T17:27:05.818745+0000","last_active":"2024-08-17T17:30:08.010258+0000","last_peered":"2024-08-17T17:30:08.010258+0000","last_clean":"2024-08-17T17:30:08.010258+0000","last_became_active":"2024-08-17T17:27:05.818533+0000","last_became_peered":"2024-08-17T17:27:05.818533+0000","last_unstale":"2024-08-17T17:30:08.010258+0000","last_undegraded":"2024-08-17T17:30:08.010258+0000","last_fullsized":"2024-08-17T17:30:08.010258+0000","mapping_epoch":283,"log_start":"289'3500","ondisk_log_start":"289'3500","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8613,"log_dups_size":0,"ondisk_log_size":8613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:41.413190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12113,"num_write_kb":494144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":277,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325176+0000","last_change":"2024-08-17T17:26:32.216646+0000","last_active":"2024-08-17T17:29:56.325176+0000","last_peered":"2024-08-17T17:29:56.325176+0000","last_clean":"2024-08-17T17:29:56.325176+0000","last_became_active":"2024-08-17T17:26:32.216155+0000","last_became_peered":"2024-08-17T17:26:32.216155+0000","last_unstale":"2024-08-17T17:29:56.325176+0000","last_undegraded":"2024-08-17T17:29:56.325176+0000","last_fullsized":"2024-08-17T17:29:56.325176+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:55.449331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325656+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:29:56.325656+0000","last_peered":"2024-08-17T17:29:56.325656+0000","last_clean":"2024-08-17T17:29:56.325656+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:29:56.325656+0000","last_undegraded":"2024-08-17T17:29:56.325656+0000","last_fullsized":"2024-08-17T17:29:56.325656+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"305'12048","reported_seq":12104,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.730298+0000","last_change":"2024-08-17T17:27:05.297991+0000","last_active":"2024-08-17T17:30:09.730298+0000","last_peered":"2024-08-17T17:30:09.730298+0000","last_clean":"2024-08-17T17:30:09.730298+0000","last_became_active":"2024-08-17T17:27:05.297666+0000","last_became_peered":"2024-08-17T17:27:05.297666+0000","last_unstale":"2024-08-17T17:30:09.730298+0000","last_undegraded":"2024-08-17T17:30:09.730298+0000","last_fullsized":"2024-08-17T17:30:09.730298+0000","mapping_epoch":283,"log_start":"289'4100","ondisk_log_start":"289'4100","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7948,"log_dups_size":0,"ondisk_log_size":7948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:35.344484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219152384,"num_objects":3344,"num_object_clones":0,"num_object_copies":10032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12048,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":412,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.340278+0000","last_change":"2024-08-17T17:26:32.212135+0000","last_active":"2024-08-17T17:30:03.340278+0000","last_peered":"2024-08-17T17:30:03.340278+0000","last_clean":"2024-08-17T17:30:03.340278+0000","last_became_active":"2024-08-17T17:26:32.211604+0000","last_became_peered":"2024-08-17T17:26:32.211604+0000","last_unstale":"2024-08-17T17:30:03.340278+0000","last_undegraded":"2024-08-17T17:30:03.340278+0000","last_fullsized":"2024-08-17T17:30:03.340278+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:49.886809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":356,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.337394+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:30:03.337394+0000","last_peered":"2024-08-17T17:30:03.337394+0000","last_clean":"2024-08-17T17:30:03.337394+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:30:03.337394+0000","last_undegraded":"2024-08-17T17:30:03.337394+0000","last_fullsized":"2024-08-17T17:30:03.337394+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.a","version":"305'12116","reported_seq":12172,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.825908+0000","last_change":"2024-08-17T17:27:05.297901+0000","last_active":"2024-08-17T17:30:09.825908+0000","last_peered":"2024-08-17T17:30:09.825908+0000","last_clean":"2024-08-17T17:30:09.825908+0000","last_became_active":"2024-08-17T17:27:05.297625+0000","last_became_peered":"2024-08-17T17:27:05.297625+0000","last_unstale":"2024-08-17T17:30:09.825908+0000","last_undegraded":"2024-08-17T17:30:09.825908+0000","last_fullsized":"2024-08-17T17:30:09.825908+0000","mapping_epoch":283,"log_start":"289'4200","ondisk_log_start":"289'4200","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7916,"log_dups_size":0,"ondisk_log_size":7916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:27:29.568335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12116,"num_write_kb":491584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":627,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.311032+0000","last_change":"2024-08-17T17:26:33.204740+0000","last_active":"2024-08-17T17:29:56.311032+0000","last_peered":"2024-08-17T17:29:56.311032+0000","last_clean":"2024-08-17T17:29:56.311032+0000","last_became_active":"2024-08-17T17:26:33.204340+0000","last_became_peered":"2024-08-17T17:26:33.204340+0000","last_unstale":"2024-08-17T17:29:56.311032+0000","last_undegraded":"2024-08-17T17:29:56.311032+0000","last_fullsized":"2024-08-17T17:29:56.311032+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:43.802253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":212,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324508+0000","last_change":"2024-08-17T17:26:32.219571+0000","last_active":"2024-08-17T17:29:56.324508+0000","last_peered":"2024-08-17T17:29:56.324508+0000","last_clean":"2024-08-17T17:29:56.324508+0000","last_became_active":"2024-08-17T17:26:32.219094+0000","last_became_peered":"2024-08-17T17:26:32.219094+0000","last_unstale":"2024-08-17T17:29:56.324508+0000","last_undegraded":"2024-08-17T17:29:56.324508+0000","last_fullsized":"2024-08-17T17:29:56.324508+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:09.450772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":398,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336987+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:30:03.336987+0000","last_peered":"2024-08-17T17:30:03.336987+0000","last_clean":"2024-08-17T17:30:03.336987+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:30:03.336987+0000","last_undegraded":"2024-08-17T17:30:03.336987+0000","last_fullsized":"2024-08-17T17:30:03.336987+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.310284+0000","last_change":"2024-08-17T17:26:32.215221+0000","last_active":"2024-08-17T17:29:56.310284+0000","last_peered":"2024-08-17T17:29:56.310284+0000","last_clean":"2024-08-17T17:29:56.310284+0000","last_became_active":"2024-08-17T17:26:32.214756+0000","last_became_peered":"2024-08-17T17:26:32.214756+0000","last_unstale":"2024-08-17T17:29:56.310284+0000","last_undegraded":"2024-08-17T17:29:56.310284+0000","last_fullsized":"2024-08-17T17:29:56.310284+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.545194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"305'12365","reported_seq":12420,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.801037+0000","last_change":"2024-08-17T17:27:05.819104+0000","last_active":"2024-08-17T17:30:09.801037+0000","last_peered":"2024-08-17T17:30:09.801037+0000","last_clean":"2024-08-17T17:30:09.801037+0000","last_became_active":"2024-08-17T17:27:05.818813+0000","last_became_peered":"2024-08-17T17:27:05.818813+0000","last_unstale":"2024-08-17T17:30:09.801037+0000","last_undegraded":"2024-08-17T17:30:09.801037+0000","last_fullsized":"2024-08-17T17:30:09.801037+0000","mapping_epoch":283,"log_start":"289'4400","ondisk_log_start":"289'4400","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:35:28.659641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218693632,"num_objects":3337,"num_object_clones":0,"num_object_copies":10011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12365,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":301,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.311096+0000","last_change":"2024-08-17T17:26:32.212907+0000","last_active":"2024-08-17T17:29:56.311096+0000","last_peered":"2024-08-17T17:29:56.311096+0000","last_clean":"2024-08-17T17:29:56.311096+0000","last_became_active":"2024-08-17T17:26:32.212549+0000","last_became_peered":"2024-08-17T17:26:32.212549+0000","last_unstale":"2024-08-17T17:29:56.311096+0000","last_undegraded":"2024-08-17T17:29:56.311096+0000","last_fullsized":"2024-08-17T17:29:56.311096+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:35.071333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":241,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325007+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:29:56.325007+0000","last_peered":"2024-08-17T17:29:56.325007+0000","last_clean":"2024-08-17T17:29:56.325007+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:29:56.325007+0000","last_undegraded":"2024-08-17T17:29:56.325007+0000","last_fullsized":"2024-08-17T17:29:56.325007+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":126,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.349046+0000","last_change":"2024-08-17T17:24:37.073148+0000","last_active":"2024-08-17T17:30:03.349046+0000","last_peered":"2024-08-17T17:30:03.349046+0000","last_clean":"2024-08-17T17:30:03.349046+0000","last_became_active":"2024-08-17T17:24:37.072372+0000","last_became_peered":"2024-08-17T17:24:37.072372+0000","last_unstale":"2024-08-17T17:30:03.349046+0000","last_undegraded":"2024-08-17T17:30:03.349046+0000","last_fullsized":"2024-08-17T17:30:03.349046+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.659683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"305'11946","reported_seq":12002,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.833754+0000","last_change":"2024-08-17T17:27:05.298546+0000","last_active":"2024-08-17T17:30:09.833754+0000","last_peered":"2024-08-17T17:30:09.833754+0000","last_clean":"2024-08-17T17:30:09.833754+0000","last_became_active":"2024-08-17T17:27:05.297541+0000","last_became_peered":"2024-08-17T17:27:05.297541+0000","last_unstale":"2024-08-17T17:30:09.833754+0000","last_undegraded":"2024-08-17T17:30:09.833754+0000","last_fullsized":"2024-08-17T17:30:09.833754+0000","mapping_epoch":283,"log_start":"289'4000","ondisk_log_start":"289'4000","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:41.264251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222298112,"num_objects":3392,"num_object_clones":0,"num_object_copies":10176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":490816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":474,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.339421+0000","last_change":"2024-08-17T17:24:36.969933+0000","last_active":"2024-08-17T17:30:03.339421+0000","last_peered":"2024-08-17T17:30:03.339421+0000","last_clean":"2024-08-17T17:30:03.339421+0000","last_became_active":"2024-08-17T17:24:36.969657+0000","last_became_peered":"2024-08-17T17:24:36.969657+0000","last_unstale":"2024-08-17T17:30:03.339421+0000","last_undegraded":"2024-08-17T17:30:03.339421+0000","last_fullsized":"2024-08-17T17:30:03.339421+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:10.367450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":125,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.339189+0000","last_change":"2024-08-17T17:26:50.368470+0000","last_active":"2024-08-17T17:30:03.339189+0000","last_peered":"2024-08-17T17:30:03.339189+0000","last_clean":"2024-08-17T17:30:03.339189+0000","last_became_active":"2024-08-17T17:26:50.367844+0000","last_became_peered":"2024-08-17T17:26:50.367844+0000","last_unstale":"2024-08-17T17:30:03.339189+0000","last_undegraded":"2024-08-17T17:30:03.339189+0000","last_fullsized":"2024-08-17T17:30:03.339189+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:02.244611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":491,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.343871+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:30:03.343871+0000","last_peered":"2024-08-17T17:30:03.343871+0000","last_clean":"2024-08-17T17:30:03.343871+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:30:03.343871+0000","last_undegraded":"2024-08-17T17:30:03.343871+0000","last_fullsized":"2024-08-17T17:30:03.343871+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"305'12260","reported_seq":12316,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:09.848158+0000","last_change":"2024-08-17T17:27:05.819139+0000","last_active":"2024-08-17T17:30:09.848158+0000","last_peered":"2024-08-17T17:30:09.848158+0000","last_clean":"2024-08-17T17:30:09.848158+0000","last_became_active":"2024-08-17T17:27:05.818873+0000","last_became_peered":"2024-08-17T17:27:05.818873+0000","last_unstale":"2024-08-17T17:30:09.848158+0000","last_undegraded":"2024-08-17T17:30:09.848158+0000","last_fullsized":"2024-08-17T17:30:09.848158+0000","mapping_epoch":283,"log_start":"289'4300","ondisk_log_start":"289'4300","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7960,"log_dups_size":0,"ondisk_log_size":7960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:16.338176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219545600,"num_objects":3350,"num_object_clones":0,"num_object_copies":10050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12260,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":282,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.335345+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:30:03.335345+0000","last_peered":"2024-08-17T17:30:03.335345+0000","last_clean":"2024-08-17T17:30:03.335345+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:30:03.335345+0000","last_undegraded":"2024-08-17T17:30:03.335345+0000","last_fullsized":"2024-08-17T17:30:03.335345+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":280,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325516+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:29:56.325516+0000","last_peered":"2024-08-17T17:29:56.325516+0000","last_clean":"2024-08-17T17:29:56.325516+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:29:56.325516+0000","last_undegraded":"2024-08-17T17:29:56.325516+0000","last_fullsized":"2024-08-17T17:29:56.325516+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":190,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.339745+0000","last_change":"2024-08-17T17:26:50.310453+0000","last_active":"2024-08-17T17:30:03.339745+0000","last_peered":"2024-08-17T17:30:03.339745+0000","last_clean":"2024-08-17T17:30:03.339745+0000","last_became_active":"2024-08-17T17:26:50.310180+0000","last_became_peered":"2024-08-17T17:26:50.310180+0000","last_unstale":"2024-08-17T17:30:03.339745+0000","last_undegraded":"2024-08-17T17:30:03.339745+0000","last_fullsized":"2024-08-17T17:30:03.339745+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:38.334936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"305'11900","reported_seq":11957,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:08.981439+0000","last_change":"2024-08-17T17:27:05.299496+0000","last_active":"2024-08-17T17:30:08.981439+0000","last_peered":"2024-08-17T17:30:08.981439+0000","last_clean":"2024-08-17T17:30:08.981439+0000","last_became_active":"2024-08-17T17:27:05.299195+0000","last_became_peered":"2024-08-17T17:27:05.299195+0000","last_unstale":"2024-08-17T17:30:08.981439+0000","last_undegraded":"2024-08-17T17:30:08.981439+0000","last_fullsized":"2024-08-17T17:30:08.981439+0000","mapping_epoch":283,"log_start":"287'1800","ondisk_log_start":"287'1800","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:06.961502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212140056,"num_objects":3238,"num_object_clones":0,"num_object_copies":9714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3238,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11900,"num_write_kb":484353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":209,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324527+0000","last_change":"2024-08-17T17:25:46.633494+0000","last_active":"2024-08-17T17:29:56.324527+0000","last_peered":"2024-08-17T17:29:56.324527+0000","last_clean":"2024-08-17T17:29:56.324527+0000","last_became_active":"2024-08-17T17:25:46.633216+0000","last_became_peered":"2024-08-17T17:25:46.633216+0000","last_unstale":"2024-08-17T17:29:56.324527+0000","last_undegraded":"2024-08-17T17:29:56.324527+0000","last_fullsized":"2024-08-17T17:29:56.324527+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:14.957437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":466,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307577+0000","last_change":"2024-08-17T17:26:50.338185+0000","last_active":"2024-08-17T17:29:56.307577+0000","last_peered":"2024-08-17T17:29:56.307577+0000","last_clean":"2024-08-17T17:29:56.307577+0000","last_became_active":"2024-08-17T17:26:50.337992+0000","last_became_peered":"2024-08-17T17:26:50.337992+0000","last_unstale":"2024-08-17T17:29:56.307577+0000","last_undegraded":"2024-08-17T17:29:56.307577+0000","last_fullsized":"2024-08-17T17:29:56.307577+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:26.720733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":441,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.335600+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:30:03.335600+0000","last_peered":"2024-08-17T17:30:03.335600+0000","last_clean":"2024-08-17T17:30:03.335600+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:30:03.335600+0000","last_undegraded":"2024-08-17T17:30:03.335600+0000","last_fullsized":"2024-08-17T17:30:03.335600+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"305'11941","reported_seq":11995,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:06.266433+0000","last_change":"2024-08-17T17:27:05.296016+0000","last_active":"2024-08-17T17:30:06.266433+0000","last_peered":"2024-08-17T17:30:06.266433+0000","last_clean":"2024-08-17T17:30:06.266433+0000","last_became_active":"2024-08-17T17:27:05.295582+0000","last_became_peered":"2024-08-17T17:27:05.295582+0000","last_unstale":"2024-08-17T17:30:06.266433+0000","last_undegraded":"2024-08-17T17:30:06.266433+0000","last_fullsized":"2024-08-17T17:30:06.266433+0000","mapping_epoch":283,"log_start":"290'4400","ondisk_log_start":"290'4400","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:39:20.962510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224329728,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11941,"num_write_kb":491648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":299,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.347759+0000","last_change":"2024-08-17T17:26:50.315231+0000","last_active":"2024-08-17T17:30:03.347759+0000","last_peered":"2024-08-17T17:30:03.347759+0000","last_clean":"2024-08-17T17:30:03.347759+0000","last_became_active":"2024-08-17T17:26:50.314945+0000","last_became_peered":"2024-08-17T17:26:50.314945+0000","last_unstale":"2024-08-17T17:30:03.347759+0000","last_undegraded":"2024-08-17T17:30:03.347759+0000","last_fullsized":"2024-08-17T17:30:03.347759+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:13.740267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":284,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307152+0000","last_change":"2024-08-17T17:26:32.215417+0000","last_active":"2024-08-17T17:29:56.307152+0000","last_peered":"2024-08-17T17:29:56.307152+0000","last_clean":"2024-08-17T17:29:56.307152+0000","last_became_active":"2024-08-17T17:26:32.215160+0000","last_became_peered":"2024-08-17T17:26:32.215160+0000","last_unstale":"2024-08-17T17:29:56.307152+0000","last_undegraded":"2024-08-17T17:29:56.307152+0000","last_fullsized":"2024-08-17T17:29:56.307152+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:17.370439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":347,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324414+0000","last_change":"2024-08-17T17:26:32.221201+0000","last_active":"2024-08-17T17:29:56.324414+0000","last_peered":"2024-08-17T17:29:56.324414+0000","last_clean":"2024-08-17T17:29:56.324414+0000","last_became_active":"2024-08-17T17:26:32.220913+0000","last_became_peered":"2024-08-17T17:26:32.220913+0000","last_unstale":"2024-08-17T17:29:56.324414+0000","last_undegraded":"2024-08-17T17:29:56.324414+0000","last_fullsized":"2024-08-17T17:29:56.324414+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:20.201903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"305'11948","reported_seq":11978,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:05.297503+0000","last_change":"2024-08-17T17:27:05.300239+0000","last_active":"2024-08-17T17:30:05.297503+0000","last_peered":"2024-08-17T17:30:05.297503+0000","last_clean":"2024-08-17T17:30:05.297503+0000","last_became_active":"2024-08-17T17:27:05.296428+0000","last_became_peered":"2024-08-17T17:27:05.296428+0000","last_unstale":"2024-08-17T17:30:05.297503+0000","last_undegraded":"2024-08-17T17:30:05.297503+0000","last_fullsized":"2024-08-17T17:30:05.297503+0000","mapping_epoch":283,"log_start":"289'3600","ondisk_log_start":"289'3600","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8348,"log_dups_size":0,"ondisk_log_size":8348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:37:11.949968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11948,"num_write_kb":495424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":461,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.337470+0000","last_change":"2024-08-17T17:26:32.209014+0000","last_active":"2024-08-17T17:30:03.337470+0000","last_peered":"2024-08-17T17:30:03.337470+0000","last_clean":"2024-08-17T17:30:03.337470+0000","last_became_active":"2024-08-17T17:26:32.208539+0000","last_became_peered":"2024-08-17T17:26:32.208539+0000","last_unstale":"2024-08-17T17:30:03.337470+0000","last_undegraded":"2024-08-17T17:30:03.337470+0000","last_fullsized":"2024-08-17T17:30:03.337470+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:40:15.666378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":440,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336201+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:30:03.336201+0000","last_peered":"2024-08-17T17:30:03.336201+0000","last_clean":"2024-08-17T17:30:03.336201+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:30:03.336201+0000","last_undegraded":"2024-08-17T17:30:03.336201+0000","last_fullsized":"2024-08-17T17:30:03.336201+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":450,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.340081+0000","last_change":"2024-08-17T17:26:32.211876+0000","last_active":"2024-08-17T17:30:03.340081+0000","last_peered":"2024-08-17T17:30:03.340081+0000","last_clean":"2024-08-17T17:30:03.340081+0000","last_became_active":"2024-08-17T17:26:32.211212+0000","last_became_peered":"2024-08-17T17:26:32.211212+0000","last_unstale":"2024-08-17T17:30:03.340081+0000","last_undegraded":"2024-08-17T17:30:03.340081+0000","last_fullsized":"2024-08-17T17:30:03.340081+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:50.032892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"305'11855","reported_seq":11885,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:07.535997+0000","last_change":"2024-08-17T17:27:05.291199+0000","last_active":"2024-08-17T17:30:07.535997+0000","last_peered":"2024-08-17T17:30:07.535997+0000","last_clean":"2024-08-17T17:30:07.535997+0000","last_became_active":"2024-08-17T17:27:05.290910+0000","last_became_peered":"2024-08-17T17:27:05.290910+0000","last_unstale":"2024-08-17T17:30:07.535997+0000","last_undegraded":"2024-08-17T17:30:07.535997+0000","last_fullsized":"2024-08-17T17:30:07.535997+0000","mapping_epoch":283,"log_start":"289'3700","ondisk_log_start":"289'3700","created":283,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:27:04.247901+0000","last_clean_scrub_stamp":"2024-08-17T17:27:04.247901+0000","objects_scrubbed":0,"log_size":8155,"log_dups_size":0,"ondisk_log_size":8155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:32.168692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11855,"num_write_kb":484544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":130,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.348950+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:30:03.348950+0000","last_peered":"2024-08-17T17:30:03.348950+0000","last_clean":"2024-08-17T17:30:03.348950+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:30:03.348950+0000","last_undegraded":"2024-08-17T17:30:03.348950+0000","last_fullsized":"2024-08-17T17:30:03.348950+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":289,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.309165+0000","last_change":"2024-08-17T17:26:32.218111+0000","last_active":"2024-08-17T17:29:56.309165+0000","last_peered":"2024-08-17T17:29:56.309165+0000","last_clean":"2024-08-17T17:29:56.309165+0000","last_became_active":"2024-08-17T17:26:32.217705+0000","last_became_peered":"2024-08-17T17:26:32.217705+0000","last_unstale":"2024-08-17T17:29:56.309165+0000","last_undegraded":"2024-08-17T17:29:56.309165+0000","last_fullsized":"2024-08-17T17:29:56.309165+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:50.751085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325167+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:29:56.325167+0000","last_peered":"2024-08-17T17:29:56.325167+0000","last_clean":"2024-08-17T17:29:56.325167+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:29:56.325167+0000","last_undegraded":"2024-08-17T17:29:56.325167+0000","last_fullsized":"2024-08-17T17:29:56.325167+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325417+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:29:56.325417+0000","last_peered":"2024-08-17T17:29:56.325417+0000","last_clean":"2024-08-17T17:29:56.325417+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:29:56.325417+0000","last_undegraded":"2024-08-17T17:29:56.325417+0000","last_fullsized":"2024-08-17T17:29:56.325417+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":282,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325601+0000","last_change":"2024-08-17T17:26:32.215288+0000","last_active":"2024-08-17T17:29:56.325601+0000","last_peered":"2024-08-17T17:29:56.325601+0000","last_clean":"2024-08-17T17:29:56.325601+0000","last_became_active":"2024-08-17T17:26:32.214754+0000","last_became_peered":"2024-08-17T17:26:32.214754+0000","last_unstale":"2024-08-17T17:29:56.325601+0000","last_undegraded":"2024-08-17T17:29:56.325601+0000","last_fullsized":"2024-08-17T17:29:56.325601+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.684058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":440,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.335453+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:30:03.335453+0000","last_peered":"2024-08-17T17:30:03.335453+0000","last_clean":"2024-08-17T17:30:03.335453+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:30:03.335453+0000","last_undegraded":"2024-08-17T17:30:03.335453+0000","last_fullsized":"2024-08-17T17:30:03.335453+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":236,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307081+0000","last_change":"2024-08-17T17:26:32.207981+0000","last_active":"2024-08-17T17:29:56.307081+0000","last_peered":"2024-08-17T17:29:56.307081+0000","last_clean":"2024-08-17T17:29:56.307081+0000","last_became_active":"2024-08-17T17:26:32.207636+0000","last_became_peered":"2024-08-17T17:26:32.207636+0000","last_unstale":"2024-08-17T17:29:56.307081+0000","last_undegraded":"2024-08-17T17:29:56.307081+0000","last_fullsized":"2024-08-17T17:29:56.307081+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:32.412636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":199,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324787+0000","last_change":"2024-08-17T17:26:32.210741+0000","last_active":"2024-08-17T17:29:56.324787+0000","last_peered":"2024-08-17T17:29:56.324787+0000","last_clean":"2024-08-17T17:29:56.324787+0000","last_became_active":"2024-08-17T17:26:32.210417+0000","last_became_peered":"2024-08-17T17:26:32.210417+0000","last_unstale":"2024-08-17T17:29:56.324787+0000","last_undegraded":"2024-08-17T17:29:56.324787+0000","last_fullsized":"2024-08-17T17:29:56.324787+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:41.290842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":398,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336426+0000","last_change":"2024-08-17T17:26:32.211602+0000","last_active":"2024-08-17T17:30:03.336426+0000","last_peered":"2024-08-17T17:30:03.336426+0000","last_clean":"2024-08-17T17:30:03.336426+0000","last_became_active":"2024-08-17T17:26:32.211288+0000","last_became_peered":"2024-08-17T17:26:32.211288+0000","last_unstale":"2024-08-17T17:30:03.336426+0000","last_undegraded":"2024-08-17T17:30:03.336426+0000","last_fullsized":"2024-08-17T17:30:03.336426+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:49.979367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":353,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324804+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:29:56.324804+0000","last_peered":"2024-08-17T17:29:56.324804+0000","last_clean":"2024-08-17T17:29:56.324804+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:29:56.324804+0000","last_undegraded":"2024-08-17T17:29:56.324804+0000","last_fullsized":"2024-08-17T17:29:56.324804+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":399,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.339998+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:30:03.339998+0000","last_peered":"2024-08-17T17:30:03.339998+0000","last_clean":"2024-08-17T17:30:03.339998+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:30:03.339998+0000","last_undegraded":"2024-08-17T17:30:03.339998+0000","last_fullsized":"2024-08-17T17:30:03.339998+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":276,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.309065+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:29:56.309065+0000","last_peered":"2024-08-17T17:29:56.309065+0000","last_clean":"2024-08-17T17:29:56.309065+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:29:56.309065+0000","last_undegraded":"2024-08-17T17:29:56.309065+0000","last_fullsized":"2024-08-17T17:29:56.309065+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":431,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.340187+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:30:03.340187+0000","last_peered":"2024-08-17T17:30:03.340187+0000","last_clean":"2024-08-17T17:30:03.340187+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:30:03.340187+0000","last_undegraded":"2024-08-17T17:30:03.340187+0000","last_fullsized":"2024-08-17T17:30:03.340187+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":233,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.307411+0000","last_change":"2024-08-17T17:26:32.215823+0000","last_active":"2024-08-17T17:29:56.307411+0000","last_peered":"2024-08-17T17:29:56.307411+0000","last_clean":"2024-08-17T17:29:56.307411+0000","last_became_active":"2024-08-17T17:26:32.214110+0000","last_became_peered":"2024-08-17T17:26:32.214110+0000","last_unstale":"2024-08-17T17:29:56.307411+0000","last_undegraded":"2024-08-17T17:29:56.307411+0000","last_fullsized":"2024-08-17T17:29:56.307411+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.088172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":439,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336837+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:30:03.336837+0000","last_peered":"2024-08-17T17:30:03.336837+0000","last_clean":"2024-08-17T17:30:03.336837+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:30:03.336837+0000","last_undegraded":"2024-08-17T17:30:03.336837+0000","last_fullsized":"2024-08-17T17:30:03.336837+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":399,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.336532+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:30:03.336532+0000","last_peered":"2024-08-17T17:30:03.336532+0000","last_clean":"2024-08-17T17:30:03.336532+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:30:03.336532+0000","last_undegraded":"2024-08-17T17:30:03.336532+0000","last_fullsized":"2024-08-17T17:30:03.336532+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":278,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325380+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:29:56.325380+0000","last_peered":"2024-08-17T17:29:56.325380+0000","last_clean":"2024-08-17T17:29:56.325380+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:29:56.325380+0000","last_undegraded":"2024-08-17T17:29:56.325380+0000","last_fullsized":"2024-08-17T17:29:56.325380+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":462,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.339633+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:30:03.339633+0000","last_peered":"2024-08-17T17:30:03.339633+0000","last_clean":"2024-08-17T17:30:03.339633+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:30:03.339633+0000","last_undegraded":"2024-08-17T17:30:03.339633+0000","last_fullsized":"2024-08-17T17:30:03.339633+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":137,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.348777+0000","last_change":"2024-08-17T17:26:32.214552+0000","last_active":"2024-08-17T17:30:03.348777+0000","last_peered":"2024-08-17T17:30:03.348777+0000","last_clean":"2024-08-17T17:30:03.348777+0000","last_became_active":"2024-08-17T17:26:32.214338+0000","last_became_peered":"2024-08-17T17:26:32.214338+0000","last_unstale":"2024-08-17T17:30:03.348777+0000","last_undegraded":"2024-08-17T17:30:03.348777+0000","last_fullsized":"2024-08-17T17:30:03.348777+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:21.212607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":286,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325390+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:29:56.325390+0000","last_peered":"2024-08-17T17:29:56.325390+0000","last_clean":"2024-08-17T17:29:56.325390+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:29:56.325390+0000","last_undegraded":"2024-08-17T17:29:56.325390+0000","last_fullsized":"2024-08-17T17:29:56.325390+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":159,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325533+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:29:56.325533+0000","last_peered":"2024-08-17T17:29:56.325533+0000","last_clean":"2024-08-17T17:29:56.325533+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:29:56.325533+0000","last_undegraded":"2024-08-17T17:29:56.325533+0000","last_fullsized":"2024-08-17T17:29:56.325533+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":296,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.325110+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:29:56.325110+0000","last_peered":"2024-08-17T17:29:56.325110+0000","last_clean":"2024-08-17T17:29:56.325110+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:29:56.325110+0000","last_undegraded":"2024-08-17T17:29:56.325110+0000","last_fullsized":"2024-08-17T17:29:56.325110+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":159,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.327832+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:29:56.327832+0000","last_peered":"2024-08-17T17:29:56.327832+0000","last_clean":"2024-08-17T17:29:56.327832+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:29:56.327832+0000","last_undegraded":"2024-08-17T17:29:56.327832+0000","last_fullsized":"2024-08-17T17:29:56.327832+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":444,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.337478+0000","last_change":"2024-08-17T17:24:36.972596+0000","last_active":"2024-08-17T17:30:03.337478+0000","last_peered":"2024-08-17T17:30:03.337478+0000","last_clean":"2024-08-17T17:30:03.337478+0000","last_became_active":"2024-08-17T17:24:36.971541+0000","last_became_peered":"2024-08-17T17:24:36.971541+0000","last_unstale":"2024-08-17T17:30:03.337478+0000","last_undegraded":"2024-08-17T17:30:03.337478+0000","last_fullsized":"2024-08-17T17:30:03.337478+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:38.457003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":416,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.337055+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:30:03.337055+0000","last_peered":"2024-08-17T17:30:03.337055+0000","last_clean":"2024-08-17T17:30:03.337055+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:30:03.337055+0000","last_undegraded":"2024-08-17T17:30:03.337055+0000","last_fullsized":"2024-08-17T17:30:03.337055+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2024-08-17T17:29:56.324994+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:29:56.324994+0000","last_peered":"2024-08-17T17:29:56.324994+0000","last_clean":"2024-08-17T17:29:56.324994+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:29:56.324994+0000","last_undegraded":"2024-08-17T17:29:56.324994+0000","last_fullsized":"2024-08-17T17:29:56.324994+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":132,"reported_epoch":305,"state":"active+clean","last_fresh":"2024-08-17T17:30:03.349008+0000","last_change":"2024-08-17T17:24:37.032048+0000","last_active":"2024-08-17T17:30:03.349008+0000","last_peered":"2024-08-17T17:30:03.349008+0000","last_clean":"2024-08-17T17:30:03.349008+0000","last_became_active":"2024-08-17T17:24:36.972161+0000","last_became_peered":"2024-08-17T17:24:36.972161+0000","last_unstale":"2024-08-17T17:30:03.349008+0000","last_undegraded":"2024-08-17T17:30:03.349008+0000","last_fullsized":"2024-08-17T17:30:03.349008+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:55.363820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3525443608,"num_objects":53795,"num_object_clones":0,"num_object_copies":161385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53795,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193039,"num_write_kb":7898625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10638536704,"data_stored":10638524488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138139,"ondisk_log_size":138139,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1162,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":270,"seq":1159641169964,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1733520,"kb_used_data":1354948,"kb_used_omap":12,"kb_used_meta":378547,"kb_avail":92011632,"statfs":{"total":95995035648,"available":94219911168,"internally_reserved":0,"allocated":1387466752,"data_stored":1385891343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12410,"internal_metadata":387633030},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463334,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2653064,"kb_used_data":1751040,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":91092088,"statfs":{"total":95995035648,"available":93278298112,"internally_reserved":0,"allocated":1793064960,"data_stored":1791493528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492009,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1254192,"kb_used_data":858856,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92490960,"statfs":{"total":95995035648,"available":94710743040,"internally_reserved":0,"allocated":879468544,"data_stored":877915896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986608,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1766276,"kb_used_data":865340,"kb_used_omap":1,"kb_used_meta":900926,"kb_avail":91978876,"statfs":{"total":95995035648,"available":94186369024,"internally_reserved":0,"allocated":886108160,"data_stored":884540080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":922548568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225428,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2229292,"kb_used_data":1487268,"kb_used_omap":4,"kb_used_meta":741819,"kb_avail":91515860,"statfs":{"total":95995035648,"available":93712240640,"internally_reserved":0,"allocated":1522962432,"data_stored":1521414108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":759623104},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075753,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1823032,"kb_used_data":1276592,"kb_used_omap":6,"kb_used_meta":546425,"kb_avail":91922120,"statfs":{"total":95995035648,"available":94128250880,"internally_reserved":0,"allocated":1307230208,"data_stored":1305670423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":559539643},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533645,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2660844,"kb_used_data":1788132,"kb_used_omap":5,"kb_used_meta":872634,"kb_avail":91084308,"statfs":{"total":95995035648,"available":93270331392,"internally_reserved":0,"allocated":1831047168,"data_stored":1829497642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":893577456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089622,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2005816,"kb_used_data":1109040,"kb_used_omap":5,"kb_used_meta":896762,"kb_avail":91739336,"statfs":{"total":95995035648,"available":93941080064,"internally_reserved":0,"allocated":1135656960,"data_stored":1134097683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":918284347},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1115226112,"data_stored":1115226112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1811742720,"data_stored":1811742720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1298927616,"data_stored":1298923544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1518075904,"data_stored":1518075904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":875827200,"data_stored":875823128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":865075200,"data_stored":865075200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1771376640,"data_stored":1771372568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1382285312,"data_stored":1382285312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:30:12.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-17T17:30:12.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:12.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:12.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:12.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:12.746 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:12.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:12.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:12 smithi152 ceph-mon[93456]: from='client.46589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:30:12.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3967893020' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:12.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:12.747 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:12 smithi152 ceph-mon[93456]: pgmap v1034: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T17:30:12.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:12.857 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:12.857 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:12.857 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[106729]: from='client.46589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:30:12.857 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3967893020' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:12.857 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:12.857 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[106729]: pgmap v1034: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T17:30:12.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:12.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:12.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:12.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[103614]: from='client.46589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:30:12.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3967893020' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:12.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:12.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:12 smithi142 ceph-mon[103614]: pgmap v1034: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T17:30:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:12.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:13.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:13.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:13.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:13.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:13.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:13.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:13.414 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:13.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:13.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:13.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:13.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:13.700 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:13 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:13.700 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:13 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:13.701 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:13 smithi142 ceph-mon[103614]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:30:13.704 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:13 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:13.704 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:13 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:13.705 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:13 smithi142 ceph-mon[106729]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:30:13.722 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:13.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:13 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:13 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:13 smithi152 ceph-mon[93456]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:30:13.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:13.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:13.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:14.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:14.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:14.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:14.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:14.353 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:14.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:14.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:14.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:14.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:14.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:14.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:14.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:14 smithi152 ceph-mon[93456]: pgmap v1036: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:30:14.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:14.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:14.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:14 smithi142 ceph-mon[106729]: pgmap v1036: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:30:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:14 smithi142 ceph-mon[103614]: pgmap v1036: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:30:14.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:14.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:14.996 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:30:14.996 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:30:14.996 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:30:14.996 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:30:15.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:15.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:15.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:15.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:15.222 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:30:15.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:15.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:15.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:15.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:15.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:15.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:15.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:30:15.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:15.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:15.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:15.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:16.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:16.110 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:16.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:16.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:16.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:16.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:16.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.534 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:16.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:16.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:16.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:16.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:16.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:16.815 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:16.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:16.917 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:16.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:17.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:17.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:17.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:17.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:17.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:17.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:17.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:17.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:17 smithi142 ceph-mon[106729]: pgmap v1037: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:30:17.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:17 smithi142 ceph-mon[103614]: pgmap v1037: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:30:17.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:17.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:17 smithi152 ceph-mon[93456]: pgmap v1037: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:30:17.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:17.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:17.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:17.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:17.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:17.771 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:17.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:17.925 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:30:17.925 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:17.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:17.958 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:30:17.962 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:17.996 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:30:17.996 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:18.033 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:30:18.034 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:18.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:18.068 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:30:18.068 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:18.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:18.104 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:30:18.105 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:18.150 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:30:18.150 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:30:18.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:18.185 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:18.186 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:18.190 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:30:18.190 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:18.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:18.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:18.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/310675584' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:18.335 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:18.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:18.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:18.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.510 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:18.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:18.558 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/310675584' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:18.559 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/310675584' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:18.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:18.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:18.748 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:18.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:18.896 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:18.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:30:18.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.925 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:18.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:19.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:19.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:19.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:19.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:19.214 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:19.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:19.321 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:19.336 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:19.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:19.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:19.469 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:19 smithi152 ceph-mon[93456]: pgmap v1038: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-17T17:30:19.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:19 smithi142 ceph-mon[106729]: pgmap v1038: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-17T17:30:19.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:19 smithi142 ceph-mon[103614]: pgmap v1038: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-17T17:30:19.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:19.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:19.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:19.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:19.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:19.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.791 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:19.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:19.892 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:19.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:19.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:20.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:20.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:20.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:20.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:20.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:20.204 INFO:teuthology.orchestra.run.smithi142.stderr:set pool 1 pgp_num to 1 2024-08-17T17:30:20.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:20.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:20.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:20.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:20.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4022841570' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:20 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:20.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:30:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:30:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:20.591 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4022841570' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:20.592 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:20 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:20.592 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4022841570' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:20.592 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:20 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:30:20.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:20.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:20.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:20.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:20.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:20.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:20.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:21.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:21.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:21.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:21 smithi152 ceph-mon[93456]: pgmap v1039: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:30:21.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:21 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:30:21.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:21 smithi152 ceph-mon[93456]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:30:21.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:21.279 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:21.279 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:21.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:21.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:21.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:30:21.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:21.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:21.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:21.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:21.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-mon[106729]: pgmap v1039: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:30:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:30:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-mon[106729]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:30:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-mon[103614]: pgmap v1039: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2024-08-17T17:30:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:30:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-mon[103614]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:30:21.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.731 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:21.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:21.875 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:21.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:22.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:30:22.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:22.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:30:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:30:21] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:30:22.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:22.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:22.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:22.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:22.408 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:22.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:22.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:22.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:22.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:22.636 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:22.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:22.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:22.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:22.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:22.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:22.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:22.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:23.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:23.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:23.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:23.208 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:30:23.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:23.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:23.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:23.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:23.468 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:23 smithi142 ceph-mon[103614]: pgmap v1041: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T17:30:23.468 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1791958532' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:23.468 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:23 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:23.468 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:23 smithi142 ceph-mon[106729]: pgmap v1041: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T17:30:23.468 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1791958532' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:23.468 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:23 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:23.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:23.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:23 smithi152 ceph-mon[93456]: pgmap v1041: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T17:30:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1791958532' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:23 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:23.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:23.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.669 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:23.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:23.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:23.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:23.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:23.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:23.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:23.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:24.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:24.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:24.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:24.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:24.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:24.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:24.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:24.382 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:24 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:24.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:24 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:24.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:24 smithi152 ceph-mon[93456]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:30:24.412 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:24 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:24.413 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:24 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:24.413 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:24 smithi142 ceph-mon[106729]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:30:24.413 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:24 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:24.413 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:24 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:24.413 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:24 smithi142 ceph-mon[103614]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:30:24.413 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:24.413 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:24.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:24.503 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:24.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:24.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:24.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:24.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:24.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:24.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:24.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:25.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:25.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:25.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:30:25.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:25.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:25.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:25.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:25 smithi152 ceph-mon[93456]: pgmap v1043: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-08-17T17:30:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:25.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:25.440 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:25.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:25.484 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:30:25.484 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:25.501 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:25.524 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:30:25.524 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:25.524 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:25 smithi142 ceph-mon[103614]: pgmap v1043: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-08-17T17:30:25.525 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:25 smithi142 ceph-mon[106729]: pgmap v1043: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-08-17T17:30:25.555 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:30:25.555 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:25.596 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:30:25.597 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:25.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:25.633 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:30:25.633 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:25.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:25.659 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:30:25.660 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:25.689 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:30:25.689 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:30:25.719 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:30:25.719 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:25.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.771 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:25.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:25.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:25.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:26.000 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:26.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:26.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:26.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:26.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:26.201 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:30:26.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:30:26.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:26.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:30:26.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:26.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:26.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:26.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:26.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:26.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:26.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:26 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3524047777' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:26 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3524047777' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:26.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:26 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3524047777' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:26.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:26.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:26.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:26.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:26.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:26.871 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:26.941 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:26.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:26.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:26.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:27.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:27.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:27.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:27.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:27.273 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:30:27.274 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:30:27.274 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:30:27.274 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:30:27.274 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:30:27.349 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:27 smithi142 ceph-mon[103614]: pgmap v1044: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-08-17T17:30:27.349 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:27.349 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:27 smithi142 ceph-mon[106729]: pgmap v1044: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-08-17T17:30:27.349 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:27.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:27.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:27.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:27.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:27 smithi152 ceph-mon[93456]: pgmap v1044: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-08-17T17:30:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:27.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:27.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:27.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:27.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:27.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:28.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:28.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:28.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:30:28.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:28.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:28.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:28.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:28.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:28.500 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:28.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:28.538 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:28.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:28.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:28.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:28.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:28.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:28.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:28.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:29.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:29.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:29.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:29.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:29.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:29 smithi152 ceph-mon[93456]: pgmap v1045: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-08-17T17:30:29.349 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:29.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:29 smithi142 ceph-mon[106729]: pgmap v1045: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-08-17T17:30:29.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:29 smithi142 ceph-mon[103614]: pgmap v1045: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-08-17T17:30:29.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:29.489 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:29.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:29.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:29.610 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:29.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:29.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:29.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:29.745 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:29.847 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:29.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:29.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:30.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:30.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:30.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:30.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:30.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:30.275 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:30:30.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:30.350 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1156353255' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:30:30.350 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1156353255' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:30:30.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:30.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:30.511 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1156353255' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:30:30.511 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:30:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:30:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:30.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:30.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:30.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:30.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:30.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:30.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:30.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:30.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:30.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:30.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:30.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:31.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:31.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:31.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:31.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:31.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:31.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:30:31.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:31.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:31.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:31.489 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:31 smithi152 ceph-mon[93456]: pgmap v1046: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-08-17T17:30:31.489 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1156353255' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:30:31.489 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:31 smithi152 ceph-mon[93456]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:30:31.514 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:31.536 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:30:31.536 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:25333 11.f 11:f29eb768:::benchmark_data_smithi142_227468_object25332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:22.098842+0000", 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 189.433110838, 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.106074852, 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.537 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 25333 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098842+0000", 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098842+0000", 2024-08-17T17:30:31.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098841+0000", 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098887+0000", 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5683000000000003e-05 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098903+0000", 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5739e-05 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098909+0000", 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9519999999999999e-06 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.098967+0000", 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8288000000000002e-05 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.099071+0000", 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103701 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.099184+0000", 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011389500000000001 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.100360+0000", 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001175284 2024-08-17T17:30:31.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.101352+0000", 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00099259900000000004 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.204861+0000", 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.103508768 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.204894+0000", 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2712000000000001e-05 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:22.204917+0000", 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3215000000000001e-05 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:33585 11.f 11:fbe36d80:::benchmark_data_smithi142_227468_object33584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:26.617437+0000", 2024-08-17T17:30:31.543 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.91451606499999, 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25483171500000001, 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 33585 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617437+0000", 2024-08-17T17:30:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617437+0000", 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617435+0000", 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617491+0000", 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5705000000000003e-05 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617504+0000", 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3178e-05 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617510+0000", 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9170000000000003e-06 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617558+0000", 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8062999999999997e-05 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617692+0000", 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013362500000000001 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.617904+0000", 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000211912 2024-08-17T17:30:31.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.619348+0000", 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014445949999999999 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.619832+0000", 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00048332399999999998 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.872149+0000", 2024-08-17T17:30:31.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25231700899999998 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.872229+0000", 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0766000000000006e-05 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.872269+0000", 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9223e-05 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:33594 11.f 11:f4575d4d:::benchmark_data_smithi142_227468_object33593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:26.622955+0000", 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.908998427, 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25517420899999999, 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 33594 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.550 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.622955+0000", 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.622955+0000", 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.622953+0000", 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.622988+0000", 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4697999999999999e-05 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.622999+0000", 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1139e-05 2024-08-17T17:30:31.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.623005+0000", 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5140000000000001e-06 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.623090+0000", 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5711999999999995e-05 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.623200+0000", 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109425 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.623323+0000", 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012344100000000001 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.625228+0000", 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019051319999999999 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.625745+0000", 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051630299999999997 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.878038+0000", 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25229359299999998 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.878090+0000", 2024-08-17T17:30:31.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1316999999999997e-05 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.878129+0000", 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9137000000000002e-05 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:33603 11.f 11:f4a0b415:::benchmark_data_smithi142_227468_object33602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.556 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:26.626680+0000", 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.905272572, 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25283259800000002, 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 33603 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626680+0000", 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626680+0000", 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626679+0000", 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:31.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626707+0000", 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8039999999999999e-05 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626718+0000", 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0448000000000001e-05 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626723+0000", 2024-08-17T17:30:31.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0660000000000003e-06 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626764+0000", 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1440999999999998e-05 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.626900+0000", 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013580600000000001 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.627046+0000", 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014563 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.628380+0000", 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001333919 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.629038+0000", 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065822500000000002 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.879423+0000", 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25038541199999997 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.879477+0000", 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3992000000000001e-05 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:26.879513+0000", 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5828999999999999e-05 2024-08-17T17:30:31.562 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34146 11.4 11:22c64800:::benchmark_data_smithi142_227468_object34145:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.115737+0000", 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.41621624800001, 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.108635601, 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34146 2024-08-17T17:30:31.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.115737+0000", 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.115737+0000", 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.115736+0000", 2024-08-17T17:30:31.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.115759+0000", 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3411999999999999e-05 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.115768+0000", 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1480000000000006e-06 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.115771+0000", 2024-08-17T17:30:31.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8419999999999998e-06 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.117356+0000", 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001584343 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.117466+0000", 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110066 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.117596+0000", 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013024299999999999 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.118698+0000", 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011014180000000001 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.119401+0000", 2024-08-17T17:30:31.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070317899999999996 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.224312+0000", 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10491091900000001 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.224347+0000", 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5348000000000001e-05 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.224372+0000", 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5439000000000002e-05 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34157 11.f 11:f6d4611e:::benchmark_data_smithi142_227468_object34156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.121929+0000", 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.410024093, 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10740216900000001, 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34157 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.121929+0000", 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.121929+0000", 2024-08-17T17:30:31.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.121928+0000", 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.121961+0000", 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3209e-05 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.121975+0000", 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4032000000000001e-05 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.121980+0000", 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1580000000000004e-06 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.122032+0000", 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1669000000000003e-05 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.122154+0000", 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012228599999999999 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.122286+0000", 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013179000000000001 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.123421+0000", 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001134559 2024-08-17T17:30:31.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.125385+0000", 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019644290000000002 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.229246+0000", 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.103860932 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.229297+0000", 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0856999999999998e-05 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.229331+0000", 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4348999999999999e-05 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.576 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34282 11.5 11:ada37598:::benchmark_data_smithi142_227468_object34281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.272059+0000", 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25989412300001, 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.158716792, 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34282 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272059+0000", 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272059+0000", 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272058+0000", 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272113+0000", 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5526999999999999e-05 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272125+0000", 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2187999999999999e-05 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272131+0000", 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0209999999999996e-06 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272172+0000", 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0627999999999998e-05 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272283+0000", 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110509 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.272417+0000", 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134101 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273965+0000", 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001548275 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428121+0000", 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.154156506 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430691+0000", 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025694149999999998 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430739+0000", 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8214000000000003e-05 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430776+0000", 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6708000000000002e-05 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34283 11.4 11:2fb93ffc:::benchmark_data_smithi142_227468_object34282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.273633+0000", 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25831984199999, 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15566482000000001, 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.583 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34283 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273633+0000", 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273633+0000", 2024-08-17T17:30:31.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273632+0000", 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273664+0000", 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1813000000000002e-05 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273676+0000", 2024-08-17T17:30:31.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2495e-05 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273682+0000", 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4260000000000002e-06 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273728+0000", 2024-08-17T17:30:31.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6186000000000001e-05 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273954+0000", 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022608900000000001 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274055+0000", 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010142 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429132+0000", 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15507616900000001 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429137+0000", 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7520000000000002e-06 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429216+0000", 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8688000000000003e-05 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429261+0000", 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5071000000000003e-05 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429298+0000", 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6971999999999998e-05 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.589 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34284 11.5 11:aa778257:::benchmark_data_smithi142_227468_object34283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.273698+0000", 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25825459800001, 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15717678800000001, 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.590 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34284 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273698+0000", 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273698+0000", 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273698+0000", 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273727+0000", 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8592999999999999e-05 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273738+0000", 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1097999999999999e-05 2024-08-17T17:30:31.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.273744+0000", 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9279999999999998e-06 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274021+0000", 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027720499999999999 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274129+0000", 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010787 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274258+0000", 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000129183 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275714+0000", 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001455685 2024-08-17T17:30:31.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428160+0000", 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15244695799999999 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430816+0000", 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002655265 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430843+0000", 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7350000000000001e-05 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430875+0000", 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2165999999999997e-05 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34288 11.4 11:235ca31e:::benchmark_data_smithi142_227468_object34287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.274065+0000", 2024-08-17T17:30:31.596 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25788855299999, 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.155322878, 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34288 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274065+0000", 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274065+0000", 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274064+0000", 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274155+0000", 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1655000000000003e-05 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274166+0000", 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1049e-05 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274172+0000", 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5169999999999999e-06 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275326+0000", 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001154242 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275421+0000", 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4827000000000005e-05 2024-08-17T17:30:31.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275534+0000", 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113218 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429142+0000", 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15360743900000001 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429166+0000", 2024-08-17T17:30:31.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4831e-05 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429328+0000", 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016178899999999999 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429361+0000", 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2496999999999997e-05 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429387+0000", 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6648999999999998e-05 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34289 11.4 11:2e65286f:::benchmark_data_smithi142_227468_object34288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.274184+0000", 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25776929, 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15529182799999999, 2024-08-17T17:30:31.603 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34289 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274184+0000", 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274184+0000", 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274183+0000", 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274212+0000", 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8691e-05 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274219+0000", 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.486e-06 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274224+0000", 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4279999999999998e-06 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276869+0000", 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002645044 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276967+0000", 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8528000000000003e-05 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:30:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.277081+0000", 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011400600000000001 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429171+0000", 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15208944299999999 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429194+0000", 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2750000000000001e-05 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429418+0000", 2024-08-17T17:30:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000224038 2024-08-17T17:30:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429450+0000", 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2144999999999998e-05 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429476+0000", 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5809e-05 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34290 11.f 11:f310936c:::benchmark_data_smithi142_227468_object34289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.274229+0000", 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25772368299999, 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15407401600000001, 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34290 2024-08-17T17:30:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274229+0000", 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274229+0000", 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274229+0000", 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274257+0000", 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8238999999999998e-05 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274265+0000", 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4370000000000001e-06 2024-08-17T17:30:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274269+0000", 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5190000000000003e-06 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274309+0000", 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9944000000000002e-05 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274415+0000", 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010566800000000001 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.274531+0000", 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011666 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428055+0000", 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.153523941 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428094+0000", 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9038000000000001e-05 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428228+0000", 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000133566 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428272+0000", 2024-08-17T17:30:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4592999999999998e-05 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428303+0000", 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0953e-05 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34292 11.5 11:a8b06fb2:::benchmark_data_smithi142_227468_object34291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.275958+0000", 2024-08-17T17:30:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.255994724, 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15501974299999999, 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34292 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275958+0000", 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275958+0000", 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.275957+0000", 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276021+0000", 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3629999999999999e-05 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276033+0000", 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2605e-05 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276039+0000", 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0630000000000002e-06 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276081+0000", 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1822e-05 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276193+0000", 2024-08-17T17:30:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111807 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276326+0000", 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000133001 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428170+0000", 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.151844227 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428221+0000", 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1048999999999999e-05 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430913+0000", 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026911410000000002 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430950+0000", 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7932999999999997e-05 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430978+0000", 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7597e-05 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34293 11.f 11:f338b797:::benchmark_data_smithi142_227468_object34292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.276194+0000", 2024-08-17T17:30:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.255758832, 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.153841164, 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34293 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276194+0000", 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276194+0000", 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276193+0000", 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276293+0000", 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9797999999999996e-05 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276304+0000", 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0977e-05 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276310+0000", 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3430000000000003e-06 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276352+0000", 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2465000000000001e-05 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276454+0000", 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102159 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276569+0000", 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114542 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428130+0000", 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.151561684 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428137+0000", 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6969999999999996e-06 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429958+0000", 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001820948 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430001+0000", 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2831999999999999e-05 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430035+0000", 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.451e-05 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36603.0:34282 11.5 e287/283 ondisk, result = 0)", 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.276221+0000", 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.255732166, 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.151903966, 2024-08-17T17:30:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276221+0000", 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276221+0000", 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276220+0000", 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276226+0000", 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2090000000000002e-06 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276238+0000", 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1222e-05 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276243+0000", 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3369999999999999e-06 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428107+0000", 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15186367100000001 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428114+0000", 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.576e-06 2024-08-17T17:30:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428125+0000", 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0749e-05 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36603.0:34284 11.5 e287/283 ondisk, result = 0)", 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.276249+0000", 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25570404999999, 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.151922275, 2024-08-17T17:30:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276249+0000", 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276249+0000", 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276249+0000", 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276251+0000", 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.401e-06 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276256+0000", 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3349999999999997e-06 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.276260+0000", 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2619999999999999e-06 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428150+0000", 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15189024200000001 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428156+0000", 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.784e-06 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428171+0000", 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5525000000000001e-05 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36603.0:34294 11.e e287/283 11:79d59591:::benchmark_data_smithi142_227468_object34293:head v 287'2116, mlcod=287'2114)", 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.277240+0000", 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.25471333799999, 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15352043100000001, 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.277240+0000", 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.277240+0000", 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.277239+0000", 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.277264+0000", 2024-08-17T17:30:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.474e-05 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.427573+0000", 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15030897200000001 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.427583+0000", 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8090000000000004e-06 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.427625+0000", 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.244e-05 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.427648+0000", 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2653000000000001e-05 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430720+0000", 2024-08-17T17:30:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00307231 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430760+0000", 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0117999999999998e-05 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:34296 11.5 11:a0ce933e:::benchmark_data_smithi142_227468_object34295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.278239+0000", 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.253713605, 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15542877099999999, 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34296 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.278239+0000", 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.278239+0000", 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.278238+0000", 2024-08-17T17:30:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.278294+0000", 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5779000000000002e-05 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428061+0000", 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14976750899999999 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428069+0000", 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.559e-06 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428251+0000", 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000181979 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428376+0000", 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012528600000000001 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.429465+0000", 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010890260000000001 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.430989+0000", 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001523707 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.431648+0000", 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065939199999999996 2024-08-17T17:30:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.433592+0000", 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019440130000000001 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.433635+0000", 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2821999999999998e-05 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.433668+0000", 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3000000000000003e-05 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36603.0:34292 11.5 e287/283 ondisk, result = 0)", 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:27:27.279133+0000", 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 184.252820316, 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14909454, 2024-08-17T17:30:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.279133+0000", 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.279133+0000", 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.279132+0000", 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.279136+0000", 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0010000000000003e-06 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.427562+0000", 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14842577800000001 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.427576+0000", 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4309e-05 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428210+0000", 2024-08-17T17:30:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00063387800000000005 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428216+0000", 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8719999999999999e-06 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:27:27.428227+0000", 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1313e-05 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:82312 11.4 11:27dd5f09:::benchmark_data_smithi142_227468_object82311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:28:00.945138+0000", 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 150.586815086, 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.106036166, 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 82312 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:30:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945138+0000", 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945138+0000", 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945136+0000", 2024-08-17T17:30:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945179+0000", 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3025999999999998e-05 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945193+0000", 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3523000000000001e-05 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945201+0000", 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2919999999999996e-06 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945261+0000", 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0278000000000001e-05 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945361+0000", 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9724999999999998e-05 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.945449+0000", 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7389e-05 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.947804+0000", 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023555429999999999 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:00.953368+0000", 2024-08-17T17:30:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055637919999999997 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.051042+0000", 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.097674239999999996 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.051126+0000", 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3645000000000003e-05 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:30:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.051174+0000", 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8412999999999999e-05 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:30:31.662 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:30:31.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:31.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-mon[106729]: pgmap v1046: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-08-17T17:30:31.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1156353255' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:30:31.663 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-mon[106729]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:30:31.663 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-mon[103614]: pgmap v1046: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-08-17T17:30:31.663 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1156353255' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:30:31.664 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-mon[103614]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:30:31.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:31.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:31.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:31.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:31.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:31.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:31.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:32.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:32.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:32.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:30:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:30:31] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:30:32.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:32.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:32.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:32.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:32.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:32.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:32.617 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:30:32.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:32.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:32.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:32.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:32.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:32.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:33.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:33.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:33.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:33.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:33.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:33.378 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:33.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:33.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:33.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:33 smithi152 ceph-mon[93456]: pgmap v1048: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-08-17T17:30:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:33 smithi142 ceph-mon[106729]: pgmap v1048: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-08-17T17:30:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:33 smithi142 ceph-mon[103614]: pgmap v1048: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2024-08-17T17:30:33.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:33.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:33.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:33.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:33.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:33.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:33.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:34.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:34.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:34.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:34.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:34.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:34.377 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:34.394 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:30:34.394 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:34.417 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:30:34.432 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:30:34.432 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:34.468 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:30:34.469 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:34.479 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:34.511 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:30:34.511 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:34.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:34.535 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:30:34.535 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:34.566 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1542550103' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:34.566 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1542550103' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:34.566 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:30:34.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:34.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1542550103' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:34.600 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:30:34.600 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:30:34.616 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:34.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:34.631 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:30:34.631 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:34.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:34.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:34.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:34.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:34.869 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:34.907 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:35.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:35.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:35.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:35.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:35.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:35.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:35.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:35.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:35.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[103614]: pgmap v1049: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1542550103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[103614]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2617672099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:35.533 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[106729]: pgmap v1049: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T17:30:35.534 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:30:35.534 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:30:35.534 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1542550103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:35.534 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[106729]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:30:35.534 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2617672099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:35.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:35 smithi152 ceph-mon[93456]: pgmap v1049: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T17:30:35.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:35 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:30:35.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:35 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:30:35.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1542550103' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:35.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:35 smithi152 ceph-mon[93456]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:30:35.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2617672099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:35.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:35.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:35.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:35.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:35.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:35.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:35.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:36.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:36.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:36.124 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:36.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:36.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:36.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:36.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:36.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:36.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:36.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:36.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:36.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:36.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:36.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:36.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:37.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:37.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:37.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:37.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:30:37.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:37.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:37.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:37.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:37.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:37 smithi142 ceph-mon[106729]: pgmap v1051: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:30:37.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:37 smithi142 ceph-mon[103614]: pgmap v1051: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:30:37.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:37.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:37 smithi152 ceph-mon[93456]: pgmap v1051: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:30:37.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:37.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:37.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:37.783 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:37.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:37.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:38.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:38.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:30:38.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:38.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:38.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:38.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:38.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:38.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:38.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:38.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:38.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:38.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:38.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:38.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:38.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:38.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:38.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:39.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:39.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:39.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:39.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:39.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:39.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:39.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:39 smithi142 ceph-mon[106729]: pgmap v1052: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-08-17T17:30:39.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:39 smithi142 ceph-mon[103614]: pgmap v1052: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-08-17T17:30:39.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:39.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:39.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:39 smithi152 ceph-mon[93456]: pgmap v1052: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-08-17T17:30:39.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:39.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:39.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:39.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:39.837 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:39.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:39.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:40.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:40.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:40.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:40.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:40.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:40.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:40.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:40.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:40.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:30:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:30:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:40.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:40.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:40.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:40.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:40.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:40.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:40.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:41.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:41.059 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:41.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:41.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:41.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:41.331 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:41.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:41.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:41.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:41.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:41 smithi152 ceph-mon[93456]: pgmap v1053: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-08-17T17:30:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.595 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:41.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:41 smithi142 ceph-mon[103614]: pgmap v1053: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-08-17T17:30:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:41 smithi142 ceph-mon[106729]: pgmap v1053: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-08-17T17:30:41.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:41.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:41.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:41.830 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:41.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:41.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:41.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:41.997 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:42.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:30:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:30:41] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:30:42.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.125 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:42.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:42.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:42.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:42.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:42.378 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:42.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:42.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:42.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:42.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:42.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:42.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:42.797 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:42.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:42.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:42 smithi142 ceph-mon[106729]: pgmap v1054: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-08-17T17:30:42.798 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:42.798 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:42.798 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:42 smithi142 ceph-mon[103614]: pgmap v1054: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-08-17T17:30:42.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:42 smithi152 ceph-mon[93456]: pgmap v1054: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-08-17T17:30:42.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:42.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:42.956 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:42.956 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:43.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:43.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:43.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:43.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:43.221 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:43.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:43.334 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:43.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:43.464 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:30:43.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:43.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:43.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:43.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:43.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/102568309' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:30:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:43.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/102568309' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:30:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/102568309' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:30:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:43.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:44.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:44.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:30:44.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:44.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:44.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:44.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:44.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:44.355 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:44.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:44.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:44.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:44.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:44.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:44.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/102568309' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:30:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:44 smithi152 ceph-mon[93456]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:30:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:44 smithi152 ceph-mon[93456]: pgmap v1056: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T17:30:44.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/102568309' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:30:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:44 smithi142 ceph-mon[103614]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:30:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:44 smithi142 ceph-mon[103614]: pgmap v1056: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T17:30:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/102568309' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:30:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:44 smithi142 ceph-mon[106729]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:30:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:44 smithi142 ceph-mon[106729]: pgmap v1056: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T17:30:44.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:30:44.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:45.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:45.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:45.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:45.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:45.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:45.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:45.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:45.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:45.422 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:45.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:45.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:45.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:45.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:45.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:45.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:45.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:45.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:46.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:46.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:46.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:46.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:46.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:46.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:46.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:46.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:46.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:46.649 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:46.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:46.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:46.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:46.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:46.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:47.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:47.105 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:47.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:47.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:47.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:47 smithi152 ceph-mon[93456]: pgmap v1057: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-08-17T17:30:47.339 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:47 smithi142 ceph-mon[106729]: pgmap v1057: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-08-17T17:30:47.340 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:47 smithi142 ceph-mon[103614]: pgmap v1057: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-08-17T17:30:47.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:47.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:47.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:47.516 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:47.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:47.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:47.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:47.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:47.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:47.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:47.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:48.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:48.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:48.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:48.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:48.309 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:30:48.309 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:48.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:48.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:48.362 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:30:48.362 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:48.411 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:30:48.412 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:48.449 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:30:48.450 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:48.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:48.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:48.489 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:30:48.489 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:48.530 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:30:48.530 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:48.568 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:30:48.569 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:30:48.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:48.601 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:30:48.601 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:48.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:48.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:48.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:48.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:48.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:48.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:49.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:49.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:49.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:49.194 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:30:49.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:49.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:49.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:49 smithi142 ceph-mon[106729]: pgmap v1058: 89 pgs: 89 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T17:30:49.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2111754641' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:49.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2358950992' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:49 smithi142 ceph-mon[103614]: pgmap v1058: 89 pgs: 89 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T17:30:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2111754641' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2358950992' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:49.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:49.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:49.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:49.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:49 smithi152 ceph-mon[93456]: pgmap v1058: 89 pgs: 89 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T17:30:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2111754641' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2358950992' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:49.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:49.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:49.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:49.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:49.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:49.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:49.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:49.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:50.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:50.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:50.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:50.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:50.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:50.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:50.479 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:50.479 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2358950992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:50.479 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[103614]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:30:50.479 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:50.479 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:50.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:50.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2358950992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:50.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[106729]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:30:50.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:50.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:50.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:50.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:50 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2358950992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:50 smithi152 ceph-mon[93456]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:30:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:30:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:30:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:50.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:50.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:50.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:50.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:50.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:50.925 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:50.960 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:50.970 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:51.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:51.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:51.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:51.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:51.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:51.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:51.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:51 smithi142 ceph-mon[106729]: pgmap v1060: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T17:30:51.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:51 smithi142 ceph-mon[103614]: pgmap v1060: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T17:30:51.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:51.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:51.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:51.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:51 smithi152 ceph-mon[93456]: pgmap v1060: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T17:30:51.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:51.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:51.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:51.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:51.994 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:30:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:30:51] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:30:51.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:51.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:52.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:52.142 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:52.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:52.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:52.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:52.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:52.421 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:52.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:52.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:52.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:52.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:52.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:52.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:52.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:52.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:52.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:52.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:52.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:52.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:52.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:53.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:53.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:53.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:53.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:53.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:53.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:53.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:53.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:30:53.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:53.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:53.540 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:53 smithi142 ceph-mon[106729]: pgmap v1061: 89 pgs: 89 active+clean; 1.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2024-08-17T17:30:53.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:53 smithi142 ceph-mon[103614]: pgmap v1061: 89 pgs: 89 active+clean; 1.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2024-08-17T17:30:53.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:53 smithi152 ceph-mon[93456]: pgmap v1061: 89 pgs: 89 active+clean; 1.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2024-08-17T17:30:53.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:53.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:53.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:53.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:53.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:53.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:53.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:53.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:54.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:54.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:54.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:54.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:54.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:54.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:54.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:54.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:54.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:54.617 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:54.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:54.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:54.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:54.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:54.808 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:30:54.808 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:30:54.809 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:30:54.809 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:30:54.809 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:30:54.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:54.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:54.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:55.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:55.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:55.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:55.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:55.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:55.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:55.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:55.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:55.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:55.482 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:55 smithi152 ceph-mon[93456]: pgmap v1062: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:30:55.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:55.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:55.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:55 smithi142 ceph-mon[106729]: pgmap v1062: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:30:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:55 smithi142 ceph-mon[103614]: pgmap v1062: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:30:55.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:55.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:55.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:55.773 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:55.785 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:55.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:30:55.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:55.885 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:55.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:56.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:56.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.192 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:56.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:56.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:56.312 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:30:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:56.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:56.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/822747547' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:30:56.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:30:56.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/822747547' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:30:56.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:30:56.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:56.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/822747547' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:30:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:30:56.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:56.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:56.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:56.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:56.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:56.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:57.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:57.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:57.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:57.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:57.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:57.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:57.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:30:57.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.334 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:57.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:57.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:57.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:57.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:57 smithi152 ceph-mon[93456]: pgmap v1063: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:30:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:57 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:30:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:57 smithi152 ceph-mon[93456]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:30:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:57.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:57.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[106729]: pgmap v1063: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:30:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[106729]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[103614]: pgmap v1063: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[103614]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:30:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:57.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:57.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:57.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:57.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:57.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:57.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:57.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:58.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:58.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:58.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:58.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:58.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:58.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:58.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:58.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:58.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:58.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:58.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:58.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:58.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:58.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:30:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:58.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:30:58.899 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:58.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:59.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.076 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:59.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:59.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:59.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:59.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:59.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:59.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:59.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:59.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:30:59 smithi152 ceph-mon[93456]: pgmap v1065: 89 pgs: 89 active+clean; 1011 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:30:59.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:59.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:30:59 smithi142 ceph-mon[103614]: pgmap v1065: 89 pgs: 89 active+clean; 1011 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:30:59.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:30:59 smithi142 ceph-mon[106729]: pgmap v1065: 89 pgs: 89 active+clean; 1011 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-17T17:30:59.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:59.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:59.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:30:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:59.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:59.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:30:59.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:00.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:00.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:00.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:00.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:00.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:00.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:00.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:00.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:00.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:00.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:00.481 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:31:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:31:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:00.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:00.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:00.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:00.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:00.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:00.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:01.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:01.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:01.115 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:01.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:01.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:01.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:01.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:01.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:01.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:01.506 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-mon[103614]: pgmap v1066: 89 pgs: 89 active+clean; 866 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-08-17T17:31:01.506 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/510307483' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:01.506 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:01.506 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-mon[106729]: pgmap v1066: 89 pgs: 89 active+clean; 866 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-08-17T17:31:01.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/510307483' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:01.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:01.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:01 smithi152 ceph-mon[93456]: pgmap v1066: 89 pgs: 89 active+clean; 866 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-08-17T17:31:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/510307483' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:01 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:01.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:01.624 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:31:01.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:01.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:01.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:01.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:01.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:01.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:02.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:31:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:31:01] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:31:02.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:02.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:02.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:02.551 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:02 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:02.552 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:02.552 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:02 smithi142 ceph-mon[106729]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:31:02.552 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:02 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:02.552 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:02.552 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:02 smithi142 ceph-mon[103614]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:31:02.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:02.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:02 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:02.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:02.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:02 smithi152 ceph-mon[93456]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:31:02.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:02.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:02.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:02.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:02.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:02.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:02.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:02.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:02.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:03.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:03.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:03.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:03.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:03.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:03.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:03.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:03.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:03.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:03.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:03 smithi152 ceph-mon[93456]: pgmap v1068: 89 pgs: 89 active+clean; 803 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2024-08-17T17:31:03.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:03 smithi142 ceph-mon[106729]: pgmap v1068: 89 pgs: 89 active+clean; 803 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2024-08-17T17:31:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:03 smithi142 ceph-mon[103614]: pgmap v1068: 89 pgs: 89 active+clean; 803 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2024-08-17T17:31:03.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:03.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:03.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:03.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:03.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:03.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:04.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.065 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.167 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:04.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:04.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:04.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:04.462 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:04.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:04.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:04.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:04.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:04.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:04.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:04.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:04.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:04.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:05.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:05.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:05.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:05.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:05.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.394 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:05.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:05.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:05.524 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:05 smithi142 ceph-mon[106729]: pgmap v1069: 89 pgs: 89 active+clean; 757 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-08-17T17:31:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:05 smithi142 ceph-mon[103614]: pgmap v1069: 89 pgs: 89 active+clean; 757 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-08-17T17:31:05.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:05.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:05.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:05.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:05 smithi152 ceph-mon[93456]: pgmap v1069: 89 pgs: 89 active+clean; 757 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-08-17T17:31:05.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.917 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:06.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.057 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:06.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:06.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:06.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:06.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:06.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:06.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:06.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:06.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:06.468 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:06.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:06.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:06.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:06.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:06.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:06.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:06.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:06.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:07.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:07.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.159 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:07.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:07.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:07.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:07.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:07.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:07.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:07.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:07 smithi142 ceph-mon[103614]: pgmap v1070: 89 pgs: 89 active+clean; 574 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:31:07.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:07 smithi142 ceph-mon[106729]: pgmap v1070: 89 pgs: 89 active+clean; 574 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:31:07.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:07.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:07.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:07 smithi152 ceph-mon[93456]: pgmap v1070: 89 pgs: 89 active+clean; 574 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:31:07.856 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:07.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:08.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:08.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:08.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:08.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:08.225 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:08.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:08.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:08.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:08.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:08.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:31:08.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:08.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:08.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:08.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3624714430' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:08.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:08.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:08 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:08.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3624714430' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:08.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:08 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:08.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:08.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:08.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:08.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:31:08.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:08.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:08.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:08.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:08.733 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:08.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:08.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3624714430' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:08 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:08.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:08.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:08.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:08.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:08.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:09.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:09.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:09.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:09.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:09.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:09.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:09.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:09.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:09.516 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[103614]: pgmap v1071: 89 pgs: 89 active+clean; 510 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-08-17T17:31:09.516 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:09.516 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:09.517 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:09.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[106729]: pgmap v1071: 89 pgs: 89 active+clean; 510 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-08-17T17:31:09.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:09.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:09.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:09.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:09.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:09.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:09.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:09.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:09.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:09 smithi152 ceph-mon[93456]: pgmap v1071: 89 pgs: 89 active+clean; 510 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-08-17T17:31:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:09.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:09.919 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:10.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:10.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:10.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:10.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:10.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.284 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:10.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:10.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:10.371 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:31:10.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:10.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2277633543' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:31:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:31:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:31:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:31:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:10.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2277633543' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:31:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:31:10.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2277633543' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:31:10.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:31:10.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:10.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:10.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:10.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:10.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:10.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:10.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:11.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:11.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:11.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:11.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:11.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:31:11.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:11.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:11.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:11.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:11.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:11.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:11.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-mon[106729]: pgmap v1072: 89 pgs: 89 active+clean; 372 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-08-17T17:31:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:31:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-mon[106729]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:31:11.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-mon[103614]: pgmap v1072: 89 pgs: 89 active+clean; 372 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-08-17T17:31:11.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:31:11.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-mon[103614]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:31:11.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:11.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:11.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:11 smithi152 ceph-mon[93456]: pgmap v1072: 89 pgs: 89 active+clean; 372 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-08-17T17:31:11.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:11 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:31:11.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:11 smithi152 ceph-mon[93456]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:31:11.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:11.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:11.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:12.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:12.117 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:31:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:31:11] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-08-17T17:31:12.220 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:12.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:12.362 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:12.405 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:12.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:12.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:12.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:12.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:12.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:12.764 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:12.767 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:12.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:12.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:12.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:12.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:13.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:13.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:13.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:13.179 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:13.282 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:13.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:13.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:13.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:13.452 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:13 smithi142 ceph-mon[106729]: pgmap v1074: 89 pgs: 89 active+clean; 311 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-08-17T17:31:13.452 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:13 smithi142 ceph-mon[103614]: pgmap v1074: 89 pgs: 89 active+clean; 311 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-08-17T17:31:13.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:13.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:13.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:13.605 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:13.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:13.707 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:13.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:13 smithi152 ceph-mon[93456]: pgmap v1074: 89 pgs: 89 active+clean; 311 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-08-17T17:31:13.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:13.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:13.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:14.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:14.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:14.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:31:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:14.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:14.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:14.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:14.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:14.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:14.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:14.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:14 smithi142 ceph-mon[106729]: pgmap v1075: 89 pgs: 89 active+clean; 265 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-08-17T17:31:14.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:14 smithi142 ceph-mon[103614]: pgmap v1075: 89 pgs: 89 active+clean; 265 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-08-17T17:31:14.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:14.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:14.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:14 smithi152 ceph-mon[93456]: pgmap v1075: 89 pgs: 89 active+clean; 265 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-08-17T17:31:14.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:14.871 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:14.871 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:14.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:15.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:15.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:15.309 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:15.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:15.411 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:31:15.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:15.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2526544410' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:15.498 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:15 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:15.500 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2526544410' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:15.500 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:15 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:15.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:15.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:31:15.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:15.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:15.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:15.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2526544410' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:15 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:15.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:15.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:15.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:16.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:16.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:16.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:16.207 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:16.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.260 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:16.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:16.273 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:16.298 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 123416 objects 2024-08-17T17:31:16.298 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :152.656 2024-08-17T17:31:16.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:16.382 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:16.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:16.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:16.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:16.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:16.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:16.696 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:16.696 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:16.696 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[106729]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:31:16.696 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[106729]: pgmap v1077: 89 pgs: 89 active+clean; 90 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:31:16.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:16.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:16.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[103614]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:31:16.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:16 smithi142 ceph-mon[103614]: pgmap v1077: 89 pgs: 89 active+clean; 90 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:31:16.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:16.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:16.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:16 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:16 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:16 smithi152 ceph-mon[93456]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:31:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:16 smithi152 ceph-mon[93456]: pgmap v1077: 89 pgs: 89 active+clean; 90 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:31:16.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:16.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:16.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:17.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:17.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:17.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:17.171 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:17.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:17.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:17.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:17.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:17.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:17.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:17.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:17.625 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-08-17T17:31:17.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-08-17T17:31:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:17.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:17.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:17.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:17.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:17.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:17.877 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:17.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:18.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:18.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:18.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:18.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:18.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:31:18.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:18.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:18.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:18.481 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:18.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:18.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:18.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:18.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:18.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:18.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:18.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:18.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:19.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:19.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:19.109 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:19.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:19.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:19.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.260 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:19.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:19.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:19.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:19.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:19.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:19.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:19 smithi152 ceph-mon[93456]: pgmap v1078: 89 pgs: 89 active+clean; 50 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-08-17T17:31:19.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:19 smithi142 ceph-mon[106729]: pgmap v1078: 89 pgs: 89 active+clean; 50 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-08-17T17:31:19.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:19 smithi142 ceph-mon[103614]: pgmap v1078: 89 pgs: 89 active+clean; 50 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-08-17T17:31:19.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:19.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:19.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:19.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:19.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:19.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.024 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:20.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:20.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:20.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:20.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:20.275 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:20.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:20.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:20.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:20.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:20.523 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:31:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:31:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:20.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:20.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:20.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:20.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:20.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:20.899 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:21.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:21.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:21.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:21.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:21.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:21.269 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:21 smithi142 ceph-mon[106729]: pgmap v1079: 89 pgs: 89 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2024-08-17T17:31:21.269 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:21 smithi142 ceph-mon[103614]: pgmap v1079: 89 pgs: 89 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2024-08-17T17:31:21.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:21.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:21.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:21.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:21.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:21.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:21 smithi152 ceph-mon[93456]: pgmap v1079: 89 pgs: 89 active+clean; 8.6 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2024-08-17T17:31:21.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:21.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:21.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:21.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:21.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:21.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:21.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:21.966 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:31:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:31:21] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-08-17T17:31:22.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:22.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:22.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:22.266 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:22.266 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:22.273 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:22.375 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:22.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.719 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:31:22.719 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:31:22.719 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:22.719 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:22.724 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:31:22.724 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:22.725 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:22.743 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:31:22.744 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:22.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:22.760 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:31:22.761 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:22.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:22.779 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:31:22.779 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:22.794 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:31:22.794 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:31:22.810 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:31:22.810 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:22.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:22.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:22.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:23.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:23.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:23.147 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:23.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:31:23.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:23.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:23.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:23.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:23.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:23.419 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:23 smithi142 ceph-mon[106729]: pgmap v1080: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 497 op/s 2024-08-17T17:31:23.419 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3001849714' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:23.419 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:23 smithi142 ceph-mon[103614]: pgmap v1080: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 497 op/s 2024-08-17T17:31:23.419 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3001849714' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:23.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:23.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:23 smithi152 ceph-mon[93456]: pgmap v1080: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 497 op/s 2024-08-17T17:31:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3001849714' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:23.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:23.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:23.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:23.794 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:23.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:23.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:23.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:24.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:24.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:24.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:24.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:24.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:24.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:24.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:24.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:24.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:24.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:24.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:24.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:24.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:24.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:24.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:24.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:24.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:24.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:24.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:25.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:25.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:25.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:25.055 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:25.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:25.166 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:25.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:25.212 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:31:25.267 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:25.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:25.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:25.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:25.420 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:25 smithi142 ceph-mon[103614]: pgmap v1081: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 422 op/s 2024-08-17T17:31:25.421 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:25 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2887605281' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:31:25.421 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:25 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:31:25.421 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:25 smithi142 ceph-mon[106729]: pgmap v1081: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 422 op/s 2024-08-17T17:31:25.421 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:25 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2887605281' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:31:25.421 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:25 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:31:25.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:25.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:25 smithi152 ceph-mon[93456]: pgmap v1081: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 422 op/s 2024-08-17T17:31:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:25 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2887605281' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:31:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:25 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:31:25.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:25.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:25.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:25.771 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:25.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:25.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:25.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:26.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:26.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:26.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:26.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:31:26.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:26.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:26.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.300 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:26.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:26.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:26 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:31:26.354 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:26 smithi142 ceph-mon[106729]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:31:26.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:26 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:31:26.355 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:26 smithi142 ceph-mon[103614]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:31:26.361 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:26.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:26.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:26.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:26.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:26.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:26 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:31:26.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:26 smithi152 ceph-mon[93456]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:31:26.647 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:26.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:26.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:26.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:26.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:26.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:27.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:27.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:27.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:27.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:27.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:27.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:27.228 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_6' removed 2024-08-17T17:31:27.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:27.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:27.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:27.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:27.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:31:27.518 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:27.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:27 smithi152 ceph-mon[93456]: pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 143 op/s 2024-08-17T17:31:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1522200059' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:31:27.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:27 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:31:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[106729]: pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 143 op/s 2024-08-17T17:31:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1522200059' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:31:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:31:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[103614]: pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 143 op/s 2024-08-17T17:31:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1522200059' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:31:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:27 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:31:27.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:27.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:27.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:27.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:27.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:27.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:28.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:28.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:28.070 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:31:28.071 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:31:28.071 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2024-08-17T17:31:28.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_7 16 2024-08-17T17:31:28.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:28.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:28.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:28.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:28.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:28.303 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:31:28.303 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:28 smithi142 ceph-mon[103614]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:31:28.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:31:28.303 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:28 smithi142 ceph-mon[106729]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:31:28.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:28.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:28.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:28.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:28.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:28.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:31:28.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:28 smithi152 ceph-mon[93456]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:31:28.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:28.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:28.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:28.875 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:28.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:28.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:28.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:29.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:29.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:29.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:29.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:29.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:29.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:29.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:29.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:29.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:29.403 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:29.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:29.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:29.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:29.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:29 smithi142 ceph-mon[106729]: pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:29.507 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:29 smithi142 ceph-mon[103614]: pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.527 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:29.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:29.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:29 smithi152 ceph-mon[93456]: pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:29.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:29.649 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:29.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:29.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:29.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:29.839 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:29.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:29.895 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:29.997 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.020 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:30.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:30.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:30.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:30.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:30.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:30.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:30.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:30.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:31:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:31:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:30.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.640 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:30.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:30.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:30.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:30.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:30.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:31.018 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:31.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:31.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:31.146 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:31.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:31.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:31.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:31.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:31.415 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:31 smithi142 ceph-mon[106729]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:31.415 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:31 smithi142 ceph-mon[103614]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:31.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:31.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:31 smithi152 ceph-mon[93456]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:31.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:31.650 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:31.752 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:31.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:31.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:31.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:31.901 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:32.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:32.027 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:31:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:31:31] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.51.0" 2024-08-17T17:31:32.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:32.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:32.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:32.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:32.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:32.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:32.503 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:32.525 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:32.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:32.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:32.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:32.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:32.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:32.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:32.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:32.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:32.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:33.018 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:33.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:33.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:33.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:33.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:33.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:33.330 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:31:33.330 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:31:33.330 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:31:33.330 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:31:33.330 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:31:33.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:33.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:33 smithi142 ceph-mon[103614]: pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:33.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:33 smithi142 ceph-mon[106729]: pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:33.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:33.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:33.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:33.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:33 smithi152 ceph-mon[93456]: pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:33.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:33.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:33.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:33.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:33.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:33.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:31:33.903 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:33.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:34.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:34.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:34.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:34.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:34.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:34.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:34.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:34.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:34.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:34.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:34.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:34.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:34.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:34.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:34.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:34.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:34.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:34.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:34.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:35.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:35.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:35.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:35.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:35.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:35.255 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_7' created 2024-08-17T17:31:35.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:35.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:35.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:35.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:35.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:35.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:35.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:35.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:35 smithi152 ceph-mon[93456]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3468773678' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-08-17T17:31:35.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:35 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-08-17T17:31:35.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:35 smithi142 ceph-mon[106729]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:35.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3468773678' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-08-17T17:31:35.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:35 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-08-17T17:31:35.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:35 smithi142 ceph-mon[103614]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:35.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3468773678' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-08-17T17:31:35.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:35 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-08-17T17:31:35.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:35.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:35.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:35.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:35.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:35.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:35.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:36.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:36.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:36.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:36.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:36.171 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2024-08-17T17:31:36.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:36.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:36.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:36.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-08-17T17:31:36.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[106729]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:31:36.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3335019951' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:31:36.400 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:31:36.400 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-08-17T17:31:36.401 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[103614]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:31:36.401 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3335019951' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:31:36.401 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:36 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:31:36.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:36.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:36.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:36.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:36 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-08-17T17:31:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:36 smithi152 ceph-mon[93456]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:31:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3335019951' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:31:36.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:36 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:31:36.643 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:36.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:36.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:36.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:36.793 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:31:36.806 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:36.807 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:36.807 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:36.807 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:36.807 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:36.807 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:36.809 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:31:36.810 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:31:36.825 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:31:36.825 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:36.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:36.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:37.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:37.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:37.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:37.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:37.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:37.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:37.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:37.269 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:31:37.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:37.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:37.417 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: pgmap v1090: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1885187862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3563939337' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: pgmap v1090: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:37.511 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:37.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:31:37.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:31:37.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1885187862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:37.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3563939337' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:37.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:37 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:37.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:37.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: pgmap v1090: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1885187862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3563939337' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:37 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:37.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:37.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.669 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:37.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:37.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:37.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:37.820 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:31:37.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:37.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.916 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:37.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:38.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:38.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:38.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:38.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:38.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:38.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:38.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:38.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:38.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:38 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:38.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:38 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:38.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:38 smithi142 ceph-mon[103614]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:31:38.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:38 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:38.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:38 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:38.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:38 smithi142 ceph-mon[106729]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:31:38.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:38.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:38.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:31:38.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:38.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:38.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:38 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:38.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:38 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:38 smithi152 ceph-mon[93456]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:31:38.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:38.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:38.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:38.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:38.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:38.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:38.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:39.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:39.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:39.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:39.169 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:39.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:39.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:39.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:39.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:39.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:39.422 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:39 smithi142 ceph-mon[106729]: pgmap v1093: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:39.422 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:39 smithi142 ceph-mon[103614]: pgmap v1093: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:39.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:39.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:39.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:39 smithi152 ceph-mon[93456]: pgmap v1093: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:39.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:39.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:39.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:39.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:39.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:39.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:39.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:39.921 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:39.921 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:40.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:40.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:40.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:40.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:40.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:40.213 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:31:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=cleanup t=2024-08-17T17:31:39.870493856Z level=info msg="Completed cleanup jobs" duration=12.513677ms 2024-08-17T17:31:40.267 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:40.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:40.390 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:40.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:40.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:40.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:31:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:31:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:40.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:40.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:40.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:31:40.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:40.796 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:40.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:40.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:41.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:31:41.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:41.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:41 smithi142 ceph-mon[103614]: pgmap v1094: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:41.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:41 smithi142 ceph-mon[106729]: pgmap v1094: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:41.390 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:41.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:41.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:41.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:41.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:41 smithi152 ceph-mon[93456]: pgmap v1094: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:41.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:41.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:41.677 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:41.677 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:41.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:41.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:41.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:31:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:31:41] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-08-17T17:31:41.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:41.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:42.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:42.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:42.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:42.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:42.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:42.262 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:42.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:42.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:42.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:42.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:42.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:42.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:42.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:42.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:42.759 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:42.759 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:42.759 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:42 smithi142 ceph-mon[106729]: pgmap v1095: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:42.760 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:42.760 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:42.760 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:42 smithi142 ceph-mon[103614]: pgmap v1095: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:42.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:42.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:31:42.793 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:42 smithi152 ceph-mon[93456]: pgmap v1095: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:42.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:42.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:43.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:43.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:43.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:43.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:43.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:43.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:43.296 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:43.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:43.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:43.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:43.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:43.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:43.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:43.611 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:43.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:43.670 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:43.772 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:43.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:43.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:43.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:43.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:44.023 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:44.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:44.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:44.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:44.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.424 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.677 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:44.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:44.815 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:44.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:44.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:44.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:45.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:45.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:45.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:45.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:45.207 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_7' 2024-08-17T17:31:45.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:45.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:45.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:45.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:45.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:45.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:45.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:45 smithi152 ceph-mon[93456]: pgmap v1096: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/754674755' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.587 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:45 smithi142 ceph-mon[106729]: pgmap v1096: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:45.588 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/754674755' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.588 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:45 smithi142 ceph-mon[103614]: pgmap v1096: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:45.588 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/754674755' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:45.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:45.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:45.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:45.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:45.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:45.956 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:46.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:46.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:46.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:46.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:46.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:46.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:46.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:46.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:46.510 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/754674755' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:46.510 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:46 smithi142 ceph-mon[106729]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:31:46.510 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/754674755' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:46.510 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:46 smithi142 ceph-mon[103614]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:31:46.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:46.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:46.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/754674755' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:46.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:46 smithi152 ceph-mon[93456]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:31:46.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:46.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:46.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:46.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:46.913 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2024-08-17T17:31:46.917 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:31:46.917 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:31:46.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:46.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:46.964 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:31:46.964 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:31:46.964 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_261410 2024-08-17T17:31:46.964 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:31:46.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:31:47.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:47.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:47.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:47.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:47.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:47.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:47.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[106729]: pgmap v1098: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:47.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:47.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:31:47.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[103614]: pgmap v1098: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:31:47.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:47.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:47.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:47.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:47 smithi152 ceph-mon[93456]: pgmap v1098: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:31:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:47 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:47 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:31:47.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:47.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:47.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:47.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:47.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:47.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:47.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:47.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 3655 3639 227.426 227.438 0.00275113 0.00438486 2024-08-17T17:31:48.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:48.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:48.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:48.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:48.242 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:31:48.281 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:48.309 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:48.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:48.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:48.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:48.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2757328975' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:31:48.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:48.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2757328975' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:31:48.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2757328975' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:31:48.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:48.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:48.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:48.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:48.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:48.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:48.964 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 6933 6917 216.142 204.875 0.00513264 0.00461537 2024-08-17T17:31:48.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:49.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:49.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:49.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.258 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:31:49.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:49.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:49.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:49.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:49.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:49 smithi152 ceph-mon[93456]: pgmap v1099: 89 pgs: 89 active+clean; 129 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s wr, 204 op/s 2024-08-17T17:31:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2757328975' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:31:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:49 smithi152 ceph-mon[93456]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:31:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3915703842' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[106729]: pgmap v1099: 89 pgs: 89 active+clean; 129 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s wr, 204 op/s 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2757328975' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[106729]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3915703842' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[103614]: pgmap v1099: 89 pgs: 89 active+clean; 129 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s wr, 204 op/s 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2757328975' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[103614]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3915703842' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:49.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:49.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:49.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:49.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:49.964 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 9985 9969 207.672 190.75 0.00493846 0.00480658 2024-08-17T17:31:50.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:50.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:50.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:50.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:50.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:50.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:50.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:50 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:50.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:50.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:50 smithi152 ceph-mon[93456]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:31:50.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:31:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:31:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:50.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:50 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:50 smithi142 ceph-mon[103614]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:31:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:50 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:50 smithi142 ceph-mon[106729]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:31:50.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:50.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:50.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:50.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 13239 13223 206.593 203.375 0.0017766 0.00482367 2024-08-17T17:31:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:50.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:51.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:51.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:51.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:51.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:51.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:51.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:51.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:51.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:51.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:51.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:51 smithi152 ceph-mon[93456]: pgmap v1102: 89 pgs: 89 active+clean; 274 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2024-08-17T17:31:51.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:51 smithi142 ceph-mon[106729]: pgmap v1102: 89 pgs: 89 active+clean; 274 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2024-08-17T17:31:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:51 smithi142 ceph-mon[103614]: pgmap v1102: 89 pgs: 89 active+clean; 274 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2024-08-17T17:31:51.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:51.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:51.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:51.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:31:51.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:51.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:51.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 15339 15323 191.522 131.25 0.00256309 0.00521182 2024-08-17T17:31:52.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:52.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:52.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:31:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:31:51] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-08-17T17:31:52.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:52.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:52.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:52.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:52.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:52.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:52.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:52.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:52.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:52.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:52.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:52.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:52.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:52.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:52.934 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:52.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:52.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:52.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 17830 17814 185.547 155.688 0.00237555 0.00538196 2024-08-17T17:31:53.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:53.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:53.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:53.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:53.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:53.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:53.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:53.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:53.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:53.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:53.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:53.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:53 smithi152 ceph-mon[93456]: pgmap v1103: 89 pgs: 89 active+clean; 421 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T17:31:53.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:53 smithi142 ceph-mon[106729]: pgmap v1103: 89 pgs: 89 active+clean; 421 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T17:31:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:53 smithi142 ceph-mon[103614]: pgmap v1103: 89 pgs: 89 active+clean; 421 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T17:31:53.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:53.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:53.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:53.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:53.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:53.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 19707 19691 175.798 117.312 0.00750487 0.00568132 2024-08-17T17:31:53.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:54.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:54.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:54.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:54.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:54.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:54.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:54.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:54.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:54.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:54.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:54.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:54.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.829 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:54.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:54.931 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 21974 21958 171.533 141.688 0.00587079 0.00582266 2024-08-17T17:31:54.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:55.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:55.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:55.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:55.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:55.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:55.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:55.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:55.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:55.563 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:55 smithi142 ceph-mon[106729]: pgmap v1104: 89 pgs: 89 active+clean; 914 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2024-08-17T17:31:55.563 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:55 smithi142 ceph-mon[103614]: pgmap v1104: 89 pgs: 89 active+clean; 914 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2024-08-17T17:31:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:55 smithi152 ceph-mon[93456]: pgmap v1104: 89 pgs: 89 active+clean; 914 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2024-08-17T17:31:55.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:55.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:55.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:55.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:55.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:55.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:55.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 24357 24341 169.017 148.938 0.00703695 0.00591067 2024-08-17T17:31:56.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:56.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:56.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:56.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:56.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:56.332 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:56.333 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:56.336 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:56.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:56.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:56.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:56.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:56.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:56.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:56.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:56.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:56.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:56.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:56.965 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 26405 26389 164.914 128 0.0033455 0.00605828 2024-08-17T17:31:57.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:57.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:57.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:57.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:57.206 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:31:57.206 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:57.243 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:31:57.244 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:57.284 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:31:57.284 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:57.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:57.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:31:57.318 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:31:57.318 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:57.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:57.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:57.354 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:31:57.354 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:57.382 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:31:57.383 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:57.419 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:31:57.420 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:31:57.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:57.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:57.466 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:31:57.466 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:57.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:57.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:57 smithi152 ceph-mon[93456]: pgmap v1105: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-08-17T17:31:57.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3893357022' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:57.586 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[106729]: pgmap v1105: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-08-17T17:31:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3893357022' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:57.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[103614]: pgmap v1105: 89 pgs: 89 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-08-17T17:31:57.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:31:57.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:57.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3893357022' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:57.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:57.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:57.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:57.869 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:57.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 28304 28288 160.71 118.688 0.00196018 0.00619867 2024-08-17T17:31:57.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:58.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:58.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:58.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:58.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:58.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:58.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:58.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:58.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:58.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:58.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:58.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:58.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:58.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:58.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:58.922 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:58.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 30284 30268 157.629 123.75 0.00384778 0.00633856 2024-08-17T17:31:59.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:59.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:59.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:59.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:59.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:59.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:59.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:59.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:59.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:59.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:31:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:31:59 smithi152 ceph-mon[93456]: pgmap v1106: 89 pgs: 89 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:31:59.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:31:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:31:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:31:59 smithi142 ceph-mon[103614]: pgmap v1106: 89 pgs: 89 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:31:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:31:59 smithi142 ceph-mon[106729]: pgmap v1106: 89 pgs: 89 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:31:59.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:59.717 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:59.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:59.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:59.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:59.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:59.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 31924 31908 153.388 102.5 0.00719511 0.00649914 2024-08-17T17:31:59.978 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:31:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:00.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:00.254 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:00.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:00.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:00.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:00.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:32:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:32:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:00.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:00.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:00.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:00.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:00.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:00.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:00.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 33966 33950 151.547 127.625 0.00366669 0.00659232 2024-08-17T17:32:01.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:01.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:01.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:01.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:01.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:01.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:01.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:01.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:01.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:01.565 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:01 smithi142 ceph-mon[106729]: pgmap v1107: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:32:01.566 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:01 smithi142 ceph-mon[103614]: pgmap v1107: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:32:01.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:01 smithi152 ceph-mon[93456]: pgmap v1107: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:32:01.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:01.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:32:01.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:01.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:01.833 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:32:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:32:01] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-08-17T17:32:01.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:01.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:01.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 35724 35708 148.768 109.875 0.00543568 0.00671642 2024-08-17T17:32:02.006 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:02.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:02.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:02.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:02.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:02.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:02.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:02.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:02.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:02.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:02.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:02.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:02.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:02.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:02.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:02.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:02.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:02.953 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:02.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 37679 37663 147.104 122.188 0.017412 0.00679027 2024-08-17T17:32:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:02.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:03.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:03.131 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:03.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:03.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:03.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:03.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:03.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:03.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:03.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:03.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:03 smithi152 ceph-mon[93456]: pgmap v1108: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-08-17T17:32:03.601 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:03.604 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:03.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:03 smithi142 ceph-mon[106729]: pgmap v1108: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-08-17T17:32:03.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:03 smithi142 ceph-mon[103614]: pgmap v1108: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-08-17T17:32:03.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:03.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:03.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:03.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:03.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:03.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 39495 39479 145.127 113.5 0.0141619 0.0068844 2024-08-17T17:32:03.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:03.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:04.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:04.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:04.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:04.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:04.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:04.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:04.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:04.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:04.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:04.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:04.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:04.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:04.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:04.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:04.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:04.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:04.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:04.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:04.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 41054 41038 142.477 97.4375 0.0155823 0.00701216 2024-08-17T17:32:05.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:05.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:05.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:05.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:05.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:05.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:05.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:05.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:05.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:05.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:05.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:05.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:05 smithi152 ceph-mon[93456]: pgmap v1109: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:32:05.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/46685602' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:05.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:05.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:05 smithi142 ceph-mon[106729]: pgmap v1109: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:32:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/46685602' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:05 smithi142 ceph-mon[103614]: pgmap v1109: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-08-17T17:32:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/46685602' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:32:05.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:05.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:05.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:05.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:05.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:05.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:05.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:05.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 42387 42371 139.363 83.3125 0.0044253 0.00716654 2024-08-17T17:32:05.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:05.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:06.079 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:06.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:06.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:06.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:06.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:06.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:06.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:06.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:06.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:06.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:06.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:06.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/46685602' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:32:06.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:06 smithi152 ceph-mon[93456]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:32:06.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/46685602' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:32:06.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:06 smithi142 ceph-mon[106729]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:32:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/46685602' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:32:06.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:06 smithi142 ceph-mon[103614]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:32:06.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:06.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:06.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:06.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:06.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:06.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:06.966 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:32:06.966944+0000 min lat: 0.00117681 max lat: 0.230254 avg lat: 0.00727635 2024-08-17T17:32:06.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:32:06.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 43966 43950 137.329 98.6875 0.00759393 0.00727635 2024-08-17T17:32:07.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:07.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:07.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:07.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:07.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:07.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:07.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:07.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:07.478 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:07.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:07.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:07.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:07 smithi142 ceph-mon[106729]: pgmap v1111: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-08-17T17:32:07.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:07 smithi142 ceph-mon[103614]: pgmap v1111: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-08-17T17:32:07.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:07.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:07 smithi152 ceph-mon[93456]: pgmap v1111: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-08-17T17:32:07.830 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:32:07.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:07.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:07.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 45709 45693 135.976 108.938 0.0141749 0.00734889 2024-08-17T17:32:07.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:08.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:08.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:08.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:08.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:08.306 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:08.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:08.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:08.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:08.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:08.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:08.626 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:32:08.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:32:08.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:08.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:08.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:08.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:08.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:08.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 47340 47324 134.427 101.938 0.0113311 0.00743286 2024-08-17T17:32:08.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:08.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:09.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:09.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:09.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:09.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:09.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.414 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:09.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:09.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:09.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:09.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:09.590 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:09.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:09.666 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:32:09.667 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:32:09.667 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:32:09.667 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:32:09.667 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:32:09.668 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:09 smithi142 ceph-mon[103614]: pgmap v1112: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:32:09.668 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:09 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:09.668 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:09 smithi142 ceph-mon[106729]: pgmap v1112: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:32:09.668 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:09 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:09.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:09.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:09 smithi152 ceph-mon[93456]: pgmap v1112: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-08-17T17:32:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:09 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:09.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:09.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:09.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 48825 48809 132.617 92.8125 0.0103176 0.00753502 2024-08-17T17:32:09.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:09.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:10.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:10.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:10.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:10.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:10.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:10.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:10.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:10.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:32:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:32:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:10.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:10.662 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:10.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:10.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:10.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:10.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:10.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 50095 50079 130.398 79.375 0.00240894 0.00759846 2024-08-17T17:32:11.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:11.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:11.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:11.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:11.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:11.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:11.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:11.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:11.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:11.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:11.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:11.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:11 smithi142 ceph-mon[106729]: pgmap v1113: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-08-17T17:32:11.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:11 smithi142 ceph-mon[103614]: pgmap v1113: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-08-17T17:32:11.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:11.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:11 smithi152 ceph-mon[93456]: pgmap v1113: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-08-17T17:32:11.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:11.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:11.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:11.967 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 51580 51564 128.895 92.8125 0.00532875 0.00775157 2024-08-17T17:32:11.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:12.024 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:12.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:12.093 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:32:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:32:11] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:32:12.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:12.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:32:12.183 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:12.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.314 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:12.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:12.346 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:12.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:12.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:12.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:12.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:12.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:12.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:12.588 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:12.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:12.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:12.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:12.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:12.968 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 52877 52861 127.055 81.0625 0.00521601 0.00786499 2024-08-17T17:32:12.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:13.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:13.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:13.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:13.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:13.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:13.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:13.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:13.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:13.563 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:13.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:13 smithi152 ceph-mon[93456]: pgmap v1114: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-17T17:32:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:13 smithi142 ceph-mon[106729]: pgmap v1114: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-17T17:32:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:13 smithi142 ceph-mon[103614]: pgmap v1114: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-17T17:32:13.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:13.717 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:13.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:13.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:13.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:13.968 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 54255 54239 125.538 86.125 0.00598404 0.00795952 2024-08-17T17:32:13.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:14.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:14.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:14.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:14.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:14.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:14.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:14.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:14.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:14.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:14.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:14.968 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 55666 55650 124.204 88.1875 0.0133713 0.00804482 2024-08-17T17:32:15.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:15.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:15.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:15.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:15.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:15.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:15.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:15.483 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:15.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:15.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:15 smithi142 ceph-mon[103614]: pgmap v1115: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:32:15.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:15.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:15.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:15 smithi142 ceph-mon[106729]: pgmap v1115: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:32:15.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:15.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:15.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:15.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:15.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:15.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:15 smithi152 ceph-mon[93456]: pgmap v1115: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:32:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:15.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:15.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.913 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:15.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:15.968 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 56931 56915 122.647 79.0625 0.006208 0.0081477 2024-08-17T17:32:16.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:16.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:16.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:16.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:16.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:16.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:16.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:16.370 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:32:16.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:16.428 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:16.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:16.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:16.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/15289599' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/15289599' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:16.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:16.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:16.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:16.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/15289599' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:16.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:16.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:16.968 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 57833 57817 120.438 56.375 0.0110197 0.008298 2024-08-17T17:32:16.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:17.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:17.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:17.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:17.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:17.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:17.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:17.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:17.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:17.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:17.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:17.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:17.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[106729]: pgmap v1116: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:32:17.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:17.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/15289599' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:17.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[106729]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:32:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[103614]: pgmap v1116: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:32:17.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:17.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/15289599' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:17.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:17 smithi142 ceph-mon[103614]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:32:17.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:17.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:17.783 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:17 smithi152 ceph-mon[93456]: pgmap v1116: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:32:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:17 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/15289599' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:17 smithi152 ceph-mon[93456]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:32:17.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:17.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:17.968 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 59274 59258 119.457 90.0625 0.0168376 0.00836659 2024-08-17T17:32:18.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:18.041 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:18.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:18.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:18.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:18.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:18.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:18.551 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:18.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:18.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:18.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:18.733 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:18.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:18.969 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 60722 60706 118.548 90.5 0.00881071 0.0084294 2024-08-17T17:32:19.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:19.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:19.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:19.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:19.300 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:19.402 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:19.439 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:19.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:19.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:19.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:19 smithi142 ceph-mon[103614]: pgmap v1118: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:32:19.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:19 smithi142 ceph-mon[106729]: pgmap v1118: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:32:19.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:19.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:19 smithi152 ceph-mon[93456]: pgmap v1118: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:32:19.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:19.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:19.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:19.969 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 62073 62057 117.514 84.4375 0.01217 0.00850428 2024-08-17T17:32:19.996 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:20.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:20.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:20.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:20.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:20.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:20.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:20.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:20.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:20.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:32:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:32:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:20.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:20.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:20.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:20.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:20.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:20.943 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:32:20.943 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:20.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:20.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:20.969 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 63400 63384 116.497 82.9375 0.0106108 0.00857788 2024-08-17T17:32:20.975 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:32:20.975 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:21.007 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:32:21.007 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:21.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:32:21.053 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:32:21.053 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:21.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:21.087 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:32:21.087 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:21.127 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:32:21.128 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:21.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:21.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:21.168 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:32:21.168 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:21.199 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:32:21.199 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:21.231 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:21.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:21.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:21.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:21.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:21.553 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:21 smithi142 ceph-mon[103614]: pgmap v1119: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-08-17T17:32:21.553 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/856923170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:21.554 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:21 smithi142 ceph-mon[106729]: pgmap v1119: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-08-17T17:32:21.554 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/856923170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:21.557 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:32:21.559 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:32:21.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:21.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:21.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:21.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:21.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:21 smithi152 ceph-mon[93456]: pgmap v1119: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-08-17T17:32:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/856923170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:21.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:32:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:32:21] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:32:21.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:21.971 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 64543 64527 115.203 71.4375 0.0137657 0.0086735 2024-08-17T17:32:21.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:21.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:22.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:22.110 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:22.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:22.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:22.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:22.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:22.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:22.512 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:22.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:22.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:22.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:22.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:22.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:22.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:22.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:22.975 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 65837 65821 114.24 80.875 0.0124436 0.00874798 2024-08-17T17:32:23.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:23.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:23.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:23.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:23.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:23.305 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:23.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:23.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:23.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:23.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:23.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:23.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:23.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:23 smithi152 ceph-mon[93456]: from='client.46751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:32:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:23 smithi152 ceph-mon[93456]: pgmap v1120: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:32:23.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:23.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:23 smithi142 ceph-mon[106729]: from='client.46751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:32:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:23 smithi142 ceph-mon[106729]: pgmap v1120: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:32:23.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:23 smithi142 ceph-mon[103614]: from='client.46751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:32:23.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:23 smithi142 ceph-mon[103614]: pgmap v1120: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:32:23.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:23.975 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 67115 67099 113.309 79.875 0.0114508 0.00881927 2024-08-17T17:32:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:24.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:24.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:24.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:24.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:24.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:24.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:24.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:24.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:24.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:24.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:24.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:24.642 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:24.680 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1119,"stamp":"2024-08-17T17:32:20.170261+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3890485197,"num_objects":59381,"num_object_clones":0,"num_object_copies":178143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59381,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":59633,"num_write_kb":3801167,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2362634,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59562,"ondisk_log_size":59562,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17265972,"kb_used_data":11386988,"kb_used_omap":45,"kb_used_meta":5878802,"kb_avail":732695244,"statfs":{"total":767960285184,"available":750279929856,"internally_reserved":0,"allocated":11660275712,"data_stored":11647112023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47063,"internal_metadata":6019893289},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,6,6,3,0,4,2,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":896073728,"num_objects":13673,"num_object_clones":0,"num_object_copies":41019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13673,"num_write_kb":875072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006948"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":477,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375971+0000","last_change":"2024-08-17T17:26:32.212092+0000","last_active":"2024-08-17T17:32:16.375971+0000","last_peered":"2024-08-17T17:32:16.375971+0000","last_clean":"2024-08-17T17:32:16.375971+0000","last_became_active":"2024-08-17T17:26:32.211774+0000","last_became_peered":"2024-08-17T17:26:32.211774+0000","last_unstale":"2024-08-17T17:32:16.375971+0000","last_undegraded":"2024-08-17T17:32:16.375971+0000","last_fullsized":"2024-08-17T17:32:16.375971+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:45.518565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":270,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256629+0000","last_change":"2024-08-17T17:26:32.210903+0000","last_active":"2024-08-17T17:31:49.256629+0000","last_peered":"2024-08-17T17:31:49.256629+0000","last_clean":"2024-08-17T17:31:49.256629+0000","last_became_active":"2024-08-17T17:26:32.209823+0000","last_became_peered":"2024-08-17T17:26:32.209823+0000","last_unstale":"2024-08-17T17:31:49.256629+0000","last_undegraded":"2024-08-17T17:31:49.256629+0000","last_fullsized":"2024-08-17T17:31:49.256629+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:54:17.717352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":219,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.330065+0000","last_change":"2024-08-17T17:26:32.205305+0000","last_active":"2024-08-17T17:32:05.330065+0000","last_peered":"2024-08-17T17:32:05.330065+0000","last_clean":"2024-08-17T17:32:05.330065+0000","last_became_active":"2024-08-17T17:26:32.205000+0000","last_became_peered":"2024-08-17T17:26:32.205000+0000","last_unstale":"2024-08-17T17:32:05.330065+0000","last_undegraded":"2024-08-17T17:32:05.330065+0000","last_fullsized":"2024-08-17T17:32:05.330065+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:31.144881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256041+0000","last_change":"2024-08-17T17:26:32.213319+0000","last_active":"2024-08-17T17:31:49.256041+0000","last_peered":"2024-08-17T17:31:49.256041+0000","last_clean":"2024-08-17T17:31:49.256041+0000","last_became_active":"2024-08-17T17:26:32.213079+0000","last_became_peered":"2024-08-17T17:26:32.213079+0000","last_unstale":"2024-08-17T17:31:49.256041+0000","last_undegraded":"2024-08-17T17:31:49.256041+0000","last_fullsized":"2024-08-17T17:31:49.256041+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:43.552406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":359,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376036+0000","last_change":"2024-08-17T17:26:32.211539+0000","last_active":"2024-08-17T17:32:16.376036+0000","last_peered":"2024-08-17T17:32:16.376036+0000","last_clean":"2024-08-17T17:32:16.376036+0000","last_became_active":"2024-08-17T17:26:32.211267+0000","last_became_peered":"2024-08-17T17:26:32.211267+0000","last_unstale":"2024-08-17T17:32:16.376036+0000","last_undegraded":"2024-08-17T17:32:16.376036+0000","last_fullsized":"2024-08-17T17:32:16.376036+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:16.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":239,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256448+0000","last_change":"2024-08-17T17:26:32.233887+0000","last_active":"2024-08-17T17:31:49.256448+0000","last_peered":"2024-08-17T17:31:49.256448+0000","last_clean":"2024-08-17T17:31:49.256448+0000","last_became_active":"2024-08-17T17:26:32.230472+0000","last_became_peered":"2024-08-17T17:26:32.230472+0000","last_unstale":"2024-08-17T17:31:49.256448+0000","last_undegraded":"2024-08-17T17:31:49.256448+0000","last_fullsized":"2024-08-17T17:31:49.256448+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:09.045150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":453,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376184+0000","last_change":"2024-08-17T17:26:32.207139+0000","last_active":"2024-08-17T17:32:16.376184+0000","last_peered":"2024-08-17T17:32:16.376184+0000","last_clean":"2024-08-17T17:32:16.376184+0000","last_became_active":"2024-08-17T17:26:32.206763+0000","last_became_peered":"2024-08-17T17:26:32.206763+0000","last_unstale":"2024-08-17T17:32:16.376184+0000","last_undegraded":"2024-08-17T17:32:16.376184+0000","last_fullsized":"2024-08-17T17:32:16.376184+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:51.396647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":346,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.318622+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:32:05.318622+0000","last_peered":"2024-08-17T17:32:05.318622+0000","last_clean":"2024-08-17T17:32:05.318622+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:32:05.318622+0000","last_undegraded":"2024-08-17T17:32:05.318622+0000","last_fullsized":"2024-08-17T17:32:05.318622+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"326'3652","reported_seq":3667,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:17.652138+0000","last_change":"2024-08-17T17:31:36.270953+0000","last_active":"2024-08-17T17:32:17.652138+0000","last_peered":"2024-08-17T17:32:17.652138+0000","last_clean":"2024-08-17T17:32:17.652138+0000","last_became_active":"2024-08-17T17:31:36.270636+0000","last_became_peered":"2024-08-17T17:31:36.270636+0000","last_unstale":"2024-08-17T17:32:17.652138+0000","last_undegraded":"2024-08-17T17:32:17.652138+0000","last_fullsized":"2024-08-17T17:32:17.652138+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3652,"log_dups_size":0,"ondisk_log_size":3652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:27.516649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239337472,"num_objects":3652,"num_object_clones":0,"num_object_copies":10956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3652,"num_write_kb":233728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":301,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256828+0000","last_change":"2024-08-17T17:26:32.214168+0000","last_active":"2024-08-17T17:31:49.256828+0000","last_peered":"2024-08-17T17:31:49.256828+0000","last_clean":"2024-08-17T17:31:49.256828+0000","last_became_active":"2024-08-17T17:26:32.213844+0000","last_became_peered":"2024-08-17T17:26:32.213844+0000","last_unstale":"2024-08-17T17:31:49.256828+0000","last_undegraded":"2024-08-17T17:31:49.256828+0000","last_fullsized":"2024-08-17T17:31:49.256828+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:52.468566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":202,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329729+0000","last_change":"2024-08-17T17:26:32.218633+0000","last_active":"2024-08-17T17:32:05.329729+0000","last_peered":"2024-08-17T17:32:05.329729+0000","last_clean":"2024-08-17T17:32:05.329729+0000","last_became_active":"2024-08-17T17:26:32.218212+0000","last_became_peered":"2024-08-17T17:26:32.218212+0000","last_unstale":"2024-08-17T17:32:05.329729+0000","last_undegraded":"2024-08-17T17:32:05.329729+0000","last_fullsized":"2024-08-17T17:32:05.329729+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:55.971779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"326'3728","reported_seq":3743,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:18.104612+0000","last_change":"2024-08-17T17:31:36.269016+0000","last_active":"2024-08-17T17:32:18.104612+0000","last_peered":"2024-08-17T17:32:18.104612+0000","last_clean":"2024-08-17T17:32:18.104612+0000","last_became_active":"2024-08-17T17:31:36.268667+0000","last_became_peered":"2024-08-17T17:31:36.268667+0000","last_unstale":"2024-08-17T17:32:18.104612+0000","last_undegraded":"2024-08-17T17:32:18.104612+0000","last_fullsized":"2024-08-17T17:32:18.104612+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3728,"log_dups_size":0,"ondisk_log_size":3728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:01:46.882039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244318208,"num_objects":3728,"num_object_clones":0,"num_object_copies":11184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3728,"num_write_kb":238592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":256,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256403+0000","last_change":"2024-08-17T17:26:32.220489+0000","last_active":"2024-08-17T17:31:49.256403+0000","last_peered":"2024-08-17T17:31:49.256403+0000","last_clean":"2024-08-17T17:31:49.256403+0000","last_became_active":"2024-08-17T17:26:32.220150+0000","last_became_peered":"2024-08-17T17:26:32.220150+0000","last_unstale":"2024-08-17T17:31:49.256403+0000","last_undegraded":"2024-08-17T17:31:49.256403+0000","last_fullsized":"2024-08-17T17:31:49.256403+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:09.954487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321297+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:32:05.321297+0000","last_peered":"2024-08-17T17:32:05.321297+0000","last_clean":"2024-08-17T17:32:05.321297+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:32:05.321297+0000","last_undegraded":"2024-08-17T17:32:05.321297+0000","last_fullsized":"2024-08-17T17:32:05.321297+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"326'3665","reported_seq":3680,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:17.649761+0000","last_change":"2024-08-17T17:31:36.268659+0000","last_active":"2024-08-17T17:32:17.649761+0000","last_peered":"2024-08-17T17:32:17.649761+0000","last_clean":"2024-08-17T17:32:17.649761+0000","last_became_active":"2024-08-17T17:31:36.268347+0000","last_became_peered":"2024-08-17T17:31:36.268347+0000","last_unstale":"2024-08-17T17:32:17.649761+0000","last_undegraded":"2024-08-17T17:32:17.649761+0000","last_fullsized":"2024-08-17T17:32:17.649761+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3665,"log_dups_size":0,"ondisk_log_size":3665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:30.225892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240189440,"num_objects":3665,"num_object_clones":0,"num_object_copies":10995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3665,"num_write_kb":234560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":266,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256944+0000","last_change":"2024-08-17T17:26:32.216517+0000","last_active":"2024-08-17T17:31:49.256944+0000","last_peered":"2024-08-17T17:31:49.256944+0000","last_clean":"2024-08-17T17:31:49.256944+0000","last_became_active":"2024-08-17T17:26:32.216161+0000","last_became_peered":"2024-08-17T17:26:32.216161+0000","last_unstale":"2024-08-17T17:31:49.256944+0000","last_undegraded":"2024-08-17T17:31:49.256944+0000","last_fullsized":"2024-08-17T17:31:49.256944+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:39.712941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.318648+0000","last_change":"2024-08-17T17:26:32.217706+0000","last_active":"2024-08-17T17:32:05.318648+0000","last_peered":"2024-08-17T17:32:05.318648+0000","last_clean":"2024-08-17T17:32:05.318648+0000","last_became_active":"2024-08-17T17:26:32.217107+0000","last_became_peered":"2024-08-17T17:26:32.217107+0000","last_unstale":"2024-08-17T17:32:05.318648+0000","last_undegraded":"2024-08-17T17:32:05.318648+0000","last_fullsized":"2024-08-17T17:32:05.318648+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:39.827920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"326'3793","reported_seq":3817,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:17.988809+0000","last_change":"2024-08-17T17:31:36.271004+0000","last_active":"2024-08-17T17:32:17.988809+0000","last_peered":"2024-08-17T17:32:17.988809+0000","last_clean":"2024-08-17T17:32:17.988809+0000","last_became_active":"2024-08-17T17:31:36.270454+0000","last_became_peered":"2024-08-17T17:31:36.270454+0000","last_unstale":"2024-08-17T17:32:17.988809+0000","last_undegraded":"2024-08-17T17:32:17.988809+0000","last_fullsized":"2024-08-17T17:32:17.988809+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3793,"log_dups_size":0,"ondisk_log_size":3793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:30.864373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248578048,"num_objects":3793,"num_object_clones":0,"num_object_copies":11379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3793,"num_write_kb":242752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":403,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376103+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:32:16.376103+0000","last_peered":"2024-08-17T17:32:16.376103+0000","last_clean":"2024-08-17T17:32:16.376103+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:32:16.376103+0000","last_undegraded":"2024-08-17T17:32:16.376103+0000","last_fullsized":"2024-08-17T17:32:16.376103+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":169,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.379730+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:32:16.379730+0000","last_peered":"2024-08-17T17:32:16.379730+0000","last_clean":"2024-08-17T17:32:16.379730+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:32:16.379730+0000","last_undegraded":"2024-08-17T17:32:16.379730+0000","last_fullsized":"2024-08-17T17:32:16.379730+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"326'3619","reported_seq":3634,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:18.115506+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:32:18.115506+0000","last_peered":"2024-08-17T17:32:18.115506+0000","last_clean":"2024-08-17T17:32:18.115506+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:32:18.115506+0000","last_undegraded":"2024-08-17T17:32:18.115506+0000","last_fullsized":"2024-08-17T17:32:18.115506+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3619,"log_dups_size":0,"ondisk_log_size":3619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237174784,"num_objects":3619,"num_object_clones":0,"num_object_copies":10857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3619,"num_write_kb":231616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":492,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.379461+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:32:16.379461+0000","last_peered":"2024-08-17T17:32:16.379461+0000","last_clean":"2024-08-17T17:32:16.379461+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:32:16.379461+0000","last_undegraded":"2024-08-17T17:32:16.379461+0000","last_fullsized":"2024-08-17T17:32:16.379461+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":280,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321479+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:32:05.321479+0000","last_peered":"2024-08-17T17:32:05.321479+0000","last_clean":"2024-08-17T17:32:05.321479+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:32:05.321479+0000","last_undegraded":"2024-08-17T17:32:05.321479+0000","last_fullsized":"2024-08-17T17:32:05.321479+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"326'3694","reported_seq":3709,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:18.104073+0000","last_change":"2024-08-17T17:31:36.271872+0000","last_active":"2024-08-17T17:32:18.104073+0000","last_peered":"2024-08-17T17:32:18.104073+0000","last_clean":"2024-08-17T17:32:18.104073+0000","last_became_active":"2024-08-17T17:31:36.271398+0000","last_became_peered":"2024-08-17T17:31:36.271398+0000","last_unstale":"2024-08-17T17:32:18.104073+0000","last_undegraded":"2024-08-17T17:32:18.104073+0000","last_fullsized":"2024-08-17T17:32:18.104073+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3694,"log_dups_size":0,"ondisk_log_size":3694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:14:11.815394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242089984,"num_objects":3694,"num_object_clones":0,"num_object_copies":11082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3694,"num_write_kb":236416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":281,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.318710+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:32:05.318710+0000","last_peered":"2024-08-17T17:32:05.318710+0000","last_clean":"2024-08-17T17:32:05.318710+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:32:05.318710+0000","last_undegraded":"2024-08-17T17:32:05.318710+0000","last_fullsized":"2024-08-17T17:32:05.318710+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321963+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:32:05.321963+0000","last_peered":"2024-08-17T17:32:05.321963+0000","last_clean":"2024-08-17T17:32:05.321963+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:32:05.321963+0000","last_undegraded":"2024-08-17T17:32:05.321963+0000","last_fullsized":"2024-08-17T17:32:05.321963+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"326'3631","reported_seq":3656,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.484983+0000","last_change":"2024-08-17T17:31:36.272676+0000","last_active":"2024-08-17T17:32:16.484983+0000","last_peered":"2024-08-17T17:32:16.484983+0000","last_clean":"2024-08-17T17:32:16.484983+0000","last_became_active":"2024-08-17T17:31:36.272403+0000","last_became_peered":"2024-08-17T17:31:36.272403+0000","last_unstale":"2024-08-17T17:32:16.484983+0000","last_undegraded":"2024-08-17T17:32:16.484983+0000","last_fullsized":"2024-08-17T17:32:16.484983+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3631,"log_dups_size":0,"ondisk_log_size":3631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:51.087810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237961216,"num_objects":3631,"num_object_clones":0,"num_object_copies":10893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3631,"num_write_kb":232384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":298,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321380+0000","last_change":"2024-08-17T17:26:32.216646+0000","last_active":"2024-08-17T17:32:05.321380+0000","last_peered":"2024-08-17T17:32:05.321380+0000","last_clean":"2024-08-17T17:32:05.321380+0000","last_became_active":"2024-08-17T17:26:32.216155+0000","last_became_peered":"2024-08-17T17:26:32.216155+0000","last_unstale":"2024-08-17T17:32:05.321380+0000","last_undegraded":"2024-08-17T17:32:05.321380+0000","last_fullsized":"2024-08-17T17:32:05.321380+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:55.449331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":280,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321653+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:32:05.321653+0000","last_peered":"2024-08-17T17:32:05.321653+0000","last_clean":"2024-08-17T17:32:05.321653+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:32:05.321653+0000","last_undegraded":"2024-08-17T17:32:05.321653+0000","last_fullsized":"2024-08-17T17:32:05.321653+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"325'3559","reported_seq":3573,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:15.397503+0000","last_change":"2024-08-17T17:31:36.272047+0000","last_active":"2024-08-17T17:32:15.397503+0000","last_peered":"2024-08-17T17:32:15.397503+0000","last_clean":"2024-08-17T17:32:15.397503+0000","last_became_active":"2024-08-17T17:31:36.269393+0000","last_became_peered":"2024-08-17T17:31:36.269393+0000","last_unstale":"2024-08-17T17:32:15.397503+0000","last_undegraded":"2024-08-17T17:32:15.397503+0000","last_fullsized":"2024-08-17T17:32:15.397503+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3559,"log_dups_size":0,"ondisk_log_size":3559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:56.899364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233242624,"num_objects":3559,"num_object_clones":0,"num_object_copies":10677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3559,"num_write_kb":227776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":455,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376626+0000","last_change":"2024-08-17T17:26:32.212135+0000","last_active":"2024-08-17T17:32:16.376626+0000","last_peered":"2024-08-17T17:32:16.376626+0000","last_clean":"2024-08-17T17:32:16.376626+0000","last_became_active":"2024-08-17T17:26:32.211604+0000","last_became_peered":"2024-08-17T17:26:32.211604+0000","last_unstale":"2024-08-17T17:32:16.376626+0000","last_undegraded":"2024-08-17T17:32:16.376626+0000","last_fullsized":"2024-08-17T17:32:16.376626+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:49.886809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":399,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376263+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:32:16.376263+0000","last_peered":"2024-08-17T17:32:16.376263+0000","last_clean":"2024-08-17T17:32:16.376263+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:32:16.376263+0000","last_undegraded":"2024-08-17T17:32:16.376263+0000","last_fullsized":"2024-08-17T17:32:16.376263+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"325'3598","reported_seq":3612,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:15.372286+0000","last_change":"2024-08-17T17:31:36.270955+0000","last_active":"2024-08-17T17:32:15.372286+0000","last_peered":"2024-08-17T17:32:15.372286+0000","last_clean":"2024-08-17T17:32:15.372286+0000","last_became_active":"2024-08-17T17:31:36.270618+0000","last_became_peered":"2024-08-17T17:31:36.270618+0000","last_unstale":"2024-08-17T17:32:15.372286+0000","last_undegraded":"2024-08-17T17:32:15.372286+0000","last_fullsized":"2024-08-17T17:32:15.372286+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3598,"log_dups_size":0,"ondisk_log_size":3598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:46.731269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235798528,"num_objects":3598,"num_object_clones":0,"num_object_copies":10794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3598,"num_write_kb":230272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":647,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256119+0000","last_change":"2024-08-17T17:26:33.204740+0000","last_active":"2024-08-17T17:31:49.256119+0000","last_peered":"2024-08-17T17:31:49.256119+0000","last_clean":"2024-08-17T17:31:49.256119+0000","last_became_active":"2024-08-17T17:26:33.204340+0000","last_became_peered":"2024-08-17T17:26:33.204340+0000","last_unstale":"2024-08-17T17:31:49.256119+0000","last_undegraded":"2024-08-17T17:31:49.256119+0000","last_fullsized":"2024-08-17T17:31:49.256119+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:43.802253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":233,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329920+0000","last_change":"2024-08-17T17:26:32.219571+0000","last_active":"2024-08-17T17:32:05.329920+0000","last_peered":"2024-08-17T17:32:05.329920+0000","last_clean":"2024-08-17T17:32:05.329920+0000","last_became_active":"2024-08-17T17:26:32.219094+0000","last_became_peered":"2024-08-17T17:26:32.219094+0000","last_unstale":"2024-08-17T17:32:05.329920+0000","last_undegraded":"2024-08-17T17:32:05.329920+0000","last_fullsized":"2024-08-17T17:32:05.329920+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:09.450772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":441,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.377735+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:32:16.377735+0000","last_peered":"2024-08-17T17:32:16.377735+0000","last_clean":"2024-08-17T17:32:16.377735+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:32:16.377735+0000","last_undegraded":"2024-08-17T17:32:16.377735+0000","last_fullsized":"2024-08-17T17:32:16.377735+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":280,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.318936+0000","last_change":"2024-08-17T17:26:32.215221+0000","last_active":"2024-08-17T17:32:05.318936+0000","last_peered":"2024-08-17T17:32:05.318936+0000","last_clean":"2024-08-17T17:32:05.318936+0000","last_became_active":"2024-08-17T17:26:32.214756+0000","last_became_peered":"2024-08-17T17:26:32.214756+0000","last_unstale":"2024-08-17T17:32:05.318936+0000","last_undegraded":"2024-08-17T17:32:05.318936+0000","last_fullsized":"2024-08-17T17:32:05.318936+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.545194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"326'3667","reported_seq":3692,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:17.979628+0000","last_change":"2024-08-17T17:31:36.270236+0000","last_active":"2024-08-17T17:32:17.979628+0000","last_peered":"2024-08-17T17:32:17.979628+0000","last_clean":"2024-08-17T17:32:17.979628+0000","last_became_active":"2024-08-17T17:31:36.269950+0000","last_became_peered":"2024-08-17T17:31:36.269950+0000","last_unstale":"2024-08-17T17:32:17.979628+0000","last_undegraded":"2024-08-17T17:32:17.979628+0000","last_fullsized":"2024-08-17T17:32:17.979628+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3667,"log_dups_size":0,"ondisk_log_size":3667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:30.882354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320512,"num_objects":3667,"num_object_clones":0,"num_object_copies":11001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3667,"num_write_kb":234688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":321,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256214+0000","last_change":"2024-08-17T17:26:32.212907+0000","last_active":"2024-08-17T17:31:49.256214+0000","last_peered":"2024-08-17T17:31:49.256214+0000","last_clean":"2024-08-17T17:31:49.256214+0000","last_became_active":"2024-08-17T17:26:32.212549+0000","last_became_peered":"2024-08-17T17:26:32.212549+0000","last_unstale":"2024-08-17T17:31:49.256214+0000","last_undegraded":"2024-08-17T17:31:49.256214+0000","last_fullsized":"2024-08-17T17:31:49.256214+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:35.071333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321143+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:32:05.321143+0000","last_peered":"2024-08-17T17:32:05.321143+0000","last_clean":"2024-08-17T17:32:05.321143+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:32:05.321143+0000","last_undegraded":"2024-08-17T17:32:05.321143+0000","last_fullsized":"2024-08-17T17:32:05.321143+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":169,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.380077+0000","last_change":"2024-08-17T17:24:37.073148+0000","last_active":"2024-08-17T17:32:16.380077+0000","last_peered":"2024-08-17T17:32:16.380077+0000","last_clean":"2024-08-17T17:32:16.380077+0000","last_became_active":"2024-08-17T17:24:37.072372+0000","last_became_peered":"2024-08-17T17:24:37.072372+0000","last_unstale":"2024-08-17T17:32:16.380077+0000","last_undegraded":"2024-08-17T17:32:16.380077+0000","last_fullsized":"2024-08-17T17:32:16.380077+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.659683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"326'3715","reported_seq":3730,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:18.099289+0000","last_change":"2024-08-17T17:31:36.269392+0000","last_active":"2024-08-17T17:32:18.099289+0000","last_peered":"2024-08-17T17:32:18.099289+0000","last_clean":"2024-08-17T17:32:18.099289+0000","last_became_active":"2024-08-17T17:31:36.269111+0000","last_became_peered":"2024-08-17T17:31:36.269111+0000","last_unstale":"2024-08-17T17:32:18.099289+0000","last_undegraded":"2024-08-17T17:32:18.099289+0000","last_fullsized":"2024-08-17T17:32:18.099289+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:37.539703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3715,"num_write_kb":237760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":517,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.388628+0000","last_change":"2024-08-17T17:24:36.969933+0000","last_active":"2024-08-17T17:32:16.388628+0000","last_peered":"2024-08-17T17:32:16.388628+0000","last_clean":"2024-08-17T17:32:16.388628+0000","last_became_active":"2024-08-17T17:24:36.969657+0000","last_became_peered":"2024-08-17T17:24:36.969657+0000","last_unstale":"2024-08-17T17:32:16.388628+0000","last_undegraded":"2024-08-17T17:32:16.388628+0000","last_fullsized":"2024-08-17T17:32:16.388628+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:10.367450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.389839+0000","last_change":"2024-08-17T17:26:50.368470+0000","last_active":"2024-08-17T17:32:16.389839+0000","last_peered":"2024-08-17T17:32:16.389839+0000","last_clean":"2024-08-17T17:32:16.389839+0000","last_became_active":"2024-08-17T17:26:50.367844+0000","last_became_peered":"2024-08-17T17:26:50.367844+0000","last_unstale":"2024-08-17T17:32:16.389839+0000","last_undegraded":"2024-08-17T17:32:16.389839+0000","last_fullsized":"2024-08-17T17:32:16.389839+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:02.244611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":534,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.374690+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:32:16.374690+0000","last_peered":"2024-08-17T17:32:16.374690+0000","last_clean":"2024-08-17T17:32:16.374690+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:32:16.374690+0000","last_undegraded":"2024-08-17T17:32:16.374690+0000","last_fullsized":"2024-08-17T17:32:16.374690+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"326'3676","reported_seq":3701,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:17.978348+0000","last_change":"2024-08-17T17:31:36.275334+0000","last_active":"2024-08-17T17:32:17.978348+0000","last_peered":"2024-08-17T17:32:17.978348+0000","last_clean":"2024-08-17T17:32:17.978348+0000","last_became_active":"2024-08-17T17:31:36.274901+0000","last_became_peered":"2024-08-17T17:31:36.274901+0000","last_unstale":"2024-08-17T17:32:17.978348+0000","last_undegraded":"2024-08-17T17:32:17.978348+0000","last_fullsized":"2024-08-17T17:32:17.978348+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3676,"log_dups_size":0,"ondisk_log_size":3676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:04.919838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240910336,"num_objects":3676,"num_object_clones":0,"num_object_copies":11028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3676,"num_write_kb":235264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":325,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.373728+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:32:16.373728+0000","last_peered":"2024-08-17T17:32:16.373728+0000","last_clean":"2024-08-17T17:32:16.373728+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:32:16.373728+0000","last_undegraded":"2024-08-17T17:32:16.373728+0000","last_fullsized":"2024-08-17T17:32:16.373728+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":301,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.322411+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:32:05.322411+0000","last_peered":"2024-08-17T17:32:05.322411+0000","last_clean":"2024-08-17T17:32:05.322411+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:32:05.322411+0000","last_undegraded":"2024-08-17T17:32:05.322411+0000","last_fullsized":"2024-08-17T17:32:05.322411+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":233,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.390057+0000","last_change":"2024-08-17T17:26:50.310453+0000","last_active":"2024-08-17T17:32:16.390057+0000","last_peered":"2024-08-17T17:32:16.390057+0000","last_clean":"2024-08-17T17:32:16.390057+0000","last_became_active":"2024-08-17T17:26:50.310180+0000","last_became_peered":"2024-08-17T17:26:50.310180+0000","last_unstale":"2024-08-17T17:32:16.390057+0000","last_undegraded":"2024-08-17T17:32:16.390057+0000","last_fullsized":"2024-08-17T17:32:16.390057+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:38.334936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"326'3887","reported_seq":3912,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:19.260111+0000","last_change":"2024-08-17T17:31:36.270808+0000","last_active":"2024-08-17T17:32:19.260111+0000","last_peered":"2024-08-17T17:32:19.260111+0000","last_clean":"2024-08-17T17:32:19.260111+0000","last_became_active":"2024-08-17T17:31:36.270536+0000","last_became_peered":"2024-08-17T17:31:36.270536+0000","last_unstale":"2024-08-17T17:32:19.260111+0000","last_undegraded":"2024-08-17T17:32:19.260111+0000","last_fullsized":"2024-08-17T17:32:19.260111+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3887,"log_dups_size":0,"ondisk_log_size":3887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:31.074486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254738432,"num_objects":3887,"num_object_clones":0,"num_object_copies":11661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3887,"num_write_kb":248768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329952+0000","last_change":"2024-08-17T17:25:46.633494+0000","last_active":"2024-08-17T17:32:05.329952+0000","last_peered":"2024-08-17T17:32:05.329952+0000","last_clean":"2024-08-17T17:32:05.329952+0000","last_became_active":"2024-08-17T17:25:46.633216+0000","last_became_peered":"2024-08-17T17:25:46.633216+0000","last_unstale":"2024-08-17T17:32:05.329952+0000","last_undegraded":"2024-08-17T17:32:05.329952+0000","last_fullsized":"2024-08-17T17:32:05.329952+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:14.957437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":486,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256159+0000","last_change":"2024-08-17T17:26:50.338185+0000","last_active":"2024-08-17T17:31:49.256159+0000","last_peered":"2024-08-17T17:31:49.256159+0000","last_clean":"2024-08-17T17:31:49.256159+0000","last_became_active":"2024-08-17T17:26:50.337992+0000","last_became_peered":"2024-08-17T17:26:50.337992+0000","last_unstale":"2024-08-17T17:31:49.256159+0000","last_undegraded":"2024-08-17T17:31:49.256159+0000","last_fullsized":"2024-08-17T17:31:49.256159+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:26.720733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":484,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.374003+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:32:16.374003+0000","last_peered":"2024-08-17T17:32:16.374003+0000","last_clean":"2024-08-17T17:32:16.374003+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:32:16.374003+0000","last_undegraded":"2024-08-17T17:32:16.374003+0000","last_fullsized":"2024-08-17T17:32:16.374003+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"326'3832","reported_seq":3856,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:20.121852+0000","last_change":"2024-08-17T17:31:36.272820+0000","last_active":"2024-08-17T17:32:20.121852+0000","last_peered":"2024-08-17T17:32:20.121852+0000","last_clean":"2024-08-17T17:32:20.121852+0000","last_became_active":"2024-08-17T17:31:36.272457+0000","last_became_peered":"2024-08-17T17:31:36.272457+0000","last_unstale":"2024-08-17T17:32:20.121852+0000","last_undegraded":"2024-08-17T17:32:20.121852+0000","last_fullsized":"2024-08-17T17:32:20.121852+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3832,"log_dups_size":0,"ondisk_log_size":3832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:39.167644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251133952,"num_objects":3832,"num_object_clones":0,"num_object_copies":11496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3832,"num_write_kb":245248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":342,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.387488+0000","last_change":"2024-08-17T17:26:50.315231+0000","last_active":"2024-08-17T17:32:16.387488+0000","last_peered":"2024-08-17T17:32:16.387488+0000","last_clean":"2024-08-17T17:32:16.387488+0000","last_became_active":"2024-08-17T17:26:50.314945+0000","last_became_peered":"2024-08-17T17:26:50.314945+0000","last_unstale":"2024-08-17T17:32:16.387488+0000","last_undegraded":"2024-08-17T17:32:16.387488+0000","last_fullsized":"2024-08-17T17:32:16.387488+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:13.740267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":304,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256627+0000","last_change":"2024-08-17T17:26:32.215417+0000","last_active":"2024-08-17T17:31:49.256627+0000","last_peered":"2024-08-17T17:31:49.256627+0000","last_clean":"2024-08-17T17:31:49.256627+0000","last_became_active":"2024-08-17T17:26:32.215160+0000","last_became_peered":"2024-08-17T17:26:32.215160+0000","last_unstale":"2024-08-17T17:31:49.256627+0000","last_undegraded":"2024-08-17T17:31:49.256627+0000","last_fullsized":"2024-08-17T17:31:49.256627+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:17.370439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":368,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329832+0000","last_change":"2024-08-17T17:26:32.221201+0000","last_active":"2024-08-17T17:32:05.329832+0000","last_peered":"2024-08-17T17:32:05.329832+0000","last_clean":"2024-08-17T17:32:05.329832+0000","last_became_active":"2024-08-17T17:26:32.220913+0000","last_became_peered":"2024-08-17T17:26:32.220913+0000","last_unstale":"2024-08-17T17:32:05.329832+0000","last_undegraded":"2024-08-17T17:32:05.329832+0000","last_fullsized":"2024-08-17T17:32:05.329832+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:20.201903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"326'3872","reported_seq":3897,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:20.103092+0000","last_change":"2024-08-17T17:31:36.271799+0000","last_active":"2024-08-17T17:32:20.103092+0000","last_peered":"2024-08-17T17:32:20.103092+0000","last_clean":"2024-08-17T17:32:20.103092+0000","last_became_active":"2024-08-17T17:31:36.271569+0000","last_became_peered":"2024-08-17T17:31:36.271569+0000","last_unstale":"2024-08-17T17:32:20.103092+0000","last_undegraded":"2024-08-17T17:32:20.103092+0000","last_fullsized":"2024-08-17T17:32:20.103092+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3872,"log_dups_size":0,"ondisk_log_size":3872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:28.450375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253755392,"num_objects":3872,"num_object_clones":0,"num_object_copies":11616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3872,"num_write_kb":247808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":504,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376339+0000","last_change":"2024-08-17T17:26:32.209014+0000","last_active":"2024-08-17T17:32:16.376339+0000","last_peered":"2024-08-17T17:32:16.376339+0000","last_clean":"2024-08-17T17:32:16.376339+0000","last_became_active":"2024-08-17T17:26:32.208539+0000","last_became_peered":"2024-08-17T17:26:32.208539+0000","last_unstale":"2024-08-17T17:32:16.376339+0000","last_undegraded":"2024-08-17T17:32:16.376339+0000","last_fullsized":"2024-08-17T17:32:16.376339+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:40:15.666378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":483,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375340+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:32:16.375340+0000","last_peered":"2024-08-17T17:32:16.375340+0000","last_clean":"2024-08-17T17:32:16.375340+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:32:16.375340+0000","last_undegraded":"2024-08-17T17:32:16.375340+0000","last_fullsized":"2024-08-17T17:32:16.375340+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":493,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376465+0000","last_change":"2024-08-17T17:26:32.211876+0000","last_active":"2024-08-17T17:32:16.376465+0000","last_peered":"2024-08-17T17:32:16.376465+0000","last_clean":"2024-08-17T17:32:16.376465+0000","last_became_active":"2024-08-17T17:26:32.211212+0000","last_became_peered":"2024-08-17T17:26:32.211212+0000","last_unstale":"2024-08-17T17:32:16.376465+0000","last_undegraded":"2024-08-17T17:32:16.376465+0000","last_fullsized":"2024-08-17T17:32:16.376465+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:50.032892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"326'3767","reported_seq":3792,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:19.256708+0000","last_change":"2024-08-17T17:31:36.269501+0000","last_active":"2024-08-17T17:32:19.256708+0000","last_peered":"2024-08-17T17:32:19.256708+0000","last_clean":"2024-08-17T17:32:19.256708+0000","last_became_active":"2024-08-17T17:31:36.269223+0000","last_became_peered":"2024-08-17T17:31:36.269223+0000","last_unstale":"2024-08-17T17:32:19.256708+0000","last_undegraded":"2024-08-17T17:32:19.256708+0000","last_fullsized":"2024-08-17T17:32:19.256708+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":3767,"log_dups_size":0,"ondisk_log_size":3767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:00.587765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246874112,"num_objects":3767,"num_object_clones":0,"num_object_copies":11301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3767,"num_write_kb":241088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":173,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.379886+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:32:16.379886+0000","last_peered":"2024-08-17T17:32:16.379886+0000","last_clean":"2024-08-17T17:32:16.379886+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:32:16.379886+0000","last_undegraded":"2024-08-17T17:32:16.379886+0000","last_fullsized":"2024-08-17T17:32:16.379886+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":310,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.318848+0000","last_change":"2024-08-17T17:26:32.218111+0000","last_active":"2024-08-17T17:32:05.318848+0000","last_peered":"2024-08-17T17:32:05.318848+0000","last_clean":"2024-08-17T17:32:05.318848+0000","last_became_active":"2024-08-17T17:26:32.217705+0000","last_became_peered":"2024-08-17T17:26:32.217705+0000","last_unstale":"2024-08-17T17:32:05.318848+0000","last_undegraded":"2024-08-17T17:32:05.318848+0000","last_fullsized":"2024-08-17T17:32:05.318848+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:50.751085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":280,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321356+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:32:05.321356+0000","last_peered":"2024-08-17T17:32:05.321356+0000","last_clean":"2024-08-17T17:32:05.321356+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:32:05.321356+0000","last_undegraded":"2024-08-17T17:32:05.321356+0000","last_fullsized":"2024-08-17T17:32:05.321356+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":280,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.322377+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:32:05.322377+0000","last_peered":"2024-08-17T17:32:05.322377+0000","last_clean":"2024-08-17T17:32:05.322377+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:32:05.322377+0000","last_undegraded":"2024-08-17T17:32:05.322377+0000","last_fullsized":"2024-08-17T17:32:05.322377+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":303,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321653+0000","last_change":"2024-08-17T17:26:32.215288+0000","last_active":"2024-08-17T17:32:05.321653+0000","last_peered":"2024-08-17T17:32:05.321653+0000","last_clean":"2024-08-17T17:32:05.321653+0000","last_became_active":"2024-08-17T17:26:32.214754+0000","last_became_peered":"2024-08-17T17:26:32.214754+0000","last_unstale":"2024-08-17T17:32:05.321653+0000","last_undegraded":"2024-08-17T17:32:05.321653+0000","last_fullsized":"2024-08-17T17:32:05.321653+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.684058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":483,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.373817+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:32:16.373817+0000","last_peered":"2024-08-17T17:32:16.373817+0000","last_clean":"2024-08-17T17:32:16.373817+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:32:16.373817+0000","last_undegraded":"2024-08-17T17:32:16.373817+0000","last_fullsized":"2024-08-17T17:32:16.373817+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":256,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256561+0000","last_change":"2024-08-17T17:26:32.207981+0000","last_active":"2024-08-17T17:31:49.256561+0000","last_peered":"2024-08-17T17:31:49.256561+0000","last_clean":"2024-08-17T17:31:49.256561+0000","last_became_active":"2024-08-17T17:26:32.207636+0000","last_became_peered":"2024-08-17T17:26:32.207636+0000","last_unstale":"2024-08-17T17:31:49.256561+0000","last_undegraded":"2024-08-17T17:31:49.256561+0000","last_fullsized":"2024-08-17T17:31:49.256561+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:32.412636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.330143+0000","last_change":"2024-08-17T17:26:32.210741+0000","last_active":"2024-08-17T17:32:05.330143+0000","last_peered":"2024-08-17T17:32:05.330143+0000","last_clean":"2024-08-17T17:32:05.330143+0000","last_became_active":"2024-08-17T17:26:32.210417+0000","last_became_peered":"2024-08-17T17:26:32.210417+0000","last_unstale":"2024-08-17T17:32:05.330143+0000","last_undegraded":"2024-08-17T17:32:05.330143+0000","last_fullsized":"2024-08-17T17:32:05.330143+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:41.290842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":441,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.378414+0000","last_change":"2024-08-17T17:26:32.211602+0000","last_active":"2024-08-17T17:32:16.378414+0000","last_peered":"2024-08-17T17:32:16.378414+0000","last_clean":"2024-08-17T17:32:16.378414+0000","last_became_active":"2024-08-17T17:26:32.211288+0000","last_became_peered":"2024-08-17T17:26:32.211288+0000","last_unstale":"2024-08-17T17:32:16.378414+0000","last_undegraded":"2024-08-17T17:32:16.378414+0000","last_fullsized":"2024-08-17T17:32:16.378414+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:49.979367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":374,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.330226+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:32:05.330226+0000","last_peered":"2024-08-17T17:32:05.330226+0000","last_clean":"2024-08-17T17:32:05.330226+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:32:05.330226+0000","last_undegraded":"2024-08-17T17:32:05.330226+0000","last_fullsized":"2024-08-17T17:32:05.330226+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":442,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376253+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:32:16.376253+0000","last_peered":"2024-08-17T17:32:16.376253+0000","last_clean":"2024-08-17T17:32:16.376253+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:32:16.376253+0000","last_undegraded":"2024-08-17T17:32:16.376253+0000","last_fullsized":"2024-08-17T17:32:16.376253+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.318833+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:32:05.318833+0000","last_peered":"2024-08-17T17:32:05.318833+0000","last_clean":"2024-08-17T17:32:05.318833+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:32:05.318833+0000","last_undegraded":"2024-08-17T17:32:05.318833+0000","last_fullsized":"2024-08-17T17:32:05.318833+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":474,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376544+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:32:16.376544+0000","last_peered":"2024-08-17T17:32:16.376544+0000","last_clean":"2024-08-17T17:32:16.376544+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:32:16.376544+0000","last_undegraded":"2024-08-17T17:32:16.376544+0000","last_fullsized":"2024-08-17T17:32:16.376544+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":324,"state":"active+clean","last_fresh":"2024-08-17T17:31:49.256475+0000","last_change":"2024-08-17T17:26:32.215823+0000","last_active":"2024-08-17T17:31:49.256475+0000","last_peered":"2024-08-17T17:31:49.256475+0000","last_clean":"2024-08-17T17:31:49.256475+0000","last_became_active":"2024-08-17T17:26:32.214110+0000","last_became_peered":"2024-08-17T17:26:32.214110+0000","last_unstale":"2024-08-17T17:31:49.256475+0000","last_undegraded":"2024-08-17T17:31:49.256475+0000","last_fullsized":"2024-08-17T17:31:49.256475+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.088172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":482,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.373550+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:32:16.373550+0000","last_peered":"2024-08-17T17:32:16.373550+0000","last_clean":"2024-08-17T17:32:16.373550+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:32:16.373550+0000","last_undegraded":"2024-08-17T17:32:16.373550+0000","last_fullsized":"2024-08-17T17:32:16.373550+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":441,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375845+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:32:16.375845+0000","last_peered":"2024-08-17T17:32:16.375845+0000","last_clean":"2024-08-17T17:32:16.375845+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:32:16.375845+0000","last_undegraded":"2024-08-17T17:32:16.375845+0000","last_fullsized":"2024-08-17T17:32:16.375845+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":299,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321635+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:32:05.321635+0000","last_peered":"2024-08-17T17:32:05.321635+0000","last_clean":"2024-08-17T17:32:05.321635+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:32:05.321635+0000","last_undegraded":"2024-08-17T17:32:05.321635+0000","last_fullsized":"2024-08-17T17:32:05.321635+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":505,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.387582+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:32:16.387582+0000","last_peered":"2024-08-17T17:32:16.387582+0000","last_clean":"2024-08-17T17:32:16.387582+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:32:16.387582+0000","last_undegraded":"2024-08-17T17:32:16.387582+0000","last_fullsized":"2024-08-17T17:32:16.387582+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":180,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.379482+0000","last_change":"2024-08-17T17:26:32.214552+0000","last_active":"2024-08-17T17:32:16.379482+0000","last_peered":"2024-08-17T17:32:16.379482+0000","last_clean":"2024-08-17T17:32:16.379482+0000","last_became_active":"2024-08-17T17:26:32.214338+0000","last_became_peered":"2024-08-17T17:26:32.214338+0000","last_unstale":"2024-08-17T17:32:16.379482+0000","last_undegraded":"2024-08-17T17:32:16.379482+0000","last_fullsized":"2024-08-17T17:32:16.379482+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:21.212607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":307,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.322263+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:32:05.322263+0000","last_peered":"2024-08-17T17:32:05.322263+0000","last_clean":"2024-08-17T17:32:05.322263+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:32:05.322263+0000","last_undegraded":"2024-08-17T17:32:05.322263+0000","last_fullsized":"2024-08-17T17:32:05.322263+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":180,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321571+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:32:05.321571+0000","last_peered":"2024-08-17T17:32:05.321571+0000","last_clean":"2024-08-17T17:32:05.321571+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:32:05.321571+0000","last_undegraded":"2024-08-17T17:32:05.321571+0000","last_fullsized":"2024-08-17T17:32:05.321571+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":317,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321510+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:32:05.321510+0000","last_peered":"2024-08-17T17:32:05.321510+0000","last_clean":"2024-08-17T17:32:05.321510+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:32:05.321510+0000","last_undegraded":"2024-08-17T17:32:05.321510+0000","last_fullsized":"2024-08-17T17:32:05.321510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":180,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321832+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:32:05.321832+0000","last_peered":"2024-08-17T17:32:05.321832+0000","last_clean":"2024-08-17T17:32:05.321832+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:32:05.321832+0000","last_undegraded":"2024-08-17T17:32:05.321832+0000","last_fullsized":"2024-08-17T17:32:05.321832+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":487,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.380294+0000","last_change":"2024-08-17T17:24:36.972596+0000","last_active":"2024-08-17T17:32:16.380294+0000","last_peered":"2024-08-17T17:32:16.380294+0000","last_clean":"2024-08-17T17:32:16.380294+0000","last_became_active":"2024-08-17T17:24:36.971541+0000","last_became_peered":"2024-08-17T17:24:36.971541+0000","last_unstale":"2024-08-17T17:32:16.380294+0000","last_undegraded":"2024-08-17T17:32:16.380294+0000","last_fullsized":"2024-08-17T17:32:16.380294+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:38.457003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":459,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376526+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:32:16.376526+0000","last_peered":"2024-08-17T17:32:16.376526+0000","last_clean":"2024-08-17T17:32:16.376526+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:32:16.376526+0000","last_undegraded":"2024-08-17T17:32:16.376526+0000","last_fullsized":"2024-08-17T17:32:16.376526+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":280,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.321136+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:32:05.321136+0000","last_peered":"2024-08-17T17:32:05.321136+0000","last_clean":"2024-08-17T17:32:05.321136+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:32:05.321136+0000","last_undegraded":"2024-08-17T17:32:05.321136+0000","last_fullsized":"2024-08-17T17:32:05.321136+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":175,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.380141+0000","last_change":"2024-08-17T17:24:37.032048+0000","last_active":"2024-08-17T17:32:16.380141+0000","last_peered":"2024-08-17T17:32:16.380141+0000","last_clean":"2024-08-17T17:32:16.380141+0000","last_became_active":"2024-08-17T17:24:36.972161+0000","last_became_peered":"2024-08-17T17:24:36.972161+0000","last_unstale":"2024-08-17T17:32:16.380141+0000","last_undegraded":"2024-08-17T17:32:16.380141+0000","last_fullsized":"2024-08-17T17:32:16.380141+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:55.363820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3889889280,"num_objects":59355,"num_object_clones":0,"num_object_copies":178065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":59355,"num_write_kb":3798720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11726094336,"data_stored":11726094336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59355,"ondisk_log_size":59355,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1162,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":270,"seq":1159641169990,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1949440,"kb_used_data":1574968,"kb_used_omap":12,"kb_used_meta":374387,"kb_avail":91795712,"statfs":{"total":95995035648,"available":93998809088,"internally_reserved":0,"allocated":1612767232,"data_stored":1611108425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12410,"internal_metadata":383373190},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,4,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463360,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2531460,"kb_used_data":1641916,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":91213692,"statfs":{"total":95995035648,"available":93402820608,"internally_reserved":0,"allocated":1681321984,"data_stored":1679667186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492035,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893808,"kb_used_data":498472,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92851344,"statfs":{"total":95995035648,"available":95079776256,"internally_reserved":0,"allocated":510435328,"data_stored":508795242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986634,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1604992,"kb_used_data":704056,"kb_used_omap":1,"kb_used_meta":900926,"kb_avail":92140160,"statfs":{"total":95995035648,"available":94351523840,"internally_reserved":0,"allocated":720953344,"data_stored":719301898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":922548568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225454,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2771948,"kb_used_data":1980132,"kb_used_omap":4,"kb_used_meta":791803,"kb_avail":90973204,"statfs":{"total":95995035648,"available":93156560896,"internally_reserved":0,"allocated":2027655168,"data_stored":2026019406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":810806720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075779,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3060920,"kb_used_data":2436716,"kb_used_omap":6,"kb_used_meta":624185,"kb_avail":90684232,"statfs":{"total":95995035648,"available":92860653568,"internally_reserved":0,"allocated":2495197184,"data_stored":2493554033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":639165883},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533671,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2086368,"kb_used_data":1147096,"kb_used_omap":5,"kb_used_meta":939258,"kb_avail":91658784,"statfs":{"total":95995035648,"available":93858594816,"internally_reserved":0,"allocated":1174626304,"data_stored":1172993380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":961800432},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089648,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2367036,"kb_used_data":1403632,"kb_used_omap":5,"kb_used_meta":963386,"kb_avail":91378116,"statfs":{"total":95995035648,"available":93571190784,"internally_reserved":0,"allocated":1437319168,"data_stored":1435672453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":986507323},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1465581568,"data_stored":1465581568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1175781376,"data_stored":1175781376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2504589312,"data_stored":2504589312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2036400128,"data_stored":2036400128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":717946880,"data_stored":717946880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":509542400,"data_stored":509542400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1690370048,"data_stored":1690370048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1625882624,"data_stored":1625882624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:32:24.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:32:24.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:24.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:24.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:24.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:24.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:25.037 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 68361 68345 112.376 77.875 0.0114287 0.00889262 2024-08-17T17:32:25.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:25.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:25.196 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:25.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:25.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:25.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:25.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:25.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:25.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:25.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:25.565 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:25 smithi142 ceph-mon[103614]: pgmap v1121: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:32:25.566 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:25 smithi142 ceph-mon[106729]: pgmap v1121: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:32:25.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:25.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:25.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:25.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:25.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:25 smithi152 ceph-mon[93456]: pgmap v1121: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:32:25.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:25.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:25.976 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 69516 69500 111.344 72.1875 0.0186001 0.00897546 2024-08-17T17:32:26.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.018 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:26.018 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:26.035 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:26.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:26.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:26.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:32:26.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:26.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:26.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:26.583 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:26.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:26.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:26.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:26.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:26.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:26.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:26.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:26.977 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:32:26.976976+0000 min lat: 0.00117681 max lat: 0.338534 avg lat: 0.00905486 2024-08-17T17:32:26.977 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:32:26.977 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 70678 70662 110.376 72.625 0.00986548 0.00905486 2024-08-17T17:32:27.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:27.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.036 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:27.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:27.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:27.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:27.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:27.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:27.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:27.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:27.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:27.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:32:27.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:27.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:27.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:27.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:27.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:27 smithi152 ceph-mon[93456]: pgmap v1122: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-08-17T17:32:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:27.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:27.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:27 smithi142 ceph-mon[106729]: pgmap v1122: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-08-17T17:32:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:27 smithi142 ceph-mon[103614]: pgmap v1122: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-08-17T17:32:27.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:27.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:27.977 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 71959 71943 109.636 80.0625 0.00354165 0.00911424 2024-08-17T17:32:27.980 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:27.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:28.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:28.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:28.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:28.252 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:28.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:28.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:28.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:28.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:28.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:28.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:28.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:28.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:28.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:28.865 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:28.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:28.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:28.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 15 72999 72984 108.57 65.0625 0.00764701 0.00920392 2024-08-17T17:32:28.992 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:29.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:29.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:29.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:29.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:29.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:29.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:29 smithi142 ceph-mon[106729]: pgmap v1123: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:32:29.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:29 smithi142 ceph-mon[103614]: pgmap v1123: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:32:29.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.645 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:29 smithi152 ceph-mon[93456]: pgmap v1123: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:32:29.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:29.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:29.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:29.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:29.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 74024 74008 107.533 64 0.0068481 0.00929376 2024-08-17T17:32:30.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:30.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:30.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.192 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:30.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:30.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:30.430 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:32:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:32:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:30.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:30.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:30.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:30.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:30.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:30.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 75184 75168 106.737 72.5 0.00901658 0.00936366 2024-08-17T17:32:30.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:31.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:31.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:31.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:31.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.314 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:31.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:31.401 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:32:31.415 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:31.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:31.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:31.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:31.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:31.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:31.671 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-mon[106729]: pgmap v1124: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:32:31.671 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/174077332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:31.671 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:31.671 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-mon[103614]: pgmap v1124: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:32:31.671 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/174077332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:31.671 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:31.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:31.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:31 smithi152 ceph-mon[93456]: pgmap v1124: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:32:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/174077332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:31.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:31.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:31.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:31.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 76393 76377 106.044 75.5625 0.00753508 0.00942432 2024-08-17T17:32:31.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:31.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:32.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:32:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:32:31] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:32:32.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:32.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:32.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:32.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:32.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:32.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:32.503 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:32.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[103614]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:32:32.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[103614]: pgmap v1126: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-17T17:32:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[106729]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:32:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:32 smithi142 ceph-mon[106729]: pgmap v1126: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-17T17:32:32.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:32.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:32.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:32.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:32 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:32 smithi152 ceph-mon[93456]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:32:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:32 smithi152 ceph-mon[93456]: pgmap v1126: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-17T17:32:32.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:32.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:32.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:32.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 77588 77572 105.363 74.6875 0.0178966 0.00948516 2024-08-17T17:32:33.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:33.082 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:33.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:33.211 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:33.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:33.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:33.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:33.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:33.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:33.589 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:33.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:33.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:33.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:33.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:33.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:33.817 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:33.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:33.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:33.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 78762 78746 104.682 73.375 0.00502624 0.00954751 2024-08-17T17:32:34.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.058 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:34.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:34.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:34.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:34.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:34.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:34.443 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:34.443 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:34.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:34.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:34.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:34.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:34.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:34.707 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:32:34.709 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:32:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:34.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:34.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:34.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:34.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:34.979 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 79793 79777 103.844 64.4375 0.00689875 0.00962415 2024-08-17T17:32:35.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:35.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:35.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:35.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:35.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:35.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:35.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:35 smithi142 ceph-mon[103614]: pgmap v1127: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:32:35.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:35 smithi142 ceph-mon[106729]: pgmap v1127: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:32:35.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:35.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:35.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:35.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:35 smithi152 ceph-mon[93456]: pgmap v1127: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T17:32:35.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:35.682 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:35.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:35.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:35.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:35.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:32:35.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:35.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:35.980 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 80947 80931 103.195 72.125 0.0343052 0.0096841 2024-08-17T17:32:36.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:36.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:36.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:36.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:36.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:36.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:36.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:36.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:36.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:36.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:36 smithi152 ceph-mon[93456]: from='client.56278 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:32:36.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:36.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:36 smithi142 ceph-mon[103614]: from='client.56278 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:32:36.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:36 smithi142 ceph-mon[106729]: from='client.56278 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:32:36.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:36.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:36.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:36.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:36.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:36.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:36.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:36.981 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 81980 81964 102.422 64.5625 0.00404852 0.00974912 2024-08-17T17:32:37.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:37.027 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:37.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:37.156 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:37.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:37.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:37.306 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:37.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:37.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:37.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:37.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:37 smithi152 ceph-mon[93456]: pgmap v1128: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:32:37.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:37 smithi142 ceph-mon[103614]: pgmap v1128: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:32:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:37 smithi142 ceph-mon[106729]: pgmap v1128: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:32:37.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:37.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:37.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:37.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:37.980 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 82960 82944 101.615 61.25 0.0135609 0.0098349 2024-08-17T17:32:37.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:37.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:38.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:38.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:38.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:38.260 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:38.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:38.362 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:38.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:38.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:38.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:38.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:38.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:38.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:38.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:38.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:38.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:38.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:38.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:38.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:38.981 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 83894 83878 100.783 58.375 0.00309978 0.00991142 2024-08-17T17:32:39.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:39.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:39.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:39.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:39.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:39.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:39.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:39.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:39.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:39 smithi152 ceph-mon[93456]: pgmap v1129: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:32:39.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:39.593 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:39 smithi142 ceph-mon[106729]: pgmap v1129: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:32:39.593 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:39 smithi142 ceph-mon[103614]: pgmap v1129: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:32:39.616 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:39.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:39.649 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:32:39.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:39.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:39.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:39.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:39.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:39.981 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 84802 84786 99.9522 56.75 0.0248013 0.00999918 2024-08-17T17:32:39.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:40.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:40.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:40.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:40.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:40.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:40.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:40.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:40.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:40.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:32:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:32:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:40.575 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1127,"stamp":"2024-08-17T17:32:34.177367+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4992604109,"num_objects":76198,"num_object_clones":0,"num_object_copies":228594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76198,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":76450,"num_write_kb":4877455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2362634,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76379,"ondisk_log_size":76379,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20279472,"kb_used_data":14400552,"kb_used_omap":45,"kb_used_meta":5878802,"kb_avail":729681744,"statfs":{"total":767960285184,"available":747194105856,"internally_reserved":0,"allocated":14746165248,"data_stored":14732981359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":47063,"internal_metadata":6019893289},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":781516800,"num_objects":11925,"num_object_clones":0,"num_object_copies":35775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11925,"num_write_kb":763200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005622"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":477,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375971+0000","last_change":"2024-08-17T17:26:32.212092+0000","last_active":"2024-08-17T17:32:16.375971+0000","last_peered":"2024-08-17T17:32:16.375971+0000","last_clean":"2024-08-17T17:32:16.375971+0000","last_became_active":"2024-08-17T17:26:32.211774+0000","last_became_peered":"2024-08-17T17:26:32.211774+0000","last_unstale":"2024-08-17T17:32:16.375971+0000","last_undegraded":"2024-08-17T17:32:16.375971+0000","last_fullsized":"2024-08-17T17:32:16.375971+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:45.518565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":271,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308821+0000","last_change":"2024-08-17T17:26:32.210903+0000","last_active":"2024-08-17T17:32:05.308821+0000","last_peered":"2024-08-17T17:32:05.308821+0000","last_clean":"2024-08-17T17:32:05.308821+0000","last_became_active":"2024-08-17T17:26:32.209823+0000","last_became_peered":"2024-08-17T17:26:32.209823+0000","last_unstale":"2024-08-17T17:32:05.308821+0000","last_undegraded":"2024-08-17T17:32:05.308821+0000","last_fullsized":"2024-08-17T17:32:05.308821+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:54:17.717352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":219,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.330065+0000","last_change":"2024-08-17T17:26:32.205305+0000","last_active":"2024-08-17T17:32:05.330065+0000","last_peered":"2024-08-17T17:32:05.330065+0000","last_clean":"2024-08-17T17:32:05.330065+0000","last_became_active":"2024-08-17T17:26:32.205000+0000","last_became_peered":"2024-08-17T17:26:32.205000+0000","last_unstale":"2024-08-17T17:32:05.330065+0000","last_undegraded":"2024-08-17T17:32:05.330065+0000","last_fullsized":"2024-08-17T17:32:05.330065+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:31.144881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":254,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308650+0000","last_change":"2024-08-17T17:26:32.213319+0000","last_active":"2024-08-17T17:32:05.308650+0000","last_peered":"2024-08-17T17:32:05.308650+0000","last_clean":"2024-08-17T17:32:05.308650+0000","last_became_active":"2024-08-17T17:26:32.213079+0000","last_became_peered":"2024-08-17T17:26:32.213079+0000","last_unstale":"2024-08-17T17:32:05.308650+0000","last_undegraded":"2024-08-17T17:32:05.308650+0000","last_fullsized":"2024-08-17T17:32:05.308650+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:43.552406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":359,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376036+0000","last_change":"2024-08-17T17:26:32.211539+0000","last_active":"2024-08-17T17:32:16.376036+0000","last_peered":"2024-08-17T17:32:16.376036+0000","last_clean":"2024-08-17T17:32:16.376036+0000","last_became_active":"2024-08-17T17:26:32.211267+0000","last_became_peered":"2024-08-17T17:26:32.211267+0000","last_unstale":"2024-08-17T17:32:16.376036+0000","last_undegraded":"2024-08-17T17:32:16.376036+0000","last_fullsized":"2024-08-17T17:32:16.376036+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:16.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":240,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308675+0000","last_change":"2024-08-17T17:26:32.233887+0000","last_active":"2024-08-17T17:32:05.308675+0000","last_peered":"2024-08-17T17:32:05.308675+0000","last_clean":"2024-08-17T17:32:05.308675+0000","last_became_active":"2024-08-17T17:26:32.230472+0000","last_became_peered":"2024-08-17T17:26:32.230472+0000","last_unstale":"2024-08-17T17:32:05.308675+0000","last_undegraded":"2024-08-17T17:32:05.308675+0000","last_fullsized":"2024-08-17T17:32:05.308675+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:09.045150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":453,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376184+0000","last_change":"2024-08-17T17:26:32.207139+0000","last_active":"2024-08-17T17:32:16.376184+0000","last_peered":"2024-08-17T17:32:16.376184+0000","last_clean":"2024-08-17T17:32:16.376184+0000","last_became_active":"2024-08-17T17:26:32.206763+0000","last_became_peered":"2024-08-17T17:26:32.206763+0000","last_unstale":"2024-08-17T17:32:16.376184+0000","last_undegraded":"2024-08-17T17:32:16.376184+0000","last_fullsized":"2024-08-17T17:32:16.376184+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:51.396647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":347,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375798+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:32:16.375798+0000","last_peered":"2024-08-17T17:32:16.375798+0000","last_clean":"2024-08-17T17:32:16.375798+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:32:16.375798+0000","last_undegraded":"2024-08-17T17:32:16.375798+0000","last_fullsized":"2024-08-17T17:32:16.375798+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"327'4797","reported_seq":4813,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:32.658774+0000","last_change":"2024-08-17T17:31:36.270953+0000","last_active":"2024-08-17T17:32:32.658774+0000","last_peered":"2024-08-17T17:32:32.658774+0000","last_clean":"2024-08-17T17:32:32.658774+0000","last_became_active":"2024-08-17T17:31:36.270636+0000","last_became_peered":"2024-08-17T17:31:36.270636+0000","last_unstale":"2024-08-17T17:32:32.658774+0000","last_undegraded":"2024-08-17T17:32:32.658774+0000","last_fullsized":"2024-08-17T17:32:32.658774+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4797,"log_dups_size":0,"ondisk_log_size":4797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:27.516649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314376192,"num_objects":4797,"num_object_clones":0,"num_object_copies":14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4797,"num_write_kb":307008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":302,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308865+0000","last_change":"2024-08-17T17:26:32.214168+0000","last_active":"2024-08-17T17:32:05.308865+0000","last_peered":"2024-08-17T17:32:05.308865+0000","last_clean":"2024-08-17T17:32:05.308865+0000","last_became_active":"2024-08-17T17:26:32.213844+0000","last_became_peered":"2024-08-17T17:26:32.213844+0000","last_unstale":"2024-08-17T17:32:05.308865+0000","last_undegraded":"2024-08-17T17:32:05.308865+0000","last_fullsized":"2024-08-17T17:32:05.308865+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:52.468566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":202,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329729+0000","last_change":"2024-08-17T17:26:32.218633+0000","last_active":"2024-08-17T17:32:05.329729+0000","last_peered":"2024-08-17T17:32:05.329729+0000","last_clean":"2024-08-17T17:32:05.329729+0000","last_became_active":"2024-08-17T17:26:32.218212+0000","last_became_peered":"2024-08-17T17:26:32.218212+0000","last_unstale":"2024-08-17T17:32:05.329729+0000","last_undegraded":"2024-08-17T17:32:05.329729+0000","last_fullsized":"2024-08-17T17:32:05.329729+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:55.971779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"327'4901","reported_seq":4917,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:33.088447+0000","last_change":"2024-08-17T17:31:36.269016+0000","last_active":"2024-08-17T17:32:33.088447+0000","last_peered":"2024-08-17T17:32:33.088447+0000","last_clean":"2024-08-17T17:32:33.088447+0000","last_became_active":"2024-08-17T17:31:36.268667+0000","last_became_peered":"2024-08-17T17:31:36.268667+0000","last_unstale":"2024-08-17T17:32:33.088447+0000","last_undegraded":"2024-08-17T17:32:33.088447+0000","last_fullsized":"2024-08-17T17:32:33.088447+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4901,"log_dups_size":0,"ondisk_log_size":4901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:01:46.882039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4901,"num_write_kb":313664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":257,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.309053+0000","last_change":"2024-08-17T17:26:32.220489+0000","last_active":"2024-08-17T17:32:05.309053+0000","last_peered":"2024-08-17T17:32:05.309053+0000","last_clean":"2024-08-17T17:32:05.309053+0000","last_became_active":"2024-08-17T17:26:32.220150+0000","last_became_peered":"2024-08-17T17:26:32.220150+0000","last_unstale":"2024-08-17T17:32:05.309053+0000","last_undegraded":"2024-08-17T17:32:05.309053+0000","last_fullsized":"2024-08-17T17:32:05.309053+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:09.954487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":298,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.383760+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:32:16.383760+0000","last_peered":"2024-08-17T17:32:16.383760+0000","last_clean":"2024-08-17T17:32:16.383760+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:32:16.383760+0000","last_undegraded":"2024-08-17T17:32:16.383760+0000","last_fullsized":"2024-08-17T17:32:16.383760+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"327'4826","reported_seq":4842,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:32.652096+0000","last_change":"2024-08-17T17:31:36.268659+0000","last_active":"2024-08-17T17:32:32.652096+0000","last_peered":"2024-08-17T17:32:32.652096+0000","last_clean":"2024-08-17T17:32:32.652096+0000","last_became_active":"2024-08-17T17:31:36.268347+0000","last_became_peered":"2024-08-17T17:31:36.268347+0000","last_unstale":"2024-08-17T17:32:32.652096+0000","last_undegraded":"2024-08-17T17:32:32.652096+0000","last_fullsized":"2024-08-17T17:32:32.652096+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:30.225892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316276736,"num_objects":4826,"num_object_clones":0,"num_object_copies":14478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4826,"num_write_kb":308864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":267,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.309005+0000","last_change":"2024-08-17T17:26:32.216517+0000","last_active":"2024-08-17T17:32:05.309005+0000","last_peered":"2024-08-17T17:32:05.309005+0000","last_clean":"2024-08-17T17:32:05.309005+0000","last_became_active":"2024-08-17T17:26:32.216161+0000","last_became_peered":"2024-08-17T17:26:32.216161+0000","last_unstale":"2024-08-17T17:32:05.309005+0000","last_undegraded":"2024-08-17T17:32:05.309005+0000","last_fullsized":"2024-08-17T17:32:05.309005+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:39.712941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":298,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375781+0000","last_change":"2024-08-17T17:26:32.217706+0000","last_active":"2024-08-17T17:32:16.375781+0000","last_peered":"2024-08-17T17:32:16.375781+0000","last_clean":"2024-08-17T17:32:16.375781+0000","last_became_active":"2024-08-17T17:26:32.217107+0000","last_became_peered":"2024-08-17T17:26:32.217107+0000","last_unstale":"2024-08-17T17:32:16.375781+0000","last_undegraded":"2024-08-17T17:32:16.375781+0000","last_fullsized":"2024-08-17T17:32:16.375781+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:39.827920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"327'4925","reported_seq":4951,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:32.970535+0000","last_change":"2024-08-17T17:31:36.271004+0000","last_active":"2024-08-17T17:32:32.970535+0000","last_peered":"2024-08-17T17:32:32.970535+0000","last_clean":"2024-08-17T17:32:32.970535+0000","last_became_active":"2024-08-17T17:31:36.270454+0000","last_became_peered":"2024-08-17T17:31:36.270454+0000","last_unstale":"2024-08-17T17:32:32.970535+0000","last_undegraded":"2024-08-17T17:32:32.970535+0000","last_fullsized":"2024-08-17T17:32:32.970535+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4925,"log_dups_size":0,"ondisk_log_size":4925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:30.864373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322764800,"num_objects":4925,"num_object_clones":0,"num_object_copies":14775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4925,"num_write_kb":315200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":403,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376103+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:32:16.376103+0000","last_peered":"2024-08-17T17:32:16.376103+0000","last_clean":"2024-08-17T17:32:16.376103+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:32:16.376103+0000","last_undegraded":"2024-08-17T17:32:16.376103+0000","last_fullsized":"2024-08-17T17:32:16.376103+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":171,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.407474+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:32:31.407474+0000","last_peered":"2024-08-17T17:32:31.407474+0000","last_clean":"2024-08-17T17:32:31.407474+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:32:31.407474+0000","last_undegraded":"2024-08-17T17:32:31.407474+0000","last_fullsized":"2024-08-17T17:32:31.407474+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"327'4750","reported_seq":4766,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:33.119404+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:32:33.119404+0000","last_peered":"2024-08-17T17:32:33.119404+0000","last_clean":"2024-08-17T17:32:33.119404+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:32:33.119404+0000","last_undegraded":"2024-08-17T17:32:33.119404+0000","last_fullsized":"2024-08-17T17:32:33.119404+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4750,"log_dups_size":0,"ondisk_log_size":4750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311296000,"num_objects":4750,"num_object_clones":0,"num_object_copies":14250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4750,"num_write_kb":304000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":492,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.379461+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:32:16.379461+0000","last_peered":"2024-08-17T17:32:16.379461+0000","last_clean":"2024-08-17T17:32:16.379461+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:32:16.379461+0000","last_undegraded":"2024-08-17T17:32:16.379461+0000","last_fullsized":"2024-08-17T17:32:16.379461+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":281,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384362+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:32:16.384362+0000","last_peered":"2024-08-17T17:32:16.384362+0000","last_clean":"2024-08-17T17:32:16.384362+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:32:16.384362+0000","last_undegraded":"2024-08-17T17:32:16.384362+0000","last_fullsized":"2024-08-17T17:32:16.384362+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"327'4854","reported_seq":4869,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:33.122921+0000","last_change":"2024-08-17T17:31:36.271872+0000","last_active":"2024-08-17T17:32:33.122921+0000","last_peered":"2024-08-17T17:32:33.122921+0000","last_clean":"2024-08-17T17:32:33.122921+0000","last_became_active":"2024-08-17T17:31:36.271398+0000","last_became_peered":"2024-08-17T17:31:36.271398+0000","last_unstale":"2024-08-17T17:32:33.122921+0000","last_undegraded":"2024-08-17T17:32:33.122921+0000","last_fullsized":"2024-08-17T17:32:33.122921+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:14:11.815394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318111744,"num_objects":4854,"num_object_clones":0,"num_object_copies":14562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4854,"num_write_kb":310656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":282,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375926+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:32:16.375926+0000","last_peered":"2024-08-17T17:32:16.375926+0000","last_clean":"2024-08-17T17:32:16.375926+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:32:16.375926+0000","last_undegraded":"2024-08-17T17:32:16.375926+0000","last_fullsized":"2024-08-17T17:32:16.375926+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":298,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.385012+0000","last_change":"2024-08-17T17:10:55.329724+0000","last_active":"2024-08-17T17:32:16.385012+0000","last_peered":"2024-08-17T17:32:16.385012+0000","last_clean":"2024-08-17T17:32:16.385012+0000","last_became_active":"2024-08-17T17:10:55.329547+0000","last_became_peered":"2024-08-17T17:10:55.329547+0000","last_unstale":"2024-08-17T17:32:16.385012+0000","last_undegraded":"2024-08-17T17:32:16.385012+0000","last_fullsized":"2024-08-17T17:32:16.385012+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:50.610910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"327'4781","reported_seq":4808,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.484640+0000","last_change":"2024-08-17T17:31:36.272676+0000","last_active":"2024-08-17T17:32:31.484640+0000","last_peered":"2024-08-17T17:32:31.484640+0000","last_clean":"2024-08-17T17:32:31.484640+0000","last_became_active":"2024-08-17T17:31:36.272403+0000","last_became_peered":"2024-08-17T17:31:36.272403+0000","last_unstale":"2024-08-17T17:32:31.484640+0000","last_undegraded":"2024-08-17T17:32:31.484640+0000","last_fullsized":"2024-08-17T17:32:31.484640+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4781,"log_dups_size":0,"ondisk_log_size":4781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:51.087810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313327616,"num_objects":4781,"num_object_clones":0,"num_object_copies":14343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4781,"num_write_kb":305984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":299,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.383967+0000","last_change":"2024-08-17T17:26:32.216646+0000","last_active":"2024-08-17T17:32:16.383967+0000","last_peered":"2024-08-17T17:32:16.383967+0000","last_clean":"2024-08-17T17:32:16.383967+0000","last_became_active":"2024-08-17T17:26:32.216155+0000","last_became_peered":"2024-08-17T17:26:32.216155+0000","last_unstale":"2024-08-17T17:32:16.383967+0000","last_undegraded":"2024-08-17T17:32:16.383967+0000","last_fullsized":"2024-08-17T17:32:16.383967+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:55.449331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":281,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.385263+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:32:16.385263+0000","last_peered":"2024-08-17T17:32:16.385263+0000","last_clean":"2024-08-17T17:32:16.385263+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:32:16.385263+0000","last_undegraded":"2024-08-17T17:32:16.385263+0000","last_fullsized":"2024-08-17T17:32:16.385263+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"326'4681","reported_seq":4696,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:30.388529+0000","last_change":"2024-08-17T17:31:36.272047+0000","last_active":"2024-08-17T17:32:30.388529+0000","last_peered":"2024-08-17T17:32:30.388529+0000","last_clean":"2024-08-17T17:32:30.388529+0000","last_became_active":"2024-08-17T17:31:36.269393+0000","last_became_peered":"2024-08-17T17:31:36.269393+0000","last_unstale":"2024-08-17T17:32:30.388529+0000","last_undegraded":"2024-08-17T17:32:30.388529+0000","last_fullsized":"2024-08-17T17:32:30.388529+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4681,"log_dups_size":0,"ondisk_log_size":4681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:56.899364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306774016,"num_objects":4681,"num_object_clones":0,"num_object_copies":14043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4681,"num_write_kb":299584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":455,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376626+0000","last_change":"2024-08-17T17:26:32.212135+0000","last_active":"2024-08-17T17:32:16.376626+0000","last_peered":"2024-08-17T17:32:16.376626+0000","last_clean":"2024-08-17T17:32:16.376626+0000","last_became_active":"2024-08-17T17:26:32.211604+0000","last_became_peered":"2024-08-17T17:26:32.211604+0000","last_unstale":"2024-08-17T17:32:16.376626+0000","last_undegraded":"2024-08-17T17:32:16.376626+0000","last_fullsized":"2024-08-17T17:32:16.376626+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:20:49.886809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":399,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376263+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:32:16.376263+0000","last_peered":"2024-08-17T17:32:16.376263+0000","last_clean":"2024-08-17T17:32:16.376263+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:32:16.376263+0000","last_undegraded":"2024-08-17T17:32:16.376263+0000","last_fullsized":"2024-08-17T17:32:16.376263+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"326'4722","reported_seq":4737,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:30.385931+0000","last_change":"2024-08-17T17:31:36.270955+0000","last_active":"2024-08-17T17:32:30.385931+0000","last_peered":"2024-08-17T17:32:30.385931+0000","last_clean":"2024-08-17T17:32:30.385931+0000","last_became_active":"2024-08-17T17:31:36.270618+0000","last_became_peered":"2024-08-17T17:31:36.270618+0000","last_unstale":"2024-08-17T17:32:30.385931+0000","last_undegraded":"2024-08-17T17:32:30.385931+0000","last_fullsized":"2024-08-17T17:32:30.385931+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4722,"log_dups_size":0,"ondisk_log_size":4722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:46.731269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309460992,"num_objects":4722,"num_object_clones":0,"num_object_copies":14166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4722,"num_write_kb":302208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":648,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308626+0000","last_change":"2024-08-17T17:26:33.204740+0000","last_active":"2024-08-17T17:32:05.308626+0000","last_peered":"2024-08-17T17:32:05.308626+0000","last_clean":"2024-08-17T17:32:05.308626+0000","last_became_active":"2024-08-17T17:26:33.204340+0000","last_became_peered":"2024-08-17T17:26:33.204340+0000","last_unstale":"2024-08-17T17:32:05.308626+0000","last_undegraded":"2024-08-17T17:32:05.308626+0000","last_fullsized":"2024-08-17T17:32:05.308626+0000","mapping_epoch":271,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:43.802253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":233,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329920+0000","last_change":"2024-08-17T17:26:32.219571+0000","last_active":"2024-08-17T17:32:05.329920+0000","last_peered":"2024-08-17T17:32:05.329920+0000","last_clean":"2024-08-17T17:32:05.329920+0000","last_became_active":"2024-08-17T17:26:32.219094+0000","last_became_peered":"2024-08-17T17:26:32.219094+0000","last_unstale":"2024-08-17T17:32:05.329920+0000","last_undegraded":"2024-08-17T17:32:05.329920+0000","last_fullsized":"2024-08-17T17:32:05.329920+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:09.450772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":441,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.377735+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:32:16.377735+0000","last_peered":"2024-08-17T17:32:16.377735+0000","last_clean":"2024-08-17T17:32:16.377735+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:32:16.377735+0000","last_undegraded":"2024-08-17T17:32:16.377735+0000","last_fullsized":"2024-08-17T17:32:16.377735+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":281,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376191+0000","last_change":"2024-08-17T17:26:32.215221+0000","last_active":"2024-08-17T17:32:16.376191+0000","last_peered":"2024-08-17T17:32:16.376191+0000","last_clean":"2024-08-17T17:32:16.376191+0000","last_became_active":"2024-08-17T17:26:32.214756+0000","last_became_peered":"2024-08-17T17:26:32.214756+0000","last_unstale":"2024-08-17T17:32:16.376191+0000","last_undegraded":"2024-08-17T17:32:16.376191+0000","last_fullsized":"2024-08-17T17:32:16.376191+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:15.545194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"327'4817","reported_seq":4844,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:32.985095+0000","last_change":"2024-08-17T17:31:36.270236+0000","last_active":"2024-08-17T17:32:32.985095+0000","last_peered":"2024-08-17T17:32:32.985095+0000","last_clean":"2024-08-17T17:32:32.985095+0000","last_became_active":"2024-08-17T17:31:36.269950+0000","last_became_peered":"2024-08-17T17:31:36.269950+0000","last_unstale":"2024-08-17T17:32:32.985095+0000","last_undegraded":"2024-08-17T17:32:32.985095+0000","last_fullsized":"2024-08-17T17:32:32.985095+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4817,"log_dups_size":0,"ondisk_log_size":4817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:30.882354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315686912,"num_objects":4817,"num_object_clones":0,"num_object_copies":14451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4817,"num_write_kb":308288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":322,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308628+0000","last_change":"2024-08-17T17:26:32.212907+0000","last_active":"2024-08-17T17:32:05.308628+0000","last_peered":"2024-08-17T17:32:05.308628+0000","last_clean":"2024-08-17T17:32:05.308628+0000","last_became_active":"2024-08-17T17:26:32.212549+0000","last_became_peered":"2024-08-17T17:26:32.212549+0000","last_unstale":"2024-08-17T17:32:05.308628+0000","last_undegraded":"2024-08-17T17:32:05.308628+0000","last_fullsized":"2024-08-17T17:32:05.308628+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:35.071333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":263,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.383635+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:32:16.383635+0000","last_peered":"2024-08-17T17:32:16.383635+0000","last_clean":"2024-08-17T17:32:16.383635+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:32:16.383635+0000","last_undegraded":"2024-08-17T17:32:16.383635+0000","last_fullsized":"2024-08-17T17:32:16.383635+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":171,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.407516+0000","last_change":"2024-08-17T17:24:37.073148+0000","last_active":"2024-08-17T17:32:31.407516+0000","last_peered":"2024-08-17T17:32:31.407516+0000","last_clean":"2024-08-17T17:32:31.407516+0000","last_became_active":"2024-08-17T17:24:37.072372+0000","last_became_peered":"2024-08-17T17:24:37.072372+0000","last_unstale":"2024-08-17T17:32:31.407516+0000","last_undegraded":"2024-08-17T17:32:31.407516+0000","last_fullsized":"2024-08-17T17:32:31.407516+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.659683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"327'4945","reported_seq":4961,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:33.095139+0000","last_change":"2024-08-17T17:31:36.269392+0000","last_active":"2024-08-17T17:32:33.095139+0000","last_peered":"2024-08-17T17:32:33.095139+0000","last_clean":"2024-08-17T17:32:33.095139+0000","last_became_active":"2024-08-17T17:31:36.269111+0000","last_became_peered":"2024-08-17T17:31:36.269111+0000","last_unstale":"2024-08-17T17:32:33.095139+0000","last_undegraded":"2024-08-17T17:32:33.095139+0000","last_fullsized":"2024-08-17T17:32:33.095139+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4945,"log_dups_size":0,"ondisk_log_size":4945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:37.539703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324075520,"num_objects":4945,"num_object_clones":0,"num_object_copies":14835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4945,"num_write_kb":316480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":517,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.388628+0000","last_change":"2024-08-17T17:24:36.969933+0000","last_active":"2024-08-17T17:32:16.388628+0000","last_peered":"2024-08-17T17:32:16.388628+0000","last_clean":"2024-08-17T17:32:16.388628+0000","last_became_active":"2024-08-17T17:24:36.969657+0000","last_became_peered":"2024-08-17T17:24:36.969657+0000","last_unstale":"2024-08-17T17:32:16.388628+0000","last_undegraded":"2024-08-17T17:32:16.388628+0000","last_fullsized":"2024-08-17T17:32:16.388628+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:10.367450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.389839+0000","last_change":"2024-08-17T17:26:50.368470+0000","last_active":"2024-08-17T17:32:16.389839+0000","last_peered":"2024-08-17T17:32:16.389839+0000","last_clean":"2024-08-17T17:32:16.389839+0000","last_became_active":"2024-08-17T17:26:50.367844+0000","last_became_peered":"2024-08-17T17:26:50.367844+0000","last_unstale":"2024-08-17T17:32:16.389839+0000","last_undegraded":"2024-08-17T17:32:16.389839+0000","last_fullsized":"2024-08-17T17:32:16.389839+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:02.244611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":536,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.393053+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:32:31.393053+0000","last_peered":"2024-08-17T17:32:31.393053+0000","last_clean":"2024-08-17T17:32:31.393053+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:32:31.393053+0000","last_undegraded":"2024-08-17T17:32:31.393053+0000","last_fullsized":"2024-08-17T17:32:31.393053+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"327'4807","reported_seq":4834,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:32.981839+0000","last_change":"2024-08-17T17:31:36.275334+0000","last_active":"2024-08-17T17:32:32.981839+0000","last_peered":"2024-08-17T17:32:32.981839+0000","last_clean":"2024-08-17T17:32:32.981839+0000","last_became_active":"2024-08-17T17:31:36.274901+0000","last_became_peered":"2024-08-17T17:31:36.274901+0000","last_unstale":"2024-08-17T17:32:32.981839+0000","last_undegraded":"2024-08-17T17:32:32.981839+0000","last_fullsized":"2024-08-17T17:32:32.981839+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4807,"log_dups_size":0,"ondisk_log_size":4807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:04.919838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315031552,"num_objects":4807,"num_object_clones":0,"num_object_copies":14421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4807,"num_write_kb":307648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":327,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.392423+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:32:31.392423+0000","last_peered":"2024-08-17T17:32:31.392423+0000","last_clean":"2024-08-17T17:32:31.392423+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:32:31.392423+0000","last_undegraded":"2024-08-17T17:32:31.392423+0000","last_fullsized":"2024-08-17T17:32:31.392423+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":302,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384564+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:32:16.384564+0000","last_peered":"2024-08-17T17:32:16.384564+0000","last_clean":"2024-08-17T17:32:16.384564+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:32:16.384564+0000","last_undegraded":"2024-08-17T17:32:16.384564+0000","last_fullsized":"2024-08-17T17:32:16.384564+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":233,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.390057+0000","last_change":"2024-08-17T17:26:50.310453+0000","last_active":"2024-08-17T17:32:16.390057+0000","last_peered":"2024-08-17T17:32:16.390057+0000","last_clean":"2024-08-17T17:32:16.390057+0000","last_became_active":"2024-08-17T17:26:50.310180+0000","last_became_peered":"2024-08-17T17:26:50.310180+0000","last_unstale":"2024-08-17T17:32:16.390057+0000","last_undegraded":"2024-08-17T17:32:16.390057+0000","last_fullsized":"2024-08-17T17:32:16.390057+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:38.334936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"326'4634","reported_seq":4659,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:29.267446+0000","last_change":"2024-08-17T17:31:36.270808+0000","last_active":"2024-08-17T17:32:29.267446+0000","last_peered":"2024-08-17T17:32:29.267446+0000","last_clean":"2024-08-17T17:32:29.267446+0000","last_became_active":"2024-08-17T17:31:36.270536+0000","last_became_peered":"2024-08-17T17:31:36.270536+0000","last_unstale":"2024-08-17T17:32:29.267446+0000","last_undegraded":"2024-08-17T17:32:29.267446+0000","last_fullsized":"2024-08-17T17:32:29.267446+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4634,"log_dups_size":0,"ondisk_log_size":4634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:31.074486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303693824,"num_objects":4634,"num_object_clones":0,"num_object_copies":13902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4634,"num_write_kb":296576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329952+0000","last_change":"2024-08-17T17:25:46.633494+0000","last_active":"2024-08-17T17:32:05.329952+0000","last_peered":"2024-08-17T17:32:05.329952+0000","last_clean":"2024-08-17T17:32:05.329952+0000","last_became_active":"2024-08-17T17:25:46.633216+0000","last_became_peered":"2024-08-17T17:25:46.633216+0000","last_unstale":"2024-08-17T17:32:05.329952+0000","last_undegraded":"2024-08-17T17:32:05.329952+0000","last_fullsized":"2024-08-17T17:32:05.329952+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:14.957437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":487,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308740+0000","last_change":"2024-08-17T17:26:50.338185+0000","last_active":"2024-08-17T17:32:05.308740+0000","last_peered":"2024-08-17T17:32:05.308740+0000","last_clean":"2024-08-17T17:32:05.308740+0000","last_became_active":"2024-08-17T17:26:50.337992+0000","last_became_peered":"2024-08-17T17:26:50.337992+0000","last_unstale":"2024-08-17T17:32:05.308740+0000","last_undegraded":"2024-08-17T17:32:05.308740+0000","last_fullsized":"2024-08-17T17:32:05.308740+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:26.720733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":486,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.392628+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:32:31.392628+0000","last_peered":"2024-08-17T17:32:31.392628+0000","last_clean":"2024-08-17T17:32:31.392628+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:32:31.392628+0000","last_undegraded":"2024-08-17T17:32:31.392628+0000","last_fullsized":"2024-08-17T17:32:31.392628+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"326'4589","reported_seq":4614,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:30.139154+0000","last_change":"2024-08-17T17:31:36.272820+0000","last_active":"2024-08-17T17:32:30.139154+0000","last_peered":"2024-08-17T17:32:30.139154+0000","last_clean":"2024-08-17T17:32:30.139154+0000","last_became_active":"2024-08-17T17:31:36.272457+0000","last_became_peered":"2024-08-17T17:31:36.272457+0000","last_unstale":"2024-08-17T17:32:30.139154+0000","last_undegraded":"2024-08-17T17:32:30.139154+0000","last_fullsized":"2024-08-17T17:32:30.139154+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4589,"log_dups_size":0,"ondisk_log_size":4589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:39.167644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300744704,"num_objects":4589,"num_object_clones":0,"num_object_copies":13767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4589,"num_write_kb":293696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":342,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.387488+0000","last_change":"2024-08-17T17:26:50.315231+0000","last_active":"2024-08-17T17:32:16.387488+0000","last_peered":"2024-08-17T17:32:16.387488+0000","last_clean":"2024-08-17T17:32:16.387488+0000","last_became_active":"2024-08-17T17:26:50.314945+0000","last_became_peered":"2024-08-17T17:26:50.314945+0000","last_unstale":"2024-08-17T17:32:16.387488+0000","last_undegraded":"2024-08-17T17:32:16.387488+0000","last_fullsized":"2024-08-17T17:32:16.387488+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:13.740267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308792+0000","last_change":"2024-08-17T17:26:32.215417+0000","last_active":"2024-08-17T17:32:05.308792+0000","last_peered":"2024-08-17T17:32:05.308792+0000","last_clean":"2024-08-17T17:32:05.308792+0000","last_became_active":"2024-08-17T17:26:32.215160+0000","last_became_peered":"2024-08-17T17:26:32.215160+0000","last_unstale":"2024-08-17T17:32:05.308792+0000","last_undegraded":"2024-08-17T17:32:05.308792+0000","last_fullsized":"2024-08-17T17:32:05.308792+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:17.370439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":368,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.329832+0000","last_change":"2024-08-17T17:26:32.221201+0000","last_active":"2024-08-17T17:32:05.329832+0000","last_peered":"2024-08-17T17:32:05.329832+0000","last_clean":"2024-08-17T17:32:05.329832+0000","last_became_active":"2024-08-17T17:26:32.220913+0000","last_became_peered":"2024-08-17T17:26:32.220913+0000","last_unstale":"2024-08-17T17:32:05.329832+0000","last_undegraded":"2024-08-17T17:32:05.329832+0000","last_fullsized":"2024-08-17T17:32:05.329832+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:20.201903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"326'4588","reported_seq":4613,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:30.117566+0000","last_change":"2024-08-17T17:31:36.271799+0000","last_active":"2024-08-17T17:32:30.117566+0000","last_peered":"2024-08-17T17:32:30.117566+0000","last_clean":"2024-08-17T17:32:30.117566+0000","last_became_active":"2024-08-17T17:31:36.271569+0000","last_became_peered":"2024-08-17T17:31:36.271569+0000","last_unstale":"2024-08-17T17:32:30.117566+0000","last_undegraded":"2024-08-17T17:32:30.117566+0000","last_fullsized":"2024-08-17T17:32:30.117566+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4588,"log_dups_size":0,"ondisk_log_size":4588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:28.450375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300679168,"num_objects":4588,"num_object_clones":0,"num_object_copies":13764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4588,"num_write_kb":293632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":504,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376339+0000","last_change":"2024-08-17T17:26:32.209014+0000","last_active":"2024-08-17T17:32:16.376339+0000","last_peered":"2024-08-17T17:32:16.376339+0000","last_clean":"2024-08-17T17:32:16.376339+0000","last_became_active":"2024-08-17T17:26:32.208539+0000","last_became_peered":"2024-08-17T17:26:32.208539+0000","last_unstale":"2024-08-17T17:32:16.376339+0000","last_undegraded":"2024-08-17T17:32:16.376339+0000","last_fullsized":"2024-08-17T17:32:16.376339+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:40:15.666378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":485,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.394582+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:32:31.394582+0000","last_peered":"2024-08-17T17:32:31.394582+0000","last_clean":"2024-08-17T17:32:31.394582+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:32:31.394582+0000","last_undegraded":"2024-08-17T17:32:31.394582+0000","last_fullsized":"2024-08-17T17:32:31.394582+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":493,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376465+0000","last_change":"2024-08-17T17:26:32.211876+0000","last_active":"2024-08-17T17:32:16.376465+0000","last_peered":"2024-08-17T17:32:16.376465+0000","last_clean":"2024-08-17T17:32:16.376465+0000","last_became_active":"2024-08-17T17:26:32.211212+0000","last_became_peered":"2024-08-17T17:26:32.211212+0000","last_unstale":"2024-08-17T17:32:16.376465+0000","last_undegraded":"2024-08-17T17:32:16.376465+0000","last_fullsized":"2024-08-17T17:32:16.376465+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:50.032892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"326'4555","reported_seq":4577,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:29.252168+0000","last_change":"2024-08-17T17:31:36.269501+0000","last_active":"2024-08-17T17:32:29.252168+0000","last_peered":"2024-08-17T17:32:29.252168+0000","last_clean":"2024-08-17T17:32:29.252168+0000","last_became_active":"2024-08-17T17:31:36.269223+0000","last_became_peered":"2024-08-17T17:31:36.269223+0000","last_unstale":"2024-08-17T17:32:29.252168+0000","last_undegraded":"2024-08-17T17:32:29.252168+0000","last_fullsized":"2024-08-17T17:32:29.252168+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:00.587765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4555,"num_write_kb":291520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":175,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.407414+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:32:31.407414+0000","last_peered":"2024-08-17T17:32:31.407414+0000","last_clean":"2024-08-17T17:32:31.407414+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:32:31.407414+0000","last_undegraded":"2024-08-17T17:32:31.407414+0000","last_fullsized":"2024-08-17T17:32:31.407414+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":311,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376207+0000","last_change":"2024-08-17T17:26:32.218111+0000","last_active":"2024-08-17T17:32:16.376207+0000","last_peered":"2024-08-17T17:32:16.376207+0000","last_clean":"2024-08-17T17:32:16.376207+0000","last_became_active":"2024-08-17T17:26:32.217705+0000","last_became_peered":"2024-08-17T17:26:32.217705+0000","last_unstale":"2024-08-17T17:32:16.376207+0000","last_undegraded":"2024-08-17T17:32:16.376207+0000","last_fullsized":"2024-08-17T17:32:16.376207+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:50.751085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":281,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.383894+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:32:16.383894+0000","last_peered":"2024-08-17T17:32:16.383894+0000","last_clean":"2024-08-17T17:32:16.383894+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:32:16.383894+0000","last_undegraded":"2024-08-17T17:32:16.383894+0000","last_fullsized":"2024-08-17T17:32:16.383894+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":281,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384468+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:32:16.384468+0000","last_peered":"2024-08-17T17:32:16.384468+0000","last_clean":"2024-08-17T17:32:16.384468+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:32:16.384468+0000","last_undegraded":"2024-08-17T17:32:16.384468+0000","last_fullsized":"2024-08-17T17:32:16.384468+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":304,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384863+0000","last_change":"2024-08-17T17:26:32.215288+0000","last_active":"2024-08-17T17:32:16.384863+0000","last_peered":"2024-08-17T17:32:16.384863+0000","last_clean":"2024-08-17T17:32:16.384863+0000","last_became_active":"2024-08-17T17:26:32.214754+0000","last_became_peered":"2024-08-17T17:26:32.214754+0000","last_unstale":"2024-08-17T17:32:16.384863+0000","last_undegraded":"2024-08-17T17:32:16.384863+0000","last_fullsized":"2024-08-17T17:32:16.384863+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.684058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":485,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.392507+0000","last_change":"2024-08-17T17:10:55.322652+0000","last_active":"2024-08-17T17:32:31.392507+0000","last_peered":"2024-08-17T17:32:31.392507+0000","last_clean":"2024-08-17T17:32:31.392507+0000","last_became_active":"2024-08-17T17:10:55.322397+0000","last_became_peered":"2024-08-17T17:10:55.322397+0000","last_unstale":"2024-08-17T17:32:31.392507+0000","last_undegraded":"2024-08-17T17:32:31.392507+0000","last_fullsized":"2024-08-17T17:32:31.392507+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:10.999233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":257,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.308713+0000","last_change":"2024-08-17T17:26:32.207981+0000","last_active":"2024-08-17T17:32:05.308713+0000","last_peered":"2024-08-17T17:32:05.308713+0000","last_clean":"2024-08-17T17:32:05.308713+0000","last_became_active":"2024-08-17T17:26:32.207636+0000","last_became_peered":"2024-08-17T17:26:32.207636+0000","last_unstale":"2024-08-17T17:32:05.308713+0000","last_undegraded":"2024-08-17T17:32:05.308713+0000","last_fullsized":"2024-08-17T17:32:05.308713+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:32.412636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.330143+0000","last_change":"2024-08-17T17:26:32.210741+0000","last_active":"2024-08-17T17:32:05.330143+0000","last_peered":"2024-08-17T17:32:05.330143+0000","last_clean":"2024-08-17T17:32:05.330143+0000","last_became_active":"2024-08-17T17:26:32.210417+0000","last_became_peered":"2024-08-17T17:26:32.210417+0000","last_unstale":"2024-08-17T17:32:05.330143+0000","last_undegraded":"2024-08-17T17:32:05.330143+0000","last_fullsized":"2024-08-17T17:32:05.330143+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:41.290842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":441,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.378414+0000","last_change":"2024-08-17T17:26:32.211602+0000","last_active":"2024-08-17T17:32:16.378414+0000","last_peered":"2024-08-17T17:32:16.378414+0000","last_clean":"2024-08-17T17:32:16.378414+0000","last_became_active":"2024-08-17T17:26:32.211288+0000","last_became_peered":"2024-08-17T17:26:32.211288+0000","last_unstale":"2024-08-17T17:32:16.378414+0000","last_undegraded":"2024-08-17T17:32:16.378414+0000","last_fullsized":"2024-08-17T17:32:16.378414+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:49.979367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":374,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.330226+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:32:05.330226+0000","last_peered":"2024-08-17T17:32:05.330226+0000","last_clean":"2024-08-17T17:32:05.330226+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:32:05.330226+0000","last_undegraded":"2024-08-17T17:32:05.330226+0000","last_fullsized":"2024-08-17T17:32:05.330226+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":442,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376253+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:32:16.376253+0000","last_peered":"2024-08-17T17:32:16.376253+0000","last_clean":"2024-08-17T17:32:16.376253+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:32:16.376253+0000","last_undegraded":"2024-08-17T17:32:16.376253+0000","last_fullsized":"2024-08-17T17:32:16.376253+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":298,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376206+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:32:16.376206+0000","last_peered":"2024-08-17T17:32:16.376206+0000","last_clean":"2024-08-17T17:32:16.376206+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:32:16.376206+0000","last_undegraded":"2024-08-17T17:32:16.376206+0000","last_fullsized":"2024-08-17T17:32:16.376206+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":474,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376544+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:32:16.376544+0000","last_peered":"2024-08-17T17:32:16.376544+0000","last_clean":"2024-08-17T17:32:16.376544+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:32:16.376544+0000","last_undegraded":"2024-08-17T17:32:16.376544+0000","last_fullsized":"2024-08-17T17:32:16.376544+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":254,"reported_epoch":325,"state":"active+clean","last_fresh":"2024-08-17T17:32:05.309122+0000","last_change":"2024-08-17T17:26:32.215823+0000","last_active":"2024-08-17T17:32:05.309122+0000","last_peered":"2024-08-17T17:32:05.309122+0000","last_clean":"2024-08-17T17:32:05.309122+0000","last_became_active":"2024-08-17T17:26:32.214110+0000","last_became_peered":"2024-08-17T17:26:32.214110+0000","last_unstale":"2024-08-17T17:32:05.309122+0000","last_undegraded":"2024-08-17T17:32:05.309122+0000","last_fullsized":"2024-08-17T17:32:05.309122+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:41.088172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":484,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.392160+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:32:31.392160+0000","last_peered":"2024-08-17T17:32:31.392160+0000","last_clean":"2024-08-17T17:32:31.392160+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:32:31.392160+0000","last_undegraded":"2024-08-17T17:32:31.392160+0000","last_fullsized":"2024-08-17T17:32:31.392160+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":441,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.375845+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:32:16.375845+0000","last_peered":"2024-08-17T17:32:16.375845+0000","last_clean":"2024-08-17T17:32:16.375845+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:32:16.375845+0000","last_undegraded":"2024-08-17T17:32:16.375845+0000","last_fullsized":"2024-08-17T17:32:16.375845+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":300,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.383758+0000","last_change":"2024-08-17T17:10:55.317120+0000","last_active":"2024-08-17T17:32:16.383758+0000","last_peered":"2024-08-17T17:32:16.383758+0000","last_clean":"2024-08-17T17:32:16.383758+0000","last_became_active":"2024-08-17T17:10:55.316813+0000","last_became_peered":"2024-08-17T17:10:55.316813+0000","last_unstale":"2024-08-17T17:32:16.383758+0000","last_undegraded":"2024-08-17T17:32:16.383758+0000","last_fullsized":"2024-08-17T17:32:16.383758+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:59.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":505,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.387582+0000","last_change":"2024-08-17T17:12:40.334361+0000","last_active":"2024-08-17T17:32:16.387582+0000","last_peered":"2024-08-17T17:32:16.387582+0000","last_clean":"2024-08-17T17:32:16.387582+0000","last_became_active":"2024-08-17T17:12:40.333846+0000","last_became_peered":"2024-08-17T17:12:40.333846+0000","last_unstale":"2024-08-17T17:32:16.387582+0000","last_undegraded":"2024-08-17T17:32:16.387582+0000","last_fullsized":"2024-08-17T17:32:16.387582+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:40:43.825525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":182,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.407217+0000","last_change":"2024-08-17T17:26:32.214552+0000","last_active":"2024-08-17T17:32:31.407217+0000","last_peered":"2024-08-17T17:32:31.407217+0000","last_clean":"2024-08-17T17:32:31.407217+0000","last_became_active":"2024-08-17T17:26:32.214338+0000","last_became_peered":"2024-08-17T17:26:32.214338+0000","last_unstale":"2024-08-17T17:32:31.407217+0000","last_undegraded":"2024-08-17T17:32:31.407217+0000","last_fullsized":"2024-08-17T17:32:31.407217+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:21.212607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":308,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384385+0000","last_change":"2024-08-17T17:12:40.330722+0000","last_active":"2024-08-17T17:32:16.384385+0000","last_peered":"2024-08-17T17:32:16.384385+0000","last_clean":"2024-08-17T17:32:16.384385+0000","last_became_active":"2024-08-17T17:12:40.330108+0000","last_became_peered":"2024-08-17T17:12:40.330108+0000","last_unstale":"2024-08-17T17:32:16.384385+0000","last_undegraded":"2024-08-17T17:32:16.384385+0000","last_fullsized":"2024-08-17T17:32:16.384385+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:22.866543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":181,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384455+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:32:16.384455+0000","last_peered":"2024-08-17T17:32:16.384455+0000","last_clean":"2024-08-17T17:32:16.384455+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:32:16.384455+0000","last_undegraded":"2024-08-17T17:32:16.384455+0000","last_fullsized":"2024-08-17T17:32:16.384455+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":318,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.383637+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:32:16.383637+0000","last_peered":"2024-08-17T17:32:16.383637+0000","last_clean":"2024-08-17T17:32:16.383637+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:32:16.383637+0000","last_undegraded":"2024-08-17T17:32:16.383637+0000","last_fullsized":"2024-08-17T17:32:16.383637+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":181,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384748+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:32:16.384748+0000","last_peered":"2024-08-17T17:32:16.384748+0000","last_clean":"2024-08-17T17:32:16.384748+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:32:16.384748+0000","last_undegraded":"2024-08-17T17:32:16.384748+0000","last_fullsized":"2024-08-17T17:32:16.384748+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":487,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.380294+0000","last_change":"2024-08-17T17:24:36.972596+0000","last_active":"2024-08-17T17:32:16.380294+0000","last_peered":"2024-08-17T17:32:16.380294+0000","last_clean":"2024-08-17T17:32:16.380294+0000","last_became_active":"2024-08-17T17:24:36.971541+0000","last_became_peered":"2024-08-17T17:24:36.971541+0000","last_unstale":"2024-08-17T17:32:16.380294+0000","last_undegraded":"2024-08-17T17:32:16.380294+0000","last_fullsized":"2024-08-17T17:32:16.380294+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:38.457003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":459,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.376526+0000","last_change":"2024-08-17T17:12:40.330325+0000","last_active":"2024-08-17T17:32:16.376526+0000","last_peered":"2024-08-17T17:32:16.376526+0000","last_clean":"2024-08-17T17:32:16.376526+0000","last_became_active":"2024-08-17T17:12:40.329967+0000","last_became_peered":"2024-08-17T17:12:40.329967+0000","last_unstale":"2024-08-17T17:32:16.376526+0000","last_undegraded":"2024-08-17T17:32:16.376526+0000","last_fullsized":"2024-08-17T17:32:16.376526+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:17:38.875529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":281,"reported_epoch":326,"state":"active+clean","last_fresh":"2024-08-17T17:32:16.384155+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:32:16.384155+0000","last_peered":"2024-08-17T17:32:16.384155+0000","last_clean":"2024-08-17T17:32:16.384155+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:32:16.384155+0000","last_undegraded":"2024-08-17T17:32:16.384155+0000","last_fullsized":"2024-08-17T17:32:16.384155+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":177,"reported_epoch":327,"state":"active+clean","last_fresh":"2024-08-17T17:32:31.407101+0000","last_change":"2024-08-17T17:24:37.032048+0000","last_active":"2024-08-17T17:32:31.407101+0000","last_peered":"2024-08-17T17:32:31.407101+0000","last_clean":"2024-08-17T17:32:31.407101+0000","last_became_active":"2024-08-17T17:24:36.972161+0000","last_became_peered":"2024-08-17T17:24:36.972161+0000","last_unstale":"2024-08-17T17:32:31.407101+0000","last_undegraded":"2024-08-17T17:32:31.407101+0000","last_fullsized":"2024-08-17T17:32:31.407101+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:55.363820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4992008192,"num_objects":76172,"num_object_clones":0,"num_object_copies":228516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76172,"num_write_kb":4875008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14817755136,"data_stored":14817755136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76172,"ondisk_log_size":76172,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1162,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":270,"seq":1159641169993,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2460044,"kb_used_data":2085636,"kb_used_omap":12,"kb_used_meta":374387,"kb_avail":91285108,"statfs":{"total":95995035648,"available":93475950592,"internally_reserved":0,"allocated":2135691264,"data_stored":2134028417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12410,"internal_metadata":383373190},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463363,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3069520,"kb_used_data":2167496,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":90675632,"statfs":{"total":95995035648,"available":92851847168,"internally_reserved":0,"allocated":2219515904,"data_stored":2217857066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492037,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":996400,"kb_used_data":596904,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92748752,"statfs":{"total":95995035648,"available":94974722048,"internally_reserved":0,"allocated":611229696,"data_stored":609589610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986637,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1808400,"kb_used_data":911620,"kb_used_omap":1,"kb_used_meta":896766,"kb_avail":91936752,"statfs":{"total":95995035648,"available":94143234048,"internally_reserved":0,"allocated":933498880,"data_stored":931843394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918288728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225456,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3157484,"kb_used_data":2361508,"kb_used_omap":4,"kb_used_meta":795963,"kb_avail":90587668,"statfs":{"total":95995035648,"available":92761772032,"internally_reserved":0,"allocated":2418184192,"data_stored":2416548430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":815066560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":120,"seq":515396075781,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3562804,"kb_used_data":2930284,"kb_used_omap":6,"kb_used_meta":632505,"kb_avail":90182348,"statfs":{"total":95995035648,"available":92346724352,"internally_reserved":0,"allocated":3000610816,"data_stored":2998967665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":647685563},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533674,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2431532,"kb_used_data":1504740,"kb_used_omap":5,"kb_used_meta":926778,"kb_avail":91313620,"statfs":{"total":95995035648,"available":93505146880,"internally_reserved":0,"allocated":1540853760,"data_stored":1539216796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":949020912},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089651,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2793288,"kb_used_data":1842364,"kb_used_omap":5,"kb_used_meta":950906,"kb_avail":90951864,"statfs":{"total":95995035648,"available":93134708736,"internally_reserved":0,"allocated":1886580736,"data_stored":1884929981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":973727803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1908015104,"data_stored":1908015104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1556086784,"data_stored":1556086784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3006136320,"data_stored":3006136320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2424373248,"data_stored":2424373248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":945160192,"data_stored":945160192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":609943552,"data_stored":609943552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2221735936,"data_stored":2221735936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2146304000,"data_stored":2146304000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:32:40.578 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:32:40.579 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:32:40.579 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-08-17T17:32:40.579 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:32:40.579 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:32:40.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-08-17T17:32:40.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.645 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:40.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:40.664 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:40.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:40.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:40.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:40.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:40.982 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 85818 85802 99.276 63.5 0.025899 0.0100674 2024-08-17T17:32:41.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:41.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:41.158 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:41.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:41.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:41.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:41.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:41.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:41.416 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:41 smithi142 ceph-mon[106729]: pgmap v1130: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:32:41.416 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:41 smithi142 ceph-mon[103614]: pgmap v1130: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:32:41.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:41.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:41 smithi152 ceph-mon[93456]: pgmap v1130: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:32:41.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:41.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:41.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:41.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:41.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:41.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:41.982 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 86863 86847 98.6586 65.3125 0.0130575 0.0101308 2024-08-17T17:32:42.032 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:32:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:32:41] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:32:42.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:42.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:42.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:42.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:42.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:42.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:42.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:42.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:42.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:42.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:42.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:42.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:42.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:42.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:42.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:42.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:42.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:42.916 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:42.982 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 87934 87918 98.0921 66.9375 0.012112 0.0101889 2024-08-17T17:32:43.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:43.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:43.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:43.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:43.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:43.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:43 smithi142 ceph-mon[106729]: pgmap v1131: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:32:43.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:43 smithi142 ceph-mon[103614]: pgmap v1131: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:32:43.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:43.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:43.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:43.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:43.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:43 smithi152 ceph-mon[93456]: pgmap v1131: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:32:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:43.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:43.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:43.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:43.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:43.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:43.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:43.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:43.982 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 88940 88924 97.4743 62.875 0.0576965 0.0102502 2024-08-17T17:32:44.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:44.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:44.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:44.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:44.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:44.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:44.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:44.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:44.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:44.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:44.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:44.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:44.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:44.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:44.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:44.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:44.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:44.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:44.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:44.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:44.982 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 89448 89432 96.3413 31.75 0.00732218 0.0103746 2024-08-17T17:32:44.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:45.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:45.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:45.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:45.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:45.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:45.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:45.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:45.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:45.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:45 smithi152 ceph-mon[93456]: pgmap v1132: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:32:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:45 smithi142 ceph-mon[106729]: pgmap v1132: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:32:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:45 smithi142 ceph-mon[103614]: pgmap v1132: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:32:45.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:45.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:45.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:45.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:45.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:45.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:45.983 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 90349 90333 95.6607 56.3125 0.00785485 0.0104479 2024-08-17T17:32:46.052 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:46.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:46.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:46.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:46.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:46.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:46.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:46.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:46.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:46.488 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:32:46.489 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:46.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:46.528 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:32:46.528 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:46.566 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:32:46.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:46.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:46.610 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:32:46.611 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:46.649 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:32:46.649 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:46.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:46.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:46.683 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:32:46.683 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:46.714 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:32:46.715 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:46.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:32:46.744 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:46.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:46.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:46.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:46.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:46.984 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:32:46.983983+0000 min lat: 0.00117681 max lat: 0.338534 avg lat: 0.0105228 2024-08-17T17:32:46.984 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:32:46.984 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 91222 91206 94.9758 54.5625 0.0137145 0.0105228 2024-08-17T17:32:47.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:47.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:47.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:47.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:47.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:47.340 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:47.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:47.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:47.546 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:47 smithi142 ceph-mon[103614]: pgmap v1133: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:32:47.546 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1212484439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:47.546 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:47 smithi142 ceph-mon[106729]: pgmap v1133: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:32:47.546 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1212484439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:47.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:47 smithi152 ceph-mon[93456]: pgmap v1133: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:32:47.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1212484439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:47.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:47.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:47.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:47.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:47.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:47.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:47.984 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 92232 92216 94.4537 63.125 0.0119147 0.0105816 2024-08-17T17:32:47.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:48.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:48.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:48.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:48.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:48.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:48.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:48.312 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:48.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:48.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:48.440 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:48.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:48.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:48.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:48.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:48.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:48.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:48.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:48.986 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 93189 93173 93.8915 59.8125 0.0100176 0.010645 2024-08-17T17:32:48.998 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:49.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:49.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:49.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:49.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:49.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:49.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:49.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:49.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:49.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:49 smithi152 ceph-mon[93456]: pgmap v1134: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:32:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:49 smithi142 ceph-mon[106729]: pgmap v1134: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:32:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:49 smithi142 ceph-mon[103614]: pgmap v1134: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:32:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:49.642 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:49.643 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:32:49.643 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:32:49.643 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:32:49.643 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:32:49.643 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:32:49.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:49.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:49.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:49.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:49.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:49.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:49.986 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 94157 94141 93.3616 60.5 0.0169368 0.0107052 2024-08-17T17:32:50.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:50.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:50.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:50.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:50.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:50.309 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:32:50.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:50.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:50.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:50.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:50.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1780623524' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:50.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:50.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:50.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:50.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:50.539 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1780623524' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:50.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:50.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:50.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:50.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:32:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:32:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1780623524' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:50.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:50.694 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:50.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:50.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:50.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:50.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:50.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:50.986 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 95098 95082 92.8218 58.8125 0.0103119 0.0107682 2024-08-17T17:32:51.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:51.075 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:51.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:51.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:51.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:51.340 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:51.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:51.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:51.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:51 smithi152 ceph-mon[93456]: pgmap v1135: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T17:32:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:51 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:51 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:32:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:51 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:51 smithi152 ceph-mon[93456]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:32:51.610 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[106729]: pgmap v1135: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T17:32:51.610 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[106729]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[103614]: pgmap v1135: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:51.611 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-mon[103614]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:32:51.616 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:51.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:51.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:51.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:51.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:32:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:32:51] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:32:51.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:51.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:51.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:32:51.986 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 96072 96056 92.3303 60.875 0.0117325 0.0108245 2024-08-17T17:32:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:52.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:52.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:52.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:52.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:52.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:52.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:52.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:52.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:52.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:52.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:52.648 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:32:52.648 INFO:teuthology.orchestra.run.smithi142.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:32:52.648 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:32:52.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:52.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:52.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:52.814 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:52.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:52.948 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:52.955 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:52.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:52.990 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 96990 96974 91.7979 57.375 0.00469012 0.010887 2024-08-17T17:32:53.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:53.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:53.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:53.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:53.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:53.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:53.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:53.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:53.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:53.550 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:53 smithi142 ceph-mon[106729]: pgmap v1137: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2024-08-17T17:32:53.551 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:53 smithi142 ceph-mon[103614]: pgmap v1137: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2024-08-17T17:32:53.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:53 smithi152 ceph-mon[93456]: pgmap v1137: 89 pgs: 89 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2024-08-17T17:32:53.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.636 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:53.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:53.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:32:53.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:53.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:53.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:53.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:53.989 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 97899 97883 91.2753 56.8125 0.0122068 0.0109496 2024-08-17T17:32:54.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:54.040 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:54.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:54.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:54.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:54.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:54.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:54.414 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:54.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:54.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:54.595 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:54.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:54.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:54.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:54.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:54.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:54.990 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 98844 98828 90.8007 59.0625 0.0132666 0.0110082 2024-08-17T17:32:55.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:55.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:55.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:55.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:55.316 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:55.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:55.431 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:55 smithi142 ceph-mon[106729]: pgmap v1138: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T17:32:55.431 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:55 smithi142 ceph-mon[103614]: pgmap v1138: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T17:32:55.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:55.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:55.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:55 smithi152 ceph-mon[93456]: pgmap v1138: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T17:32:55.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:55.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:55.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:55.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:55.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:55.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:55.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:55.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:55.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:55.990 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 99762 99746 90.3163 57.375 0.0396515 0.0110665 2024-08-17T17:32:55.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:56.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:56.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:56.162 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:56.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:56.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:56.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:56.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:56.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:56.548 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:56.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:56.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:56.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:56.797 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:32:56.797 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242765 10.f e273/222 10:f5d57385:::benchmark_data_smithi142_192453_object117017:head v 273'15354, mlcod=273'15348)", 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 374.70223958899999, 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28726092600000003, 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.798 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069107+0000", 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8599999999999997e-07 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069115+0000", 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7100000000000007e-06 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.800 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069126+0000", 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1321000000000001e-05 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069133+0000", 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4790000000000001e-06 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.070023+0000", 2024-08-17T17:32:56.801 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00089035999999999998 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.070042+0000", 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8760999999999999e-05 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.356328+0000", 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28628575499999998 2024-08-17T17:32:56.802 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.356367+0000", 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9753999999999999e-05 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242762 10.7 e273/257 10:e5d91e03:::benchmark_data_smithi142_192453_object117014:head v 273'13155, mlcod=273'13153)", 2024-08-17T17:32:56.803 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 374.70206758, 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28567881899999997, 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.804 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069279+0000", 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5199999999999997e-07 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069286+0000", 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8059999999999997e-06 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.805 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069295+0000", 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9800000000000004e-06 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069300+0000", 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4949999999999999e-06 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.806 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.071296+0000", 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0019955060000000002 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.071313+0000", 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7452000000000002e-05 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.354873+0000", 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28356008999999999 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.807 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.354957+0000", 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3937999999999995e-05 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242764 10.13 e273/257 10:cbd587ba:::benchmark_data_smithi142_192453_object117016:head v 273'12990, mlcod=273'12988)", 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.075981+0000", 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 374.69536517400002, 2024-08-17T17:32:56.808 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27613257200000002, 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075981+0000", 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075981+0000", 2024-08-17T17:32:56.809 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075982+0000", 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1480000000000001e-06 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.075990+0000", 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0209999999999999e-06 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.810 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076003+0000", 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3266e-05 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076011+0000", 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076089+0000", 2024-08-17T17:32:56.811 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7329000000000005e-05 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.076108+0000", 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9185e-05 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27596227299999998 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.812 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352113+0000", 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3398e-05 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242769 10.7 e273/257 10:e215c346:::benchmark_data_smithi142_192453_object117021:head v 273'13156, mlcod=273'13153)", 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 374.693365019, 2024-08-17T17:32:56.813 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.281981127, 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:32:56.814 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077982+0000", 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1100000000000005e-07 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077991+0000", 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1940000000000004e-06 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.815 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078003+0000", 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2264e-05 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078010+0000", 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0110000000000003e-06 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078277+0000", 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000267135 2024-08-17T17:32:56.816 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078299+0000", 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1202000000000001e-05 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359932+0000", 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28163336 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.817 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359962+0000", 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0150000000000001e-05 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242772 10.14 e273/257 10:2c787a87:::benchmark_data_smithi142_192453_object117024:head v 273'13212, mlcod=273'13210)", 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 374.69019519900002, 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.278431659, 2024-08-17T17:32:56.818 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.819 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081152+0000", 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0559999999999999e-06 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081159+0000", 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3180000000000001e-06 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.820 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352055+0000", 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27089613600000001 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4620999999999999e-05 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352219+0000", 2024-08-17T17:32:56.821 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149436 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352240+0000", 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0767e-05 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359528+0000", 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072875090000000002 2024-08-17T17:32:56.822 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359582+0000", 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4815999999999998e-05 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242786 10.5 e273/272 10:a01a2aa8:::benchmark_data_smithi142_192453_object117038:head v 273'13406, mlcod=238'8016)", 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.094527+0000", 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 374.67681920299998, 2024-08-17T17:32:56.823 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27114051900000002, 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094527+0000", 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.824 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094527+0000", 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094528+0000", 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5820000000000001e-06 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.094539+0000", 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0192e-05 2024-08-17T17:32:56.825 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352055+0000", 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25751674200000002 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4917e-05 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.826 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352261+0000", 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000190373 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352283+0000", 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2113e-05 2024-08-17T17:32:56.827 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.365645+0000", 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013362384999999999 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.828 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.365667+0000", 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2215e-05 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:29103 11.9 11:950c9362:::benchmark_data_smithi142_227468_object29102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2024-08-17T17:32:56.829 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.091474+0000", 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 331.67987160299998, 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.269225565, 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.830 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 29103 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.831 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091474+0000", 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091474+0000", 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.832 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091476+0000", 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1850000000000001e-06 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091497+0000", 2024-08-17T17:32:56.833 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1075000000000001e-05 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091508+0000", 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0883999999999999e-05 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.834 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.091514+0000", 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6660000000000002e-06 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092564+0000", 2024-08-17T17:32:56.835 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001050095 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092680+0000", 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116146 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.836 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092826+0000", 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014555199999999999 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.100629+0000", 2024-08-17T17:32:56.837 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078026399999999996 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.100675+0000", 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6579999999999998e-05 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.838 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.359858+0000", 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25918253899999999 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.359938+0000", 2024-08-17T17:32:56.839 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0249999999999999e-05 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.360700+0000", 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000761953 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.840 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104285 11.9 11:956711b1:::benchmark_data_smithi142_227468_object104284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 273.517026784, 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27987187299999999, 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.841 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104285 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:32:56.842 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.845 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.845 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.845 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.845 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254320+0000", 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.226e-06 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254340+0000", 2024-08-17T17:32:56.846 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9253999999999999e-05 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254348+0000", 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4440000000000008e-06 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254353+0000", 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3040000000000004e-06 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.847 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.257745+0000", 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003391336 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.257867+0000", 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012196099999999999 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.258003+0000", 2024-08-17T17:32:56.848 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013589099999999999 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.262664+0000", 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046612679999999997 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.267561+0000", 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048971850000000001 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.849 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534082+0000", 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26652041300000001 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534142+0000", 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0158999999999998e-05 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534191+0000", 2024-08-17T17:32:56.850 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9431999999999997e-05 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104299 11.b 11:dd1aa66c:::benchmark_data_smithi142_227468_object104298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 273.50822000800002, 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27979115300000001, 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.851 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104299 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.852 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263127+0000", 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7000000000000004e-07 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.853 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263147+0000", 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0576000000000001e-05 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263158+0000", 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0173999999999999e-05 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263164+0000", 2024-08-17T17:32:56.854 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7499999999999997e-06 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266586+0000", 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034218400000000002 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266722+0000", 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013612100000000001 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:32:56.855 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266854+0000", 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132001 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.274109+0000", 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072550760000000001 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.275555+0000", 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014456219999999999 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.856 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542810+0000", 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26725485300000001 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542864+0000", 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3872999999999997e-05 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542917+0000", 2024-08-17T17:32:56.857 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3497e-05 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104303 11.9 11:982685b7:::benchmark_data_smithi142_227468_object104302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 273.503121308, 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29623361999999998, 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.858 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104303 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.859 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4900000000000005e-07 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268246+0000", 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0990000000000001e-05 2024-08-17T17:32:56.860 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268255+0000", 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7369999999999995e-06 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268261+0000", 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6749999999999999e-06 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.861 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268310+0000", 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9447999999999998e-05 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268420+0000", 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010979 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268546+0000", 2024-08-17T17:32:56.862 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012620700000000001 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272696+0000", 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041496550000000004 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277035+0000", 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043387249999999999 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.863 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564360+0000", 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28732556599999998 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564416+0000", 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5747e-05 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564458+0000", 2024-08-17T17:32:56.864 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2330999999999998e-05 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104304 11.b 11:dcea405a:::benchmark_data_smithi142_227468_object104303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.269864+0000", 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 273.501481996, 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27317428999999999, 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.865 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104304 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269864+0000", 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.866 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269864+0000", 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269866+0000", 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.474e-06 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.867 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269892+0000", 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6038e-05 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269900+0000", 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.028e-06 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.269905+0000", 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3839999999999996e-06 2024-08-17T17:32:56.868 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.270162+0000", 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025740099999999998 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.270259+0000", 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7133000000000007e-05 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:32:56.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.270418+0000", 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015801300000000001 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277853+0000", 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0074356509999999997 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277864+0000", 2024-08-17T17:32:56.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0876999999999999e-05 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542956+0000", 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26509197200000001 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542996+0000", 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0268999999999997e-05 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.871 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.543038+0000", 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2049999999999999e-05 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104305 11.9 11:9307d714:::benchmark_data_smithi142_227468_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 273.49864644299998, 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29189848299999999, 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.872 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104305 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.873 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272701+0000", 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1990000000000001e-06 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.874 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272724+0000", 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3428999999999999e-05 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272736+0000", 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2177e-05 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272743+0000", 2024-08-17T17:32:56.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8449999999999997e-06 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273130+0000", 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00038687699999999999 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273244+0000", 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011396900000000001 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.876 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273376+0000", 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000131812 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.281412+0000", 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080358289999999995 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.282800+0000", 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00138784 2024-08-17T17:32:56.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564526+0000", 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.281726537 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564568+0000", 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1570999999999999e-05 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.878 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564598+0000", 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0397999999999999e-05 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104325 11.9 11:9715431e:::benchmark_data_smithi142_227468_object104324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 273.48031916799999, 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27710011899999998, 2024-08-17T17:32:56.879 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104325 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:32:56.880 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291028+0000", 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2929999999999999e-06 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.881 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291051+0000", 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2566999999999999e-05 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291064+0000", 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3695e-05 2024-08-17T17:32:56.882 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291071+0000", 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0219999999999999e-06 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.883 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291129+0000", 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7970000000000002e-05 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291263+0000", 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.884 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291405+0000", 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014159700000000001 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.295160+0000", 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037547869999999999 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.295314+0000", 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015397800000000001 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.567971+0000", 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27265716699999998 2024-08-17T17:32:56.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.568083+0000", 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011186799999999999 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.568127+0000", 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4175000000000003e-05 2024-08-17T17:32:56.887 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162100 11.0 11:02a7f4f2:::benchmark_data_smithi142_227468_object38681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 206.283351568, 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.30841822800000002, 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.888 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162100 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487996+0000", 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2100000000000001e-06 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.488000+0000", 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6040000000000003e-06 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499381+0000", 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011380988999999999 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.891 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499402+0000", 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0930000000000001e-05 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499544+0000", 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014133400000000001 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499675+0000", 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013168000000000001 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499788+0000", 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112625 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511105+0000", 2024-08-17T17:32:56.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011317159 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511158+0000", 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3236000000000003e-05 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796105+0000", 2024-08-17T17:32:56.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28494646400000001 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796225+0000", 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120295 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796413+0000", 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018770200000000001 2024-08-17T17:32:56.895 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162107 11.0 11:0142b711:::benchmark_data_smithi142_227468_object38688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 206.271889509, 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29756906700000002, 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.896 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162107 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:32:56.897 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499458+0000", 2024-08-17T17:32:56.898 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8780000000000001e-06 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499465+0000", 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8619999999999997e-06 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499477+0000", 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2011000000000001e-05 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499483+0000", 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4009999999999996e-06 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.504931+0000", 2024-08-17T17:32:56.900 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054487809999999998 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.505217+0000", 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00028554699999999998 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.505550+0000", 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00033330399999999998 2024-08-17T17:32:56.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511178+0000", 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0056279959999999997 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.512133+0000", 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00095433599999999996 2024-08-17T17:32:56.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796542+0000", 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28440920800000002 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796893+0000", 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035122400000000002 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797026+0000", 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132519 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162129 11.0 11:078c6a55:::benchmark_data_smithi142_227468_object38710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.524455+0000", 2024-08-17T17:32:56.904 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 206.24689079199999, 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27313411999999998, 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162129 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.905 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524455+0000", 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524455+0000", 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524456+0000", 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.13e-06 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524461+0000", 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5299999999999998e-06 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524472+0000", 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1440000000000001e-05 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524478+0000", 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2680000000000002e-06 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524533+0000", 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5126e-05 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524688+0000", 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015477000000000001 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.524817+0000", 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012918200000000001 2024-08-17T17:32:56.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.528632+0000", 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038150319999999999 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.528729+0000", 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7169000000000004e-05 2024-08-17T17:32:56.910 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797195+0000", 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26846631999999998 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797373+0000", 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017769800000000001 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797589+0000", 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000216455 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162133 11.b 11:dac48ceb:::benchmark_data_smithi142_227468_object38714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:32:56.912 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 206.24177598399999, 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29427250700000002, 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162133 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.913 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.914 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529571+0000", 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.308e-06 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529576+0000", 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0980000000000001e-06 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529588+0000", 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1579999999999999e-05 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529595+0000", 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.613e-06 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529736+0000", 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014141699999999999 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529840+0000", 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103724 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:32:56.917 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529960+0000", 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120653 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537643+0000", 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0076821010000000002 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.918 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537677+0000", 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4820999999999997e-05 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823705+0000", 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.286027592 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.919 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823799+0000", 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3799000000000001e-05 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823843+0000", 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3801000000000003e-05 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.920 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162136 11.b 11:d77466cb:::benchmark_data_smithi142_227468_object38717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 206.23985664099999, 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29716111499999998, 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.921 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162136 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.922 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531491+0000", 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2160000000000001e-06 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531562+0000", 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1204000000000005e-05 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531574+0000", 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1950000000000001e-05 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531581+0000", 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8909999999999998e-06 2024-08-17T17:32:56.924 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534124+0000", 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025436669999999999 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534233+0000", 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010820899999999999 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.925 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534413+0000", 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018054799999999999 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537671+0000", 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032577740000000002 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.926 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537731+0000", 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0050000000000003e-05 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828485+0000", 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29075420899999999 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.927 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828589+0000", 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103916 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828651+0000", 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1481000000000003e-05 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.928 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:236400 11.b 11:d7d22f27:::benchmark_data_smithi142_227468_object112981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.704385+0000", 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 113.06696124600001, 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26910604599999999, 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.929 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 236400 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704385+0000", 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.930 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704385+0000", 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704386+0000", 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5700000000000001e-07 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.931 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704390+0000", 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2590000000000002e-06 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704399+0000", 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2620000000000004e-06 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704406+0000", 2024-08-17T17:32:56.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8750000000000002e-06 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704487+0000", 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0822000000000006e-05 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.933 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704588+0000", 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010092199999999999 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.704697+0000", 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109709 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.708848+0000", 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041503900000000003 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.710982+0000", 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0021343769999999998 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973399+0000", 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26241696599999997 2024-08-17T17:32:56.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973451+0000", 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1765000000000003e-05 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973491+0000", 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9842000000000002e-05 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.936 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:236419 11.9 11:909d5552:::benchmark_data_smithi142_227468_object113000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.725389+0000", 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 113.045957245, 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26913189599999998, 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:56.937 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 236419 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.725389+0000", 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.725389+0000", 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.725390+0000", 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5090000000000001e-06 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.725397+0000", 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7580000000000003e-06 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.725411+0000", 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4083000000000001e-05 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.940 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.725421+0000", 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4700000000000008e-06 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.726222+0000", 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00080131699999999996 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:32:56.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.726342+0000", 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000119584 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.726458+0000", 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116537 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.730035+0000", 2024-08-17T17:32:56.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003576578 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.736452+0000", 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.006417613 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.994368+0000", 2024-08-17T17:32:56.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25791575700000002 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.994458+0000", 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0173000000000006e-05 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.994521+0000", 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2516999999999998e-05 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.944 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.945 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.945 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:32:56.945 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:32:56.945 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:32:56.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:56.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:56.990 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 100705 100689 89.8681 58.9375 0.0206623 0.0111222 2024-08-17T17:32:57.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:57.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:57.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:57.106 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:57.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:57.227 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:57.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:57.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:57.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:57.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:57.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:57 smithi152 ceph-mon[93456]: pgmap v1139: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-08-17T17:32:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:57.591 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:57 smithi142 ceph-mon[106729]: pgmap v1139: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-08-17T17:32:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:57.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:57 smithi142 ceph-mon[103614]: pgmap v1139: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-08-17T17:32:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:32:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:57.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:57.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:32:57.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:57.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:57.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:57.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:57.990 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 101677 101661 89.458 60.75 0.00961314 0.0111721 2024-08-17T17:32:58.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:58.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:58.088 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:58.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:58.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:58.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:58.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:58.309 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:58.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:58.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:58.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:58.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:58.551 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:32:58.551 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:32:58.552 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-08-17T17:32:58.552 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:32:58.552 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:32:58.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-08-17T17:32:58.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:58.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:58.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:58.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:58.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:58.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:58.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:58.992 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 102596 102580 89.0119 57.4375 0.0213017 0.0112284 2024-08-17T17:32:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:32:59.028 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:32:59.029 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:59.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:59.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:59.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:59.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:59.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:59.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:59.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:59.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:59.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:59.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:59 smithi152 ceph-mon[93456]: pgmap v1140: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-17T17:32:59.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:32:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/835833431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:59 smithi142 ceph-mon[106729]: pgmap v1140: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-17T17:32:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:32:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/835833431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:59 smithi142 ceph-mon[103614]: pgmap v1140: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-17T17:32:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:32:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/835833431' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:59.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:32:59.687 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:32:59.701 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:59.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:59.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:32:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:59.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:59.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:32:59.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:59.991 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 103595 103579 88.6479 62.4375 0.00610548 0.0112754 2024-08-17T17:33:00.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:00.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:00.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:00.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:00.223 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:33:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:33:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:00.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:00.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:00.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:00.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:00.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:00.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:00.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:00.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:00.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:00.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:00.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:00.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:00.991 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 104512 104496 88.2245 57.3125 0.00910409 0.0113285 2024-08-17T17:33:01.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:01.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:01.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:01.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:01.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:01.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:01.339 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:33:01.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:01.401 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:01 smithi152 ceph-mon[93456]: pgmap v1141: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-08-17T17:33:01.401 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/154093935' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:01.401 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:01 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:01.436 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:01.436 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:01.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:01.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:01.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:01.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-mon[106729]: pgmap v1141: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-08-17T17:33:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/154093935' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:01.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-mon[103614]: pgmap v1141: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-08-17T17:33:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/154093935' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:01.653 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:01.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:01.792 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:01.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:01.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:01.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:01.991 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 105430 105414 87.8132 57.375 0.0167083 0.0113829 2024-08-17T17:33:02.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:02.065 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:33:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:33:01] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:33:02.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:02.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:02.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:02.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:02.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:33:02.433 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:02.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:02.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:02.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:02 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:02.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:02.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:02 smithi152 ceph-mon[93456]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:33:02.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:02 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:02.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:02.576 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:02 smithi142 ceph-mon[103614]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:33:02.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:02 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:02.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:02.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:02 smithi142 ceph-mon[106729]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:33:02.595 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:02.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:02.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:02.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:02.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:02.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:02.992 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 106307 106291 87.3791 54.8125 0.0321373 0.0114389 2024-08-17T17:33:03.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:03.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:03.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:03.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:03.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:03.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:03.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:03.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:03.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:03.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:03.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:03.572 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:03.573 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:33:03.573 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:03.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:03 smithi152 ceph-mon[93456]: pgmap v1143: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-08-17T17:33:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:03.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:03 smithi142 ceph-mon[106729]: pgmap v1143: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-08-17T17:33:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:03 smithi142 ceph-mon[103614]: pgmap v1143: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-08-17T17:33:03.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:03.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:03.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:03.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:03.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:03.992 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 107256 107240 87.0146 59.3125 0.0158666 0.0114868 2024-08-17T17:33:04.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:04.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:04.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:04.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:04.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:04.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:04.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:33:04.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:04.417 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:04.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:04.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:04.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:04.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:04.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:04.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:04.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:04.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:04.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:04.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:04.993 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 108075 108059 86.5536 51.1875 0.0130204 0.0115478 2024-08-17T17:33:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:05.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:33:05.083 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:05.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:05.085 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:05.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:05.140 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:05.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:05.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:05.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:05.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:05.387 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:05.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:05.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:05.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:05.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:05.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:05 smithi152 ceph-mon[93456]: pgmap v1144: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2024-08-17T17:33:05.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/664291386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:05 smithi142 ceph-mon[106729]: pgmap v1144: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2024-08-17T17:33:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/664291386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:05 smithi142 ceph-mon[103614]: pgmap v1144: 89 pgs: 89 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2024-08-17T17:33:05.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/664291386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:05.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:05.688 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:05.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:05.789 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:05.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:05.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:05.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:05.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:05.994 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 109005 108989 86.193 58.125 0.0329996 0.0115965 2024-08-17T17:33:06.023 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:06.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:06.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:06.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:06.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:06.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:06.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:06.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:06.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:06.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:06.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:06.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:06.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:06.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:06.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:06.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:06.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:06.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:06.994 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:33:06.994631+0000 min lat: 0.00117681 max lat: 0.338534 avg lat: 0.0116466 2024-08-17T17:33:06.994 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:33:06.995 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 109911 109895 85.8234 56.625 0.0137057 0.0116466 2024-08-17T17:33:07.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:07.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:07.090 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:07.090 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:33:07.091 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:33:07.091 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:33:07.091 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:07.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:07.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:07.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:07.355 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:07.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:07.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:07.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:07.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:07 smithi142 ceph-mon[106729]: pgmap v1145: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-08-17T17:33:07.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:07 smithi142 ceph-mon[103614]: pgmap v1145: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-08-17T17:33:07.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:07.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:07.775 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:07 smithi152 ceph-mon[93456]: pgmap v1145: 89 pgs: 89 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2024-08-17T17:33:07.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:07.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:07.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:33:07.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:07.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:07.914 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:07.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:07.994 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 110793 110777 85.4445 55.125 0.0104506 0.0116972 2024-08-17T17:33:08.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.073 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:08.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:08.154 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:08.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:08.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:08.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:08.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:08.388 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:08.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:08.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:08.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:08.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:08.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:08.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:08.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:08.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:08.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:08.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:08.995 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 111586 111570 85.0069 49.5625 0.0214567 0.0117583 2024-08-17T17:33:09.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:09.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:09.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:09.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:09.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:09.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:09.371 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:33:09.410 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:09.450 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:09.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:09.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:09.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:09 smithi152 ceph-mon[93456]: pgmap v1146: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-08-17T17:33:09.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2977840451' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:09.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:09.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:09 smithi142 ceph-mon[106729]: pgmap v1146: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-08-17T17:33:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2977840451' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:09.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:09 smithi142 ceph-mon[103614]: pgmap v1146: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-08-17T17:33:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2977840451' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:09.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:09.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:09.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:09.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:09.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:09.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:09.994 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 112440 112424 84.6259 53.375 0.00490173 0.0118111 2024-08-17T17:33:10.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:10.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:10.030 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:33:10.030 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:33:10.030 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:33:10.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 2 2024-08-17T17:33:10.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:10.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:10.197 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:10.197 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:33:10.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116286 10.4 10:23b25cf9:::benchmark_data_smithi142_192453_object116285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:46.664127+0000", 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 563.53078200200002, 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98948754699999997, 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116286 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664127+0000", 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664127+0000", 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664125+0000", 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664186+0000", 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0749999999999999e-05 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664200+0000", 2024-08-17T17:33:10.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3976e-05 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664206+0000", 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7100000000000004e-06 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664250+0000", 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4048000000000002e-05 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.664256+0000", 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0240000000000003e-06 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.634933+0000", 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97067708200000002 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.637306+0000", 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023726519999999998 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.637412+0000", 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106477 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.646572+0000", 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091603299999999995 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.646581+0000", 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6759999999999996e-06 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653536+0000", 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069550829999999999 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653573+0000", 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6442000000000001e-05 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653614+0000", 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1631000000000002e-05 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:116345 10.4 10:20c60145:::benchmark_data_smithi142_192453_object116344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:23:46.732878+0000", 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 563.46203106400003, 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92081792600000001, 2024-08-17T17:33:10.205 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 116345 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732878+0000", 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732878+0000", 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732876+0000", 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732906+0000", 2024-08-17T17:33:10.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9699000000000001e-05 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732921+0000", 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5262000000000001e-05 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732927+0000", 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6860000000000003e-06 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.732999+0000", 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2241000000000002e-05 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:46.733008+0000", 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8179999999999993e-06 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.640666+0000", 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90765785600000004 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.640812+0000", 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000145883 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.640981+0000", 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016897799999999999 2024-08-17T17:33:10.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.646599+0000", 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056188089999999998 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653485+0000", 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068855139999999997 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653651+0000", 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0001663 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653677+0000", 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5490000000000002e-05 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:23:47.653695+0000", 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8845999999999999e-05 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:144825 10.1 10:8856b1b2:::benchmark_data_smithi142_192453_object19077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:35.865517+0000", 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 514.32939195200004, 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.157562472, 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144825 2024-08-17T17:33:10.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865517+0000", 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865517+0000", 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865515+0000", 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865523+0000", 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9109999999999993e-06 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865537+0000", 2024-08-17T17:33:10.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3396e-05 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865543+0000", 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7519999999999999e-06 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.865919+0000", 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037529399999999997 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.866059+0000", 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014045600000000001 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.866199+0000", 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013964500000000001 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.877260+0000", 2024-08-17T17:33:10.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011061720000000001 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.883033+0000", 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057721539999999998 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.883042+0000", 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3689999999999996e-06 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.001960+0000", 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1189178639999999 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.002076+0000", 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116146 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.022819+0000", 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020743167999999999 2024-08-17T17:33:10.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.023079+0000", 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025994599999999999 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:144827 10.6 10:61040ce9:::benchmark_data_smithi142_192453_object19079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:24:35.893612+0000", 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 514.30129658199996, 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2032246719999999, 2024-08-17T17:33:10.221 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144827 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893612+0000", 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893612+0000", 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893611+0000", 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893617+0000", 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.635e-06 2024-08-17T17:33:10.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893629+0000", 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1505000000000001e-05 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893636+0000", 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1790000000000002e-06 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893679+0000", 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2938999999999997e-05 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:35.893684+0000", 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4920000000000001e-06 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.089536+0000", 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1958518410000001 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.089632+0000", 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5643000000000006e-05 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.089722+0000", 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9754999999999997e-05 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.092998+0000", 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032764909999999999 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.093681+0000", 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00068252099999999995 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.096625+0000", 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029443440000000002 2024-08-17T17:33:10.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.096662+0000", 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7126000000000003e-05 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:24:37.096837+0000", 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017454999999999999 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.228 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179013 10.14 10:2fc210a1:::benchmark_data_smithi142_192453_object53265:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.761475+0000", 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 468.43343383899997, 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.061772216, 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.229 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179013 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761475+0000", 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761475+0000", 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761473+0000", 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761482+0000", 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6840000000000008e-06 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761497+0000", 2024-08-17T17:33:10.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4990999999999999e-05 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761505+0000", 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2289999999999997e-06 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090806+0000", 2024-08-17T17:33:10.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32930165099999997 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090816+0000", 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6940000000000007e-06 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823175+0000", 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.73235919699999996 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823247+0000", 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1911999999999999e-05 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179014 10.11 10:8e447741:::benchmark_data_smithi142_192453_object53266:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 468.43339732200002, 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0591031639999999, 2024-08-17T17:33:10.234 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179014 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761511+0000", 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761514+0000", 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7099999999999999e-06 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761520+0000", 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.93e-06 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761523+0000", 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5159999999999999e-06 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.097043+0000", 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33552037699999998 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.097050+0000", 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.820532+0000", 2024-08-17T17:33:10.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72348204800000004 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.820614+0000", 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2399999999999997e-05 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179019 10.4 10:21f666e9:::benchmark_data_smithi142_192453_object53271:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:33:10.239 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 468.43338156700003, 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0361314450000001, 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179019 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:33:10.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761527+0000", 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761529+0000", 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0609999999999998e-06 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761534+0000", 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2379999999999997e-06 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.761537+0000", 2024-08-17T17:33:10.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3529999999999999e-06 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.079599+0000", 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31806203500000002 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.079607+0000", 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.661e-06 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.797577+0000", 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.71796951399999998 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.797658+0000", 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1886999999999997e-05 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:179033 10.14 10:2d675116:::benchmark_data_smithi142_192453_object53285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:21.845669+0000", 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 468.34923940499999, 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98075968000000002, 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.245 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 179033 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845669+0000", 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845669+0000", 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845667+0000", 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845678+0000", 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.079e-05 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845701+0000", 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3144000000000001e-05 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:21.845709+0000", 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8830000000000006e-06 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090830+0000", 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24512108099999999 2024-08-17T17:33:10.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.090834+0000", 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8659999999999999e-06 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823498+0000", 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.73266395200000001 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823608+0000", 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110109 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.823949+0000", 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034181199999999999 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.825713+0000", 2024-08-17T17:33:10.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017633939999999999 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826128+0000", 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041489700000000001 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826350+0000", 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000222315 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826383+0000", 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3222000000000001e-05 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:22.826429+0000", 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5630000000000002e-05 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183186 10.1 10:8272017f:::benchmark_data_smithi142_192453_object57438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-17T17:33:10.252 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.770927+0000", 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 462.42398123700002, 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.041528228, 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183186 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770927+0000", 2024-08-17T17:33:10.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770927+0000", 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770926+0000", 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770931+0000", 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3939999999999998e-06 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770943+0000", 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1646e-05 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770948+0000", 2024-08-17T17:33:10.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7930000000000003e-06 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.770992+0000", 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3686999999999997e-05 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.771095+0000", 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102514 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.771464+0000", 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036939699999999999 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.773319+0000", 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001855338 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.774672+0000", 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013531299999999999 2024-08-17T17:33:10.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.797741+0000", 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023068381999999998 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:10.260 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.797748+0000", 2024-08-17T17:33:10.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6329999999999996e-06 2024-08-17T17:33:10.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.260 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.260 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812380+0000", 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014632024 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812456+0000", 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5165000000000003e-05 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183207 10.1 10:8661facc:::benchmark_data_smithi142_192453_object57459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:33:10.261 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.812883+0000", 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 462.38202566199999, 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014551669, 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183207 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812883+0000", 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812883+0000", 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812882+0000", 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812886+0000", 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4920000000000004e-06 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812896+0000", 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0750000000000004e-06 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812899+0000", 2024-08-17T17:33:10.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6500000000000002e-06 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812950+0000", 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1099e-05 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.812958+0000", 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4229999999999999e-06 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812477+0000", 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99951941 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812567+0000", 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0138999999999999e-05 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812990+0000", 2024-08-17T17:33:10.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00042253899999999999 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.814513+0000", 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001523555 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827371+0000", 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012857337 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827392+0000", 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.088e-05 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827435+0000", 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3016999999999998e-05 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183215 10.11 10:8c02d523:::benchmark_data_smithi142_192453_object57467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.818751+0000", 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 462.37615794499999, 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0085936609999999, 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183215 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.269 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818751+0000", 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818751+0000", 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818750+0000", 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818754+0000", 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.87e-06 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818765+0000", 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0712e-05 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818769+0000", 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1899999999999997e-06 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818812+0000", 2024-08-17T17:33:10.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2886000000000002e-05 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.818821+0000", 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.551e-06 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.810908+0000", 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99208728599999996 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.811041+0000", 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013248399999999999 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.811251+0000", 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021044299999999999 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.813943+0000", 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026920339999999998 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827250+0000", 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013307401 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827290+0000", 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0039000000000001e-05 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827344+0000", 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3897999999999998e-05 2024-08-17T17:33:10.275 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:183216 10.11 10:8e43b7c1:::benchmark_data_smithi142_192453_object57468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 462.37521222499998, 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0077557509999999, 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:10.276 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 183216 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819696+0000", 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819700+0000", 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6070000000000001e-06 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819711+0000", 2024-08-17T17:33:10.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0214000000000001e-05 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819716+0000", 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5670000000000002e-06 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819752+0000", 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5376999999999997e-05 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:27.819756+0000", 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8300000000000003e-06 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812388+0000", 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99263158200000001 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812498+0000", 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110532 2024-08-17T17:33:10.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.812694+0000", 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019582900000000001 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.814418+0000", 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017239149999999999 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827382+0000", 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012964128 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827408+0000", 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5443999999999999e-05 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:28.827452+0000", 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4357000000000002e-05 2024-08-17T17:33:10.282 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:162135 11.e 11:7f3f6328:::benchmark_data_smithi142_227468_object38716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:29:29.530809+0000", 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 220.66409976700001, 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.27207632100000001, 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.283 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 162135 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530809+0000", 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530809+0000", 2024-08-17T17:33:10.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530808+0000", 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530814+0000", 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8359999999999997e-06 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530825+0000", 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1752e-05 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530830+0000", 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4830000000000001e-06 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530878+0000", 2024-08-17T17:33:10.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8359999999999998e-05 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.530995+0000", 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116494 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.531070+0000", 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5053000000000001e-05 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.534329+0000", 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032594270000000001 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.534449+0000", 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120351 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.802821+0000", 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26837162599999997 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.802858+0000", 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7042000000000002e-05 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.802885+0000", 2024-08-17T17:33:10.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7053e-05 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:30775 12.2 12:420a4907:::benchmark_data_smithi142_261410_object30774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:31:59.189225+0000", 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 71.005683657999995, 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.093543162999999999, 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.290 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 30775 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189225+0000", 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189225+0000", 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189224+0000", 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189272+0000", 2024-08-17T17:33:10.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7908000000000003e-05 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189282+0000", 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.995e-06 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189287+0000", 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0359999999999997e-06 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189316+0000", 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9594000000000002e-05 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189390+0000", 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3689999999999994e-05 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:33:10.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189470+0000", 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0122000000000003e-05 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193420+0000", 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039504409999999998 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193477+0000", 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6912e-05 2024-08-17T17:33:10.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282669+0000", 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089191539 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282722+0000", 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3029999999999999e-05 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282768+0000", 2024-08-17T17:33:10.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6236000000000003e-05 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:30779 12.2 12:46346b84:::benchmark_data_smithi142_261410_object30778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:31:59.189295+0000", 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 71.005613253999996, 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.093612770999999997, 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.297 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 30779 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189295+0000", 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189295+0000", 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189295+0000", 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189333+0000", 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8109999999999999e-05 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189341+0000", 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0369999999999995e-06 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189345+0000", 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3209999999999998e-06 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.190755+0000", 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014096390000000001 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.190851+0000", 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6247000000000002e-05 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.190984+0000", 2024-08-17T17:33:10.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013244600000000001 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193465+0000", 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024816970000000002 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193482+0000", 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6994e-05 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282819+0000", 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089337072000000003 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282872+0000", 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2649999999999999e-05 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282908+0000", 2024-08-17T17:33:10.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6121999999999997e-05 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:30783 12.2 12:48ff760a:::benchmark_data_smithi142_261410_object30782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:31:59.189471+0000", 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 71.005437788999998, 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.093565224000000002, 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.304 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 30783 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189471+0000", 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189471+0000", 2024-08-17T17:33:10.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189470+0000", 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189513+0000", 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2845000000000001e-05 2024-08-17T17:33:10.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189521+0000", 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2640000000000008e-06 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189526+0000", 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0189999999999997e-06 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.192071+0000", 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002544981 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.192278+0000", 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020659399999999999 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.192407+0000", 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012906899999999999 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.194305+0000", 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001898257 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.194327+0000", 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1824e-05 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.282948+0000", 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.088621170999999999 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.283005+0000", 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6406999999999997e-05 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.283036+0000", 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1260999999999998e-05 2024-08-17T17:33:10.310 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:30784 12.2 12:47dbe869:::benchmark_data_smithi142_261410_object30783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:31:59.189532+0000", 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 71.005376147000007, 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.093618198999999999, 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.311 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 30784 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189532+0000", 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189532+0000", 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189532+0000", 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189568+0000", 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6483999999999998e-05 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189576+0000", 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7659999999999998e-06 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.189581+0000", 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.147e-06 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193504+0000", 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039223110000000004 2024-08-17T17:33:10.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193602+0000", 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8327999999999998e-05 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.193705+0000", 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103278 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.194448+0000", 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074289099999999997 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.194954+0000", 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00050583900000000001 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.283080+0000", 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.088125695000000004 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.283120+0000", 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0057e-05 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:31:59.283151+0000", 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0960000000000002e-05 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:34268 12.0 12:0abb4562:::benchmark_data_smithi142_261410_object34267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:01.106554+0000", 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 69.088354823000003, 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089082906000000003, 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 34268 2024-08-17T17:33:10.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106554+0000", 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106554+0000", 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106553+0000", 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106661+0000", 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107979 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106675+0000", 2024-08-17T17:33:10.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4960000000000001e-05 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106681+0000", 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5949999999999998e-06 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106724+0000", 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2611000000000002e-05 2024-08-17T17:33:10.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106846+0000", 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012235800000000001 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.106964+0000", 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011840099999999999 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.108669+0000", 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017044829999999999 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.108866+0000", 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019683100000000001 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.195540+0000", 2024-08-17T17:33:10.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.086674599000000005 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.195600+0000", 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9883999999999998e-05 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:01.195637+0000", 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6464999999999997e-05 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:43355 12.0 12:08770711:::benchmark_data_smithi142_261410_object43354:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:06.553832+0000", 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 63.641076718999997, 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.094167078000000001, 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 43355 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.325 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553832+0000", 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553832+0000", 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553830+0000", 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553918+0000", 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7740999999999999e-05 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553933+0000", 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5197999999999999e-05 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553940+0000", 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2920000000000001e-06 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.553984+0000", 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4554000000000001e-05 2024-08-17T17:33:10.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554109+0000", 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012459799999999999 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554245+0000", 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000136413 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.557422+0000", 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031767840000000002 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.557948+0000", 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052613300000000003 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.647911+0000", 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089962423 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.647960+0000", 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9509999999999999e-05 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.647999+0000", 2024-08-17T17:33:10.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8914e-05 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:43357 12.0 12:07bd7be0:::benchmark_data_smithi142_261410_object43356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:06.554644+0000", 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 63.640264958000003, 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.094350903, 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:10.332 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 43357 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554644+0000", 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554644+0000", 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:10.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554643+0000", 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554686+0000", 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2567000000000001e-05 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:10.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554697+0000", 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1567e-05 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.554702+0000", 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2329999999999997e-06 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.555883+0000", 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011802539999999999 2024-08-17T17:33:10.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.555998+0000", 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115889 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.556128+0000", 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012944700000000001 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.557544+0000", 2024-08-17T17:33:10.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014162249999999999 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.558069+0000", 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052499700000000003 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.648912+0000", 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.090842512 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.648960+0000", 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8171999999999999e-05 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:06.648995+0000", 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4740999999999998e-05 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:10.338 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:10.339 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:10.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:10.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:10.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:10.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:33:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:33:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:10 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:10 smithi152 ceph-mon[93456]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:33:10.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:10.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:10 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:10.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:10.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:10 smithi142 ceph-mon[103614]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:33:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:10 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:10.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:10.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:10 smithi142 ceph-mon[106729]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:33:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:10.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:10.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:10.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:10.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:10.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:10.995 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 113047 113031 84.0702 37.9375 0.0124262 0.0118897 2024-08-17T17:33:11.000 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:11.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:11.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:11.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:11.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:33:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:11.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:11.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:11.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:11.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:11.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:11.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:11.540 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:11.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:11.583 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:11.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:11.668 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:11 smithi152 ceph-mon[93456]: pgmap v1148: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2024-08-17T17:33:11.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:11.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:11 smithi142 ceph-mon[103614]: pgmap v1148: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2024-08-17T17:33:11.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:11 smithi142 ceph-mon[106729]: pgmap v1148: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2024-08-17T17:33:11.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:11.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:11.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:11.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:11.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:11.994 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 113944 113928 83.7407 56.0625 0.0139935 0.0119359 2024-08-17T17:33:12.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:33:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:33:11] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:33:12.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:12.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:12.260 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:12.362 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:12.405 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:12.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:12.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:12.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:12.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:12.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:12.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:12.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:12.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:12.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:12 smithi152 ceph-mon[93456]: pgmap v1149: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2024-08-17T17:33:12.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:12.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:12 smithi142 ceph-mon[103614]: pgmap v1149: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2024-08-17T17:33:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:12.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:12.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:12 smithi142 ceph-mon[106729]: pgmap v1149: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2024-08-17T17:33:12.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:12.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:12.995 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 114833 114817 83.4131 55.5625 0.0186489 0.0119835 2024-08-17T17:33:13.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:13.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:13.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:13.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:13.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:13.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:13.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:13.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:13.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:13.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:13.499 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:115276 12.f 12:fb49a0b2:::benchmark_data_smithi142_261410_object115275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.481444+0000", 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.016145874000000001, 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016169820000000001, 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 115276 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481444+0000", 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481444+0000", 2024-08-17T17:33:13.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481443+0000", 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481478+0000", 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5207999999999999e-05 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481494+0000", 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6209999999999999e-05 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481500+0000", 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3729999999999998e-06 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481545+0000", 2024-08-17T17:33:13.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4589000000000003e-05 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481681+0000", 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013629000000000001 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.481832+0000", 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015093600000000001 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:13.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.495612+0000", 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013779314000000001 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.495763+0000", 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015111200000000001 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.505 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:115277 12.f 12:fd10ddd2:::benchmark_data_smithi142_261410_object115276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.483626+0000", 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.013964634, 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014041968, 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 115277 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483626+0000", 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483626+0000", 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483625+0000", 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483657+0000", 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2107000000000003e-05 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483666+0000", 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4320000000000005e-06 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483671+0000", 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6190000000000002e-06 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485583+0000", 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001912335 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485739+0000", 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015626299999999999 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485880+0000", 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014057499999999999 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.495827+0000", 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0099469599999999995 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.495834+0000", 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4510000000000003e-06 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:115279 12.f 12:fece8a04:::benchmark_data_smithi142_261410_object115278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:33:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.483695+0000", 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.013894652, 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014019156, 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 115279 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483695+0000", 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483695+0000", 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483695+0000", 2024-08-17T17:33:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483725+0000", 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0071000000000001e-05 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483735+0000", 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9129999999999998e-06 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.483740+0000", 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4940000000000003e-06 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.489615+0000", 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058749240000000001 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.489724+0000", 2024-08-17T17:33:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010897 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.489881+0000", 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015679499999999999 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.495842+0000", 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059615140000000002 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.495878+0000", 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5948999999999998e-05 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:115281 12.4 12:214baa14:::benchmark_data_smithi142_261410_object115280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.485564+0000", 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.012025951999999999, 2024-08-17T17:33:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012197988999999999, 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 115281 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485564+0000", 2024-08-17T17:33:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485564+0000", 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485563+0000", 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485623+0000", 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0037999999999997e-05 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485635+0000", 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1511999999999999e-05 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485641+0000", 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8889999999999999e-06 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485742+0000", 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101644 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485848+0000", 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106168 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.485990+0000", 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000141636 2024-08-17T17:33:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.493378+0000", 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0073876810000000001 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.493457+0000", 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9485000000000002e-05 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:115284 12.4 12:21790b95:::benchmark_data_smithi142_261410_object115283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.487651+0000", 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0099386539999999999, 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010155558, 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 115284 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.487651+0000", 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.487651+0000", 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.487650+0000", 2024-08-17T17:33:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.487744+0000", 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4188e-05 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.487758+0000", 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3115999999999999e-05 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.487763+0000", 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5140000000000001e-06 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.489739+0000", 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019760910000000001 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.489872+0000", 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000133137 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.489994+0000", 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012173 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.493520+0000", 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035255989999999999 2024-08-17T17:33:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:115285 12.9 e330/319 12:9a2c479f:::benchmark_data_smithi142_261410_object115284:head v 330'7091, mlcod=330'7089)", 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.496068+0000", 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0015219770000000001, 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001805311, 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496068+0000", 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496068+0000", 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496067+0000", 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496100+0000", 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2283000000000002e-05 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496111+0000", 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0874e-05 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496116+0000", 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3170000000000001e-06 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496174+0000", 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8111e-05 2024-08-17T17:33:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496196+0000", 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2118999999999999e-05 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:115289 12.0 e330/319 12:0c451e30:::benchmark_data_smithi142_261410_object115288:head v 330'7165, mlcod=330'7163)", 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.496081+0000", 2024-08-17T17:33:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0015093280000000001, 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001848522, 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496081+0000", 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496081+0000", 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496080+0000", 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496164+0000", 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4207000000000002e-05 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496176+0000", 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1389999999999999e-05 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496181+0000", 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.31e-06 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496227+0000", 2024-08-17T17:33:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6390000000000001e-05 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.496244+0000", 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6520000000000001e-05 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46715.0:115276 12.f e330/319 ondisk, result = 0)", 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.497851+0000", 2024-08-17T17:33:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9997396, 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013181799999999999, 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497851+0000", 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497851+0000", 2024-08-17T17:33:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497850+0000", 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497861+0000", 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0739e-05 2024-08-17T17:33:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497872+0000", 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1652e-05 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497878+0000", 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.942e-06 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497916+0000", 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7423999999999999e-05 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497924+0000", 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3920000000000003e-06 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46715.0:115277 12.f e330/319)", 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:13.497946+0000", 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9996443, 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8304e-05, 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497946+0000", 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497946+0000", 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497945+0000", 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497949+0000", 2024-08-17T17:33:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.726e-06 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497961+0000", 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2179e-05 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:13.497966+0000", 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.04e-06 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:33:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 9 2024-08-17T17:33:13.544 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:13.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:13.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:13.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:13.750 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:13.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:13.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:13.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:13.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:13.995 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 115739 115723 83.1052 56.625 0.0154993 0.0120279 2024-08-17T17:33:14.018 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:14.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:14.152 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:14.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:14.275 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.311 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:14.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:14.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:14.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:14.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:14.562 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:14.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:33:14.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:14.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:14.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:14.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:14.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:14.848 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.900 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:14.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:14.928 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:14.995 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 116595 116579 82.7684 53.5 0.0128337 0.0120767 2024-08-17T17:33:15.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:15.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:15.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:15.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:15.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:15.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:15.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:15.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:15 smithi152 ceph-mon[93456]: pgmap v1150: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2024-08-17T17:33:15.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:15 smithi142 ceph-mon[106729]: pgmap v1150: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2024-08-17T17:33:15.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:15 smithi142 ceph-mon[103614]: pgmap v1150: 89 pgs: 89 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2024-08-17T17:33:15.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:15.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:15.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:15.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:15.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:15.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:15.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:33:15.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:15.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:15.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:15.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:15.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 117409 117393 82.4091 50.875 0.00976639 0.0121278 2024-08-17T17:33:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:16.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:16.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:16.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:16.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:16.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:16.345 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:33:16.346 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:16.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:16.381 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:33:16.382 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:16.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:16.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:16.419 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:33:16.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:16.467 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:33:16.467 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:16.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:33:16.498 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:16.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:16.527 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:33:16.527 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:16.534 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:16.554 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:33:16.555 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:16.587 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:33:16.588 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:16.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:16.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:16.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:16.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:16.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:16.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:16.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:16.996 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0161 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 118194 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 82.0645 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 37.2429 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 227.438 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 31.75 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1313 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 595.886 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 3639 2024-08-17T17:33:16.997 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 508 2024-08-17T17:33:16.998 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0121805 2024-08-17T17:33:16.998 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0120192 2024-08-17T17:33:16.998 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 0.439319 2024-08-17T17:33:16.998 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00117681 2024-08-17T17:33:17.004 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:33:17.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:17.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:17.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:17.207 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:17.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:17.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:17.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:17.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:17.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:17 smithi152 ceph-mon[93456]: pgmap v1151: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:33:17.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1309343027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:17.496 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:17.496 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:17.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:17 smithi142 ceph-mon[106729]: pgmap v1151: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:33:17.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1309343027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:17.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:17 smithi142 ceph-mon[103614]: pgmap v1151: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:33:17.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1309343027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:17.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:17.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:17.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:17.769 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:17.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:17.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:17.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:18.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:18.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:18.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:33:18.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:18.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:18.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:18.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:18.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:18.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:18.467 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:18.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:18.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:18.603 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:18.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:18.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:18.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:18.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:18.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:19.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:19.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:19.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:19.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:19.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:19.270 INFO:teuthology.orchestra.run.smithi142.stderr:marked out osd.2. 2024-08-17T17:33:19.273 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:33:19.017+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e330 definitely_dead 0 2024-08-17T17:33:19.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.296 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:19.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:19.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:19.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:19.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:19.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[106729]: pgmap v1152: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 49 MiB/s wr, 807 op/s 2024-08-17T17:33:19.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3980418823' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:33:19.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:33:19.539 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[106729]: Client client.admin marked osd.2 out, while it was still marked up 2024-08-17T17:33:19.539 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[103614]: pgmap v1152: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 49 MiB/s wr, 807 op/s 2024-08-17T17:33:19.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3980418823' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:33:19.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:33:19.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:19 smithi142 ceph-mon[103614]: Client client.admin marked osd.2 out, while it was still marked up 2024-08-17T17:33:19.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:19.557 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:19.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:19 smithi152 ceph-mon[93456]: pgmap v1152: 89 pgs: 89 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 49 MiB/s wr, 807 op/s 2024-08-17T17:33:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3980418823' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:33:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:19 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:33:19.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:19 smithi152 ceph-mon[93456]: Client client.admin marked osd.2 out, while it was still marked up 2024-08-17T17:33:19.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:19.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:19.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:19.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:19.894 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:19.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:20.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:20.039 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:20.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:20.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:20.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:20.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:20.248 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:20.248 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:20.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:20.298 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:33:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:33:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:20.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:20.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:20.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:20.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:20.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:20.544 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:20.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:20 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-08-17T17:33:20.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:20 smithi152 ceph-mon[93456]: osdmap e331: 8 total, 8 up, 7 in 2024-08-17T17:33:20.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:20 smithi152 ceph-mon[93456]: osdmap e332: 8 total, 8 up, 7 in 2024-08-17T17:33:20.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:20 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-08-17T17:33:20.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:20 smithi142 ceph-mon[103614]: osdmap e331: 8 total, 8 up, 7 in 2024-08-17T17:33:20.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:20 smithi142 ceph-mon[103614]: osdmap e332: 8 total, 8 up, 7 in 2024-08-17T17:33:20.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:20 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-08-17T17:33:20.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:20 smithi142 ceph-mon[106729]: osdmap e331: 8 total, 8 up, 7 in 2024-08-17T17:33:20.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:20 smithi142 ceph-mon[106729]: osdmap e332: 8 total, 8 up, 7 in 2024-08-17T17:33:20.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:20.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:20.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:20.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:20.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:20.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:20.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:20.976 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:21.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:21.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:21.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:33:21.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:21.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:21.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:21.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:21.403 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:21.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:21.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:21.459 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-mon[103614]: pgmap v1154: 89 pgs: 3 peering, 86 active+clean; 7.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 46 MiB/s wr, 751 op/s 2024-08-17T17:33:21.460 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:33:21.460 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-mon[103614]: osdmap e333: 8 total, 8 up, 7 in 2024-08-17T17:33:21.460 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-mon[106729]: pgmap v1154: 89 pgs: 3 peering, 86 active+clean; 7.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 46 MiB/s wr, 751 op/s 2024-08-17T17:33:21.460 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:33:21.460 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-mon[106729]: osdmap e333: 8 total, 8 up, 7 in 2024-08-17T17:33:21.513 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:21.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:21.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:21.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:21 smithi152 ceph-mon[93456]: pgmap v1154: 89 pgs: 3 peering, 86 active+clean; 7.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 46 MiB/s wr, 751 op/s 2024-08-17T17:33:21.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:21 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:33:21.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:21 smithi152 ceph-mon[93456]: osdmap e333: 8 total, 8 up, 7 in 2024-08-17T17:33:21.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:21.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:21.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:21.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:21.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:21.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:21.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:22.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:22.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:33:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:33:21] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:33:22.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.254 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.327 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:33:22.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:22.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:22.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:22.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:22.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4292324427' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:22 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.579 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4292324427' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.579 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:22 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.580 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4292324427' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.580 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:22 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:22.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:22.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:22.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:22.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:23.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:23.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:23.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:23.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:23.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:23.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:23.470 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:23.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:23.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[103614]: pgmap v1157: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 11 peering, 72 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 32 MiB/s wr, 764 op/s; 22149/350043 objects degraded (6.328%); 5 B/s, 1 keys/s, 0 objects/s recovering 2024-08-17T17:33:23.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 22149/350043 objects degraded (6.328%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:23.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:23.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:23.569 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[103614]: osdmap e334: 8 total, 8 up, 7 in 2024-08-17T17:33:23.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[106729]: pgmap v1157: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 11 peering, 72 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 32 MiB/s wr, 764 op/s; 22149/350043 objects degraded (6.328%); 5 B/s, 1 keys/s, 0 objects/s recovering 2024-08-17T17:33:23.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 22149/350043 objects degraded (6.328%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:23.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:23.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:23.569 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:23 smithi142 ceph-mon[106729]: osdmap e334: 8 total, 8 up, 7 in 2024-08-17T17:33:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:23 smithi152 ceph-mon[93456]: pgmap v1157: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 11 peering, 72 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 32 MiB/s wr, 764 op/s; 22149/350043 objects degraded (6.328%); 5 B/s, 1 keys/s, 0 objects/s recovering 2024-08-17T17:33:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:23 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 22149/350043 objects degraded (6.328%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:23 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:23 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:23 smithi152 ceph-mon[93456]: osdmap e334: 8 total, 8 up, 7 in 2024-08-17T17:33:23.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:23.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:23.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:23.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:23.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:23.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:23.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:23.952 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:23.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:24.024 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:33:24.025 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:24.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:24.059 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:33:24.059 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:24.089 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:33:24.089 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:24.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:24.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:24.129 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:33:24.129 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:24.160 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:33:24.161 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:24.191 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:33:24.192 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:24.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:24.223 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:33:24.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:24.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:24.262 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:33:24.262 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:24.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:24.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:24.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:24.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:24.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:24.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1171697861' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:24.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1171697861' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:24.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:24.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1171697861' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:24.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:24.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:24.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:24.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:24.937 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:24.937 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:33:24.938 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:33:24.938 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:25.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:25.033 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:25.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:25.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:25.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:25.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:25 smithi142 ceph-mon[106729]: pgmap v1159: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 11 peering, 70 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 207 B/s rd, 8.0 MiB/s wr, 502 op/s; 29522/348435 objects degraded (8.473%); 7189/348435 objects misplaced (2.063%); 299 KiB/s, 2 keys/s, 5 objects/s recovering 2024-08-17T17:33:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:25 smithi142 ceph-mon[103614]: pgmap v1159: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 11 peering, 70 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 207 B/s rd, 8.0 MiB/s wr, 502 op/s; 29522/348435 objects degraded (8.473%); 7189/348435 objects misplaced (2.063%); 299 KiB/s, 2 keys/s, 5 objects/s recovering 2024-08-17T17:33:25.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:33:25.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:25.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:25.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:25 smithi152 ceph-mon[93456]: pgmap v1159: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 11 peering, 70 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 207 B/s rd, 8.0 MiB/s wr, 502 op/s; 29522/348435 objects degraded (8.473%); 7189/348435 objects misplaced (2.063%); 299 KiB/s, 2 keys/s, 5 objects/s recovering 2024-08-17T17:33:25.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:25.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:25.678 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:25.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:33:25.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:25.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:25.837 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:25.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:25.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:25.961 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:33:25.961 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:33:25.961 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [1, 3, 4, 5, 6, 7, 0] 2024-08-17T17:33:25.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 7 2024-08-17T17:33:25.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:26.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.124 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:26.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:26.147 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:26.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:26.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:26.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:26.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:26.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:26.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:26 smithi152 ceph-mon[93456]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:33:26.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:26 smithi142 ceph-mon[106729]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:33:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:26 smithi142 ceph-mon[103614]: Health check update: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:33:26.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:26.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:26.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:26.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:26.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:26.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:26.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:27.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:27.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:27.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:27.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:27.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:27.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:27.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:27.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:27.414 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:27 smithi152 ceph-mon[93456]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 6.6 MiB/s wr, 518 op/s; 58962/346536 objects degraded (17.015%); 14364/346536 objects misplaced (4.145%); 1.0 MiB/s, 1 keys/s, 18 objects/s recovering 2024-08-17T17:33:27.414 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:27 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-17T17:33:27.414 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:27.414 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:27.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:27.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:27.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:27.516 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[106729]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 6.6 MiB/s wr, 518 op/s; 58962/346536 objects degraded (17.015%); 14364/346536 objects misplaced (4.145%); 1.0 MiB/s, 1 keys/s, 18 objects/s recovering 2024-08-17T17:33:27.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-17T17:33:27.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:27.517 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:27.517 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[103614]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 7.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 6.6 MiB/s wr, 518 op/s; 58962/346536 objects degraded (17.015%); 14364/346536 objects misplaced (4.145%); 1.0 MiB/s, 1 keys/s, 18 objects/s recovering 2024-08-17T17:33:27.517 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-17T17:33:27.517 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:27.518 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:27.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:27.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:27.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:27.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:27.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:27.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:27.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:28.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:28.079 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:28.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:28.114 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:28.115 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:33:28.115 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:33:28.115 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:33:28.115 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:28.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.211 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:28.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:28.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:28.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:28.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:28.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:28.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:28 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 59032/341079 objects degraded (17.307%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:28.605 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:28 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 59032/341079 objects degraded (17.307%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:28.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:28.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:28.749 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:28.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:28 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 59032/341079 objects degraded (17.307%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:28.782 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:28.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:33:28.830 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:28.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:28.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:29.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:29.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:29.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:29.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:29.245 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:29.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:29.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:29.397 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:33:29.456 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:29.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:29.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:29 smithi142 ceph-mon[106729]: pgmap v1161: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 128 B/s rd, 2.3 MiB/s wr, 533 op/s; 59032/341079 objects degraded (17.307%); 14200/341079 objects misplaced (4.163%); 1.5 MiB/s, 1 keys/s, 25 objects/s recovering 2024-08-17T17:33:29.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1309469263' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:29.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:29.495 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:29 smithi142 ceph-mon[103614]: pgmap v1161: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 128 B/s rd, 2.3 MiB/s wr, 533 op/s; 59032/341079 objects degraded (17.307%); 14200/341079 objects misplaced (4.163%); 1.5 MiB/s, 1 keys/s, 25 objects/s recovering 2024-08-17T17:33:29.495 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1309469263' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:29.495 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:29.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:29.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:29.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:29.737 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:29 smithi152 ceph-mon[93456]: pgmap v1161: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 128 B/s rd, 2.3 MiB/s wr, 533 op/s; 59032/341079 objects degraded (17.307%); 14200/341079 objects misplaced (4.163%); 1.5 MiB/s, 1 keys/s, 25 objects/s recovering 2024-08-17T17:33:29.737 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1309469263' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:29.737 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:29.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:29.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:29.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:29.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:30.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:30.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:30.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:30.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:30.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:30.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:30.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:30.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:30.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:33:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:33:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:30.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:30.421 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:30.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:30.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:30.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:30.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:30.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:30.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:30 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:30.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:30.741 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:30 smithi152 ceph-mon[93456]: osdmap e335: 8 total, 8 up, 7 in 2024-08-17T17:33:30.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:30.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:30.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:30 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:30 smithi142 ceph-mon[106729]: osdmap e335: 8 total, 8 up, 7 in 2024-08-17T17:33:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:30 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:30 smithi142 ceph-mon[103614]: osdmap e335: 8 total, 8 up, 7 in 2024-08-17T17:33:30.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:30.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:31.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:31.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:31.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:31.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:31.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:31 smithi152 ceph-mon[93456]: pgmap v1163: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 373 op/s; 59032/341079 objects degraded (17.307%); 14200/341079 objects misplaced (4.163%); 1.5 MiB/s, 25 objects/s recovering 2024-08-17T17:33:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:31 smithi142 ceph-mon[106729]: pgmap v1163: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 373 op/s; 59032/341079 objects degraded (17.307%); 14200/341079 objects misplaced (4.163%); 1.5 MiB/s, 25 objects/s recovering 2024-08-17T17:33:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:31 smithi142 ceph-mon[103614]: pgmap v1163: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 373 op/s; 59032/341079 objects degraded (17.307%); 14200/341079 objects misplaced (4.163%); 1.5 MiB/s, 25 objects/s recovering 2024-08-17T17:33:31.646 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196882 10.9 10:9799130f:::benchmark_data_smithi142_192453_object71134:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.632275+0000", 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 466.010106388, 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0336117419999999, 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196882 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632275+0000", 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632275+0000", 2024-08-17T17:33:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632274+0000", 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632289+0000", 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5226e-05 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632301+0000", 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1855000000000001e-05 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632308+0000", 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6559999999999992e-06 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632388+0000", 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9981999999999994e-05 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632393+0000", 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8980000000000004e-06 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.646118+0000", 2024-08-17T17:33:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.013724946 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.646236+0000", 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117889 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.646652+0000", 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041588299999999998 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657653+0000", 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011001105000000001 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665792+0000", 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081389530000000009 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665820+0000", 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8107999999999999e-05 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665887+0000", 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6853000000000005e-05 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196883 10.19 10:9ca98a34:::benchmark_data_smithi142_192453_object71135:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.632324+0000", 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 466.01005812400001, 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0347239960000001, 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196883 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632324+0000", 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632324+0000", 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632323+0000", 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632326+0000", 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4679999999999999e-06 2024-08-17T17:33:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632335+0000", 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1509999999999993e-06 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632339+0000", 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0659999999999997e-06 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632383+0000", 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3869999999999998e-05 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632391+0000", 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9189999999999999e-06 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.656198+0000", 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.02380776 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.656299+0000", 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010058199999999999 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.656591+0000", 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029256699999999998 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.662161+0000", 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055696649999999997 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666940+0000", 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047791739999999997 2024-08-17T17:33:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666970+0000", 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9424000000000002e-05 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667048+0000", 2024-08-17T17:33:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7824000000000006e-05 2024-08-17T17:33:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196884 10.9 10:932fabb8:::benchmark_data_smithi142_192453_object71136:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.664 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.632358+0000", 2024-08-17T17:33:31.664 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 466.01002345299997, 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0337072119999999, 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196884 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632358+0000", 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632358+0000", 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632358+0000", 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632365+0000", 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.131e-06 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632372+0000", 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9210000000000003e-06 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632376+0000", 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1389999999999997e-06 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632413+0000", 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7081e-05 2024-08-17T17:33:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.632420+0000", 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.092e-06 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.648279+0000", 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0158593549999999 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.648343+0000", 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4008000000000001e-05 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.648503+0000", 2024-08-17T17:33:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000159202 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657679+0000", 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091760550000000007 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665931+0000", 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082521910000000007 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665977+0000", 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5769000000000002e-05 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666065+0000", 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8729999999999999e-05 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196899 10.9 10:931bb7c9:::benchmark_data_smithi142_192453_object71151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.651570+0000", 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.99081172199999, 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0147440130000001, 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196899 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651570+0000", 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651570+0000", 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651569+0000", 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651581+0000", 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1941e-05 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651592+0000", 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1060000000000001e-05 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651596+0000", 2024-08-17T17:33:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3610000000000003e-06 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651633+0000", 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7149e-05 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.651640+0000", 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4790000000000001e-06 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.649530+0000", 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99789052199999995 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.649587+0000", 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6436e-05 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.649743+0000", 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015683299999999999 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657686+0000", 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079420529999999993 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666174+0000", 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084886649999999994 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666217+0000", 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2413999999999997e-05 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666314+0000", 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7349999999999995e-05 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196912 10.9 10:916be0ed:::benchmark_data_smithi142_192453_object71164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.671164+0000", 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.97121798299997, 2024-08-17T17:33:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99529540599999999, 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196912 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671164+0000", 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671164+0000", 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671162+0000", 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671168+0000", 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5480000000000001e-06 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671180+0000", 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2483000000000001e-05 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671185+0000", 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4700000000000004e-06 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671226+0000", 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1479e-05 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.671232+0000", 2024-08-17T17:33:31.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3580000000000002e-06 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.650711+0000", 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97947887099999997 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.650768+0000", 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7673999999999997e-05 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.650936+0000", 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000167763 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657691+0000", 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067545310000000003 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666369+0000", 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086787889999999993 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666396+0000", 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6602999999999999e-05 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666459+0000", 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3099e-05 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.689 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196931 10.9 10:93e038e5:::benchmark_data_smithi142_192453_object71183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.699037+0000", 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.94334486999998, 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96755423600000001, 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196931 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699037+0000", 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699037+0000", 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699035+0000", 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699042+0000", 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0339999999999999e-06 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699055+0000", 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2645e-05 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699061+0000", 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4250000000000003e-06 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699119+0000", 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7262999999999997e-05 2024-08-17T17:33:31.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.699126+0000", 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6850000000000001e-06 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.652023+0000", 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95289663599999996 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.652095+0000", 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1792000000000007e-05 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.652270+0000", 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017532800000000001 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657695+0000", 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054253069999999999 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666483+0000", 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087877819999999992 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666517+0000", 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3671999999999999e-05 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666591+0000", 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4296999999999997e-05 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.697 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196937 10.9 10:92194ed0:::benchmark_data_smithi142_192453_object71189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.710562+0000", 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.93181975200002, 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95620519500000001, 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.698 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196937 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710562+0000", 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710562+0000", 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710560+0000", 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710572+0000", 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2242000000000001e-05 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710587+0000", 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4275e-05 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710594+0000", 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1420000000000004e-06 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710644+0000", 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9672000000000001e-05 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.710653+0000", 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0459999999999994e-06 2024-08-17T17:33:31.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.653248+0000", 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94259508299999994 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.653308+0000", 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0547000000000002e-05 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.653466+0000", 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000158171 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657700+0000", 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042333309999999999 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666644+0000", 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089438250000000007 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666691+0000", 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7116000000000001e-05 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666767+0000", 2024-08-17T17:33:31.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6465999999999996e-05 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196979 10.19 10:9d135c22:::benchmark_data_smithi142_192453_object71231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.767961+0000", 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.87442058200003, 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89922983999999995, 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.706 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196979 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.767961+0000", 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.767961+0000", 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.767960+0000", 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.767965+0000", 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.118e-06 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.767975+0000", 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0424e-05 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.767981+0000", 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5319999999999997e-06 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.768033+0000", 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2098999999999997e-05 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.768041+0000", 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2889999999999998e-06 2024-08-17T17:33:31.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657699+0000", 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88965833999999999 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657767+0000", 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7162000000000005e-05 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657973+0000", 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020665899999999999 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.662166+0000", 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004193202 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667072+0000", 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049050689999999997 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667107+0000", 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5354000000000001e-05 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667191+0000", 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4019999999999999e-05 2024-08-17T17:33:31.713 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:196998 10.9 10:97b9acc2:::benchmark_data_smithi142_192453_object71250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.796925+0000", 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.84545689399999, 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86997010799999996, 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 196998 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.796925+0000", 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.796925+0000", 2024-08-17T17:33:31.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.796923+0000", 2024-08-17T17:33:31.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:31.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.796932+0000", 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2620000000000004e-06 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.796949+0000", 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7119e-05 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.796955+0000", 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.874e-06 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.797008+0000", 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3015e-05 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.797018+0000", 2024-08-17T17:33:31.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5529999999999998e-06 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.654419+0000", 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.85740129899999995 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.654479+0000", 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0037000000000001e-05 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.654639+0000", 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015968300000000001 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.657705+0000", 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00306609 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666801+0000", 2024-08-17T17:33:31.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090962600000000001 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666837+0000", 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5923999999999997e-05 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666895+0000", 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7754000000000003e-05 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.722 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197029 10.9 10:920a7b6f:::benchmark_data_smithi142_192453_object71281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.836807+0000", 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.80557515499999, 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.83023563499999997, 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.723 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197029 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836807+0000", 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836807+0000", 2024-08-17T17:33:31.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836805+0000", 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836811+0000", 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8549999999999998e-06 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836822+0000", 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0507e-05 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836826+0000", 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2819999999999998e-06 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836878+0000", 2024-08-17T17:33:31.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2091e-05 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.836885+0000", 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3989999999999998e-06 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.655616+0000", 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81873022500000003 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.655672+0000", 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6416000000000003e-05 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.655829+0000", 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000157307 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.659979+0000", 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041497369999999997 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666933+0000", 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069535020000000003 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666974+0000", 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1458000000000002e-05 2024-08-17T17:33:31.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667042+0000", 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8070000000000004e-05 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197032 10.9 10:946ea018:::benchmark_data_smithi142_192453_object71284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.844049+0000", 2024-08-17T17:33:31.730 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.79833217100003, 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82772621099999999, 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197032 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844049+0000", 2024-08-17T17:33:31.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844049+0000", 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844048+0000", 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844056+0000", 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2199999999999992e-06 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844070+0000", 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4277000000000001e-05 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844076+0000", 2024-08-17T17:33:31.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1380000000000004e-06 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844116+0000", 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9431000000000002e-05 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.844124+0000", 2024-08-17T17:33:31.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6310000000000003e-06 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.661017+0000", 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81689315900000004 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.661074+0000", 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7312999999999998e-05 2024-08-17T17:33:31.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.661375+0000", 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00030058100000000001 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665759+0000", 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043840620000000002 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.670438+0000", 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046796299999999997 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.670486+0000", 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7212000000000001e-05 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.671776+0000", 2024-08-17T17:33:31.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012901919999999999 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197052 10.19 10:9913b5b8:::benchmark_data_smithi142_192453_object71304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.872024+0000", 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.77035777999998, 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79531702699999995, 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.738 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197052 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872024+0000", 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872024+0000", 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872022+0000", 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872031+0000", 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5450000000000003e-06 2024-08-17T17:33:31.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872051+0000", 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.001e-05 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872058+0000", 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5939999999999996e-06 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872111+0000", 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2303999999999999e-05 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.872119+0000", 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3820000000000004e-06 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.658869+0000", 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78675025600000004 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.658975+0000", 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105399 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.659172+0000", 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019703799999999999 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.664422+0000", 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052499829999999997 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667225+0000", 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002803842 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667266+0000", 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0198999999999999e-05 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667341+0000", 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5232000000000001e-05 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.745 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197076 10.9 10:93f9694f:::benchmark_data_smithi142_192453_object71328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.906362+0000", 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.73601989500003, 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76559435600000003, 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.746 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197076 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906362+0000", 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906362+0000", 2024-08-17T17:33:31.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906360+0000", 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906366+0000", 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7359999999999998e-06 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906378+0000", 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1392e-05 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906385+0000", 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6890000000000004e-06 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906430+0000", 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4879000000000002e-05 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.906438+0000", 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4340000000000003e-06 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.662807+0000", 2024-08-17T17:33:31.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75636922500000003 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.662924+0000", 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011764000000000001 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.663189+0000", 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000264677 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666077+0000", 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028876010000000001 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.671821+0000", 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057447039999999998 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.671877+0000", 2024-08-17T17:33:31.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5569000000000003e-05 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.671956+0000", 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9076e-05 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.753 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197095 10.9 10:920f2652:::benchmark_data_smithi142_192453_object71347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:45.935074+0000", 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.707307598, 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74623474400000001, 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197095 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935074+0000", 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935074+0000", 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935072+0000", 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935079+0000", 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6819999999999997e-06 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935097+0000", 2024-08-17T17:33:31.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8050000000000002e-05 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935105+0000", 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9060000000000002e-06 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935154+0000", 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9002000000000003e-05 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:45.935161+0000", 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5120000000000002e-06 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.664433+0000", 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.729271118 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.664494+0000", 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1379999999999998e-05 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.664657+0000", 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000163487 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.666082+0000", 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001424161 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.681164+0000", 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015082629 2024-08-17T17:33:31.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.681213+0000", 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8990999999999999e-05 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.681309+0000", 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5633999999999999e-05 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.761 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197172 10.19 10:99e61580:::benchmark_data_smithi142_192453_object71424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:46.134860+0000", 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.50752134599998, 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.53265103400000002, 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.762 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197172 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134860+0000", 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134860+0000", 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134859+0000", 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134865+0000", 2024-08-17T17:33:31.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1639999999999998e-06 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134876+0000", 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1545999999999999e-05 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134882+0000", 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.993e-06 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134919+0000", 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6977999999999997e-05 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.134926+0000", 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9029999999999998e-06 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.659990+0000", 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.52506332600000005 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.660053+0000", 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3579999999999998e-05 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.660194+0000", 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000141108 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.664428+0000", 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042336350000000003 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667394+0000", 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029656079999999998 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667432+0000", 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8244000000000002e-05 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.667511+0000", 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9569999999999999e-05 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36309.0:197175 10.19 10:9d86de5b:::benchmark_data_smithi142_192453_object71427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-17T17:33:31.770 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:25:46.151841+0000", 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 465.49054071400002, 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.51714033500000001, 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36309", 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197175 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151841+0000", 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151841+0000", 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151840+0000", 2024-08-17T17:33:31.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151861+0000", 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1291999999999999e-05 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151873+0000", 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1734e-05 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151877+0000", 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8130000000000003e-06 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151918+0000", 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0995999999999998e-05 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:33:31.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.151924+0000", 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6960000000000002e-06 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.661033+0000", 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.50910887000000005 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.661137+0000", 2024-08-17T17:33:31.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104115 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2", 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.661331+0000", 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000193788 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.665649+0000", 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043182899999999998 2024-08-17T17:33:31.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.668741+0000", 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030922430000000002 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.668911+0000", 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016947300000000001 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:25:46.668981+0000", 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0332000000000004e-05 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:82766 11.7 11:e765fd9c:::benchmark_data_smithi142_227468_object82765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:28:01.420117+0000", 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 330.22226469499998, 2024-08-17T17:33:31.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.43302916200000002, 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 82766 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420117+0000", 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420117+0000", 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420115+0000", 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420167+0000", 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2197999999999997e-05 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420182+0000", 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5009e-05 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420189+0000", 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3040000000000001e-06 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420236+0000", 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7225000000000003e-05 2024-08-17T17:33:31.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420365+0000", 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012870899999999999 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:31.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.420508+0000", 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014282300000000001 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.423070+0000", 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025624659999999998 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.435822+0000", 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012751495 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853057+0000", 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41723528100000001 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853109+0000", 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1950000000000002e-05 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853146+0000", 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7391999999999997e-05 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:82772 11.7 11:ec332f3b:::benchmark_data_smithi142_227468_object82771:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:28:01.429458+0000", 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 330.21292363700002, 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.42380162999999998, 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:33:31.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 82772 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429458+0000", 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429458+0000", 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429457+0000", 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429511+0000", 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4799999999999997e-05 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429524+0000", 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2303999999999999e-05 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429532+0000", 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0099999999999995e-06 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429583+0000", 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1022e-05 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429718+0000", 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013500900000000001 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.429879+0000", 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016096400000000001 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.434646+0000", 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047668249999999997 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.438864+0000", 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042184199999999996 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853194+0000", 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41432970899999999 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853225+0000", 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1044000000000003e-05 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853260+0000", 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4879000000000003e-05 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.795 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:82775 11.7 11:e8a28a5a:::benchmark_data_smithi142_227468_object82774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:28:01.432126+0000", 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 330.21025541, 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.42127576700000002, 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 82775 2024-08-17T17:33:31.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432126+0000", 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432126+0000", 2024-08-17T17:33:31.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432125+0000", 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432195+0000", 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9983999999999999e-05 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432206+0000", 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0688e-05 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432210+0000", 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.459e-06 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432585+0000", 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037503599999999998 2024-08-17T17:33:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432675+0000", 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9105999999999997e-05 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.432764+0000", 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9351000000000006e-05 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.434792+0000", 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020277680000000001 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.438900+0000", 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041083229999999997 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853309+0000", 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41440879899999999 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853367+0000", 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8344000000000002e-05 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853402+0000", 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4854000000000002e-05 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:82801 11.7 11:eefc3534:::benchmark_data_smithi142_227468_object82800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:28:01.469562+0000", 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 330.17281957500001, 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.38396663399999997, 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:31.803 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 82801 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469562+0000", 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469562+0000", 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469561+0000", 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469594+0000", 2024-08-17T17:33:31.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3380999999999998e-05 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469606+0000", 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.223e-05 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469611+0000", 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1590000000000001e-06 2024-08-17T17:33:31.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469660+0000", 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8130000000000002e-05 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469797+0000", 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000137107 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:31.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.469962+0000", 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016532 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.472132+0000", 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021704350000000001 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.472478+0000", 2024-08-17T17:33:31.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034541999999999998 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853443+0000", 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.38096523100000002 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853497+0000", 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4283999999999997e-05 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:31.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:28:01.853529+0000", 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1333e-05 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:31.810 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:31.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:31.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:32.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:32.081 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:33:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:33:31] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.51.0" 2024-08-17T17:33:32.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:32.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:32.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:32.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:32.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:32.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:32.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:32.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:32.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:32.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:33:32.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:32.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:32.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:32.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:33.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:33.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.072 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:33.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:33.136 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:33.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:33.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:33.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:33.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:33.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:33.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:33.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:33 smithi152 ceph-mon[93456]: pgmap v1164: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 59112/335859 objects degraded (17.600%); 14031/335859 objects misplaced (4.178%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:33:33.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:33.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:33.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:33.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:33.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:33.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:33 smithi142 ceph-mon[103614]: pgmap v1164: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 59112/335859 objects degraded (17.600%); 14031/335859 objects misplaced (4.178%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:33:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:33 smithi142 ceph-mon[106729]: pgmap v1164: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 59112/335859 objects degraded (17.600%); 14031/335859 objects misplaced (4.178%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:33:33.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:33.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:33.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:34.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:34.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:34.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:34.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:34.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:34.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:34.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:34.451 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:34.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:34.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:34.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:34.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:34.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:34.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:34 smithi152 ceph-mon[93456]: pgmap v1165: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 59159/333687 objects degraded (17.729%); 13881/333687 objects misplaced (4.160%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:33:34.772 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:34 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 59159/333687 objects degraded (17.729%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:34.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:34.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:34 smithi142 ceph-mon[106729]: pgmap v1165: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 59159/333687 objects degraded (17.729%); 13881/333687 objects misplaced (4.160%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:33:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:34 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 59159/333687 objects degraded (17.729%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:34.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:34 smithi142 ceph-mon[103614]: pgmap v1165: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 59159/333687 objects degraded (17.729%); 13881/333687 objects misplaced (4.160%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:33:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:34 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 59159/333687 objects degraded (17.729%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:34.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:34.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:34.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:35.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:35.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:35.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:35.260 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:35.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:35.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:127063 12.c e335/332 12:3abb15d7:::benchmark_data_smithi142_261410_object8866:head v 335'7845, mlcod=324'521)", 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.289274+0000", 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.019462383999999999, 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019485703, 2024-08-17T17:33:35.384 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289274+0000", 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289274+0000", 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289273+0000", 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289277+0000", 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8850000000000001e-06 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289286+0000", 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0969999999999995e-06 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289290+0000", 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9609999999999999e-06 2024-08-17T17:33:35.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289585+0000", 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029475699999999998 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289608+0000", 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2628000000000001e-05 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:127081 12.c e335/332 12:337ad595:::benchmark_data_smithi142_261410_object8884:head v 335'7846, mlcod=324'521)", 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.295448+0000", 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.01328795, 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013350503, 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.295448+0000", 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.295448+0000", 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.295447+0000", 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.295456+0000", 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5830000000000007e-06 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.295472+0000", 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6174000000000001e-05 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.295479+0000", 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.272e-06 2024-08-17T17:33:35.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.299791+0000", 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043123520000000002 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.299816+0000", 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4783e-05 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:127083 12.c e335/332)", 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.297254+0000", 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.011481794, 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011583813, 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.297254+0000", 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.297254+0000", 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.297253+0000", 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.297260+0000", 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8759999999999999e-06 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.297276+0000", 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5235000000000001e-05 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.297282+0000", 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.395 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:127096 12.3 e335/319 12:c1550e37:::benchmark_data_smithi142_261410_object8899:head v 335'8037, mlcod=335'8035)", 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.307543+0000", 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0011930090000000001, 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001340567, 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307543+0000", 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307543+0000", 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307542+0000", 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307550+0000", 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3219999999999993e-06 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307567+0000", 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6194000000000001e-05 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307573+0000", 2024-08-17T17:33:35.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7139999999999996e-06 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307614+0000", 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1003e-05 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.307635+0000", 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1138e-05 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.399 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.400 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46715.0:127086 12.d e335/332)", 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.308876+0000", 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9998593, 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2593000000000002e-05, 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.308876+0000", 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.308876+0000", 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.308876+0000", 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:35.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.308882+0000", 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5520000000000001e-06 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.308899+0000", 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7297000000000001e-05 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.308906+0000", 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8920000000000003e-06 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:127086 12.d 12:bcd7b38c:::benchmark_data_smithi142_261410_object8889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.281165+0000", 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.027570647, 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027831907999999999, 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:33:35.404 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 127086 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.281165+0000", 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.281165+0000", 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.281164+0000", 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.281171+0000", 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5629999999999997e-06 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.281199+0000", 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8350000000000001e-05 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.281207+0000", 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8909999999999995e-06 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:35.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.291389+0000", 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010182319 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.291567+0000", 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017732100000000001 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.291746+0000", 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000179655 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.302028+0000", 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010282049999999999 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.302542+0000", 2024-08-17T17:33:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051320800000000002 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.309009+0000", 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064672430000000001 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:127078 12.0 e335/332 12:0b893bcb:::benchmark_data_smithi142_261410_object8881:head v 335'7933, mlcod=322'123)", 2024-08-17T17:33:35.410 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:35.289245+0000", 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.019490436, 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019843902, 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289245+0000", 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289245+0000", 2024-08-17T17:33:35.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289245+0000", 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289250+0000", 2024-08-17T17:33:35.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4819999999999996e-06 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289255+0000", 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4519999999999996e-06 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.289259+0000", 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5240000000000001e-06 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:33:35.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.306634+0000", 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017375163999999998 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:35.306653+0000", 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9490000000000001e-05 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:33:35.414 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 7 2024-08-17T17:33:35.415 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:35.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:35.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:35.545 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:35.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:35.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:35.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:35.809 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:35.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:35.954 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:36.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:36.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:36.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:36.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:36.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:36.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:33:36.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:36.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:36.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:36.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:36.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:36.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:36.650 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:36.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:36.725 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:36.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:36.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:36.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:36.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:36.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:36.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:37.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:37.059 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:37.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:37.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:37.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:37.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:37.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:37.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:37 smithi142 ceph-mon[106729]: pgmap v1166: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 59217/331278 objects degraded (17.875%); 13705/331278 objects misplaced (4.137%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:33:37.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:37 smithi142 ceph-mon[103614]: pgmap v1166: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 59217/331278 objects degraded (17.875%); 13705/331278 objects misplaced (4.137%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:33:37.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:37.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:37 smithi152 ceph-mon[93456]: pgmap v1166: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 59217/331278 objects degraded (17.875%); 13705/331278 objects misplaced (4.137%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:33:37.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:37.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:37.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:37.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:37.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:37.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:38.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:38.036 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:38.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:38.169 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:38.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:38.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:38.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:38.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:38.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:38.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:38.665 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:38.665 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:33:38.665 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:33:38.665 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:33:38.665 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:38.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:38.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:38.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:38.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:38.970 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:39.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:39.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:39.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:33:39.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:39.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:39.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:39.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:39 smithi152 ceph-mon[93456]: pgmap v1167: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 59277/325605 objects degraded (18.205%); 13544/325605 objects misplaced (4.160%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:33:39.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:39.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:33:39.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:39.493 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:39 smithi142 ceph-mon[103614]: pgmap v1167: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 59277/325605 objects degraded (18.205%); 13544/325605 objects misplaced (4.160%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:33:39.493 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:33:39.494 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:39.500 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:39 smithi142 ceph-mon[106729]: pgmap v1167: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 59277/325605 objects degraded (18.205%); 13544/325605 objects misplaced (4.160%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:33:39.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:39.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:39.542 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:33:39.543 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:39.583 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:33:39.583 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:39.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:39.642 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:33:39.642 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:39.648 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:39.657 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:39.685 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:33:39.685 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:39.729 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:33:39.729 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:39.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:39.766 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:33:39.767 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:33:39.768 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:39 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:33:39.544+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e335 definitely_dead 0 2024-08-17T17:33:39.793 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:33:39.794 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:39.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:39.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:39.817 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:39.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:39.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:40.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:40.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:40.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:40.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:40.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:40.276 INFO:teuthology.orchestra.run.smithi142.stderr:marked out osd.7. 2024-08-17T17:33:40.282 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:33:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:33:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:40.306 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/477276707' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:40.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1985672877' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-08-17T17:33:40.307 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:40 smithi142 ceph-mon[106729]: Client client.admin marked osd.7 out, while it was still marked up 2024-08-17T17:33:40.307 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/477276707' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:40.307 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1985672877' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-08-17T17:33:40.307 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:40 smithi142 ceph-mon[103614]: Client client.admin marked osd.7 out, while it was still marked up 2024-08-17T17:33:40.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:40.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:40.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:40.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.470 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:40.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:40.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:40.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/477276707' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:40.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1985672877' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-08-17T17:33:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:40 smithi152 ceph-mon[93456]: Client client.admin marked osd.7 out, while it was still marked up 2024-08-17T17:33:40.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:40.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:40.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:40.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:40.834 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:40.856 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:40.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:40.980 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:41.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:41.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:41.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:41.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:41.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:41.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:41.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:41.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:41.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:41 smithi152 ceph-mon[93456]: pgmap v1168: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 59277/325605 objects degraded (18.205%); 13544/325605 objects misplaced (4.160%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:33:41.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1985672877' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-08-17T17:33:41.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:41 smithi152 ceph-mon[93456]: osdmap e336: 8 total, 8 up, 6 in 2024-08-17T17:33:41.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 59277/325605 objects degraded (18.205%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:41.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:41 smithi152 ceph-mon[93456]: osdmap e337: 8 total, 8 up, 6 in 2024-08-17T17:33:41.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:41.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[106729]: pgmap v1168: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 59277/325605 objects degraded (18.205%); 13544/325605 objects misplaced (4.160%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:33:41.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1985672877' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-08-17T17:33:41.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[106729]: osdmap e336: 8 total, 8 up, 6 in 2024-08-17T17:33:41.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 59277/325605 objects degraded (18.205%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:41.537 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[106729]: osdmap e337: 8 total, 8 up, 6 in 2024-08-17T17:33:41.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[103614]: pgmap v1168: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 59277/325605 objects degraded (18.205%); 13544/325605 objects misplaced (4.160%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:33:41.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1985672877' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-08-17T17:33:41.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[103614]: osdmap e336: 8 total, 8 up, 6 in 2024-08-17T17:33:41.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 59277/325605 objects degraded (18.205%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:41.537 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-mon[103614]: osdmap e337: 8 total, 8 up, 6 in 2024-08-17T17:33:41.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:41.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:41.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:41.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:41.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:41.846 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:41.846 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:41.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:33:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:33:41] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:33:41.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:41.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:42.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:42.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:42.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:42.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:42.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:42.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:42.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:42.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:42.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:42.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:42.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:42.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:42.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:42.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:42.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: 12.d starting backfill to osd.4 from (0'0,0'0] MAX to 335'8199 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: 12.6 starting backfill to osd.6 from (0'0,0'0] MAX to 335'8216 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: osdmap e338: 8 total, 8 up, 6 in 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 335'8303 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 335'8081 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: 12.7 starting backfill to osd.5 from (0'0,0'0] MAX to 335'8059 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: 12.2 starting backfill to osd.6 from (0'0,0'0] MAX to 335'8071 2024-08-17T17:33:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:42 smithi152 ceph-mon[93456]: pgmap v1172: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 11 peering, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 44564/321150 objects degraded (13.876%); 40903/321150 objects misplaced (12.736%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:33:42.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:42.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: 12.d starting backfill to osd.4 from (0'0,0'0] MAX to 335'8199 2024-08-17T17:33:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: 12.6 starting backfill to osd.6 from (0'0,0'0] MAX to 335'8216 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: osdmap e338: 8 total, 8 up, 6 in 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 335'8303 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 335'8081 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: 12.7 starting backfill to osd.5 from (0'0,0'0] MAX to 335'8059 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: 12.2 starting backfill to osd.6 from (0'0,0'0] MAX to 335'8071 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[103614]: pgmap v1172: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 11 peering, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 44564/321150 objects degraded (13.876%); 40903/321150 objects misplaced (12.736%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: 12.d starting backfill to osd.4 from (0'0,0'0] MAX to 335'8199 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: 12.6 starting backfill to osd.6 from (0'0,0'0] MAX to 335'8216 2024-08-17T17:33:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: osdmap e338: 8 total, 8 up, 6 in 2024-08-17T17:33:42.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 335'8303 2024-08-17T17:33:42.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 335'8081 2024-08-17T17:33:42.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: 12.7 starting backfill to osd.5 from (0'0,0'0] MAX to 335'8059 2024-08-17T17:33:42.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: 12.2 starting backfill to osd.6 from (0'0,0'0] MAX to 335'8071 2024-08-17T17:33:42.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:42 smithi142 ceph-mon[106729]: pgmap v1172: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 11 peering, 2 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 44564/321150 objects degraded (13.876%); 40903/321150 objects misplaced (12.736%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:33:42.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:42.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:42.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:43.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:43.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:43.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:43.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:43.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:43.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:43.453 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:43.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:43.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:43.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:43.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:43.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:43.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:43.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:43.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:43.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:43 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:33:43.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:43.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:33:43.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:43.869 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:43 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:33:43.870 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:43 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:33:43.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:43.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:43.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:44.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:44.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:44.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:44.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:44.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:44.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:44.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:44.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:44.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:44.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:44.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:44.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:44.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:44.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:44 smithi152 ceph-mon[93456]: pgmap v1173: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 11 peering, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 66541/320448 objects degraded (20.765%); 34189/320448 objects misplaced (10.669%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:33:44.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:44.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:44 smithi142 ceph-mon[106729]: pgmap v1173: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 11 peering, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 66541/320448 objects degraded (20.765%); 34189/320448 objects misplaced (10.669%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:33:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:44 smithi142 ceph-mon[103614]: pgmap v1173: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 11 peering, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 286 op/s; 66541/320448 objects degraded (20.765%); 34189/320448 objects misplaced (10.669%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:33:44.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:44.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:45.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:45.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:45.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:45.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:45.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:45.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:45.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:45.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:45.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:45.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:45.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:45.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:45.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:45.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:45.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:45.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:45.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:45.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:45.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:45.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:46.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:46.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 66541/320448 objects degraded (20.765%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.078 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:46.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 66541/320448 objects degraded (20.765%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:46.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 66541/320448 objects degraded (20.765%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:46.180 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:46.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:46.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:46.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:46.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:46.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:46.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:46.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:46.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:46.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:46.697 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:46.752 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:46.752 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:46.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:46.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:46.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:46.937 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:46 smithi142 ceph-mon[106729]: pgmap v1174: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 289 op/s; 66694/320388 objects degraded (20.817%); 33786/320388 objects misplaced (10.545%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:33:46.938 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:46 smithi142 ceph-mon[103614]: pgmap v1174: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 289 op/s; 66694/320388 objects degraded (20.817%); 33786/320388 objects misplaced (10.545%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:33:46.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:46.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:47.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:47.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:47.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:46 smithi152 ceph-mon[93456]: pgmap v1174: 89 pgs: 1 active+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 289 op/s; 66694/320388 objects degraded (20.817%); 33786/320388 objects misplaced (10.545%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:33:47.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:47.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:47.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:47.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:47.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:47.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:47.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:47.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:47.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:47.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:47.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:47.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:47.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:47.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:47.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:47.941 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:48.043 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:48.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:48.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:47 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-17T17:33:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:47 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-17T17:33:48.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:47 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-17T17:33:48.167 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:48.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:48.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:48.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:48.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:48.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.440 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:48.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:48.542 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:48.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:48.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:48.688 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:48.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:48.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:48.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:48.914 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:48.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:49.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:49.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:49.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:49.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:48 smithi152 ceph-mon[93456]: pgmap v1175: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 66692/320334 objects degraded (20.820%); 33509/320334 objects misplaced (10.461%); 5.8 MiB/s, 92 objects/s recovering 2024-08-17T17:33:49.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:48 smithi142 ceph-mon[106729]: pgmap v1175: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 66692/320334 objects degraded (20.820%); 33509/320334 objects misplaced (10.461%); 5.8 MiB/s, 92 objects/s recovering 2024-08-17T17:33:49.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:48 smithi142 ceph-mon[103614]: pgmap v1175: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 66692/320334 objects degraded (20.820%); 33509/320334 objects misplaced (10.461%); 5.8 MiB/s, 92 objects/s recovering 2024-08-17T17:33:49.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:49.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:49.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:49.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:49.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:49.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:49.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:49.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:49.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:49.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:49.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:49.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:49.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.932 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:50.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:50.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:50.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:50.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:50.282 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:50.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:50.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:50.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:50.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:50.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:33:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:33:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:50.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:50.675 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:50.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:50.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:50.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:50.865 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:33:50.865 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:33:50.865 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-08-17T17:33:50.866 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:33:50.866 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:33:50.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-08-17T17:33:50.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:50.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:51.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:51.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:51.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:51.177 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:51.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:51.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:51.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:51.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:51.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:51.430 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:51.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:51.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:51 smithi152 ceph-mon[93456]: pgmap v1176: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 66692/320334 objects degraded (20.820%); 33509/320334 objects misplaced (10.461%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T17:33:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 66692/320334 objects degraded (20.820%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[106729]: pgmap v1176: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 66692/320334 objects degraded (20.820%); 33509/320334 objects misplaced (10.461%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 66692/320334 objects degraded (20.820%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[103614]: pgmap v1176: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 66692/320334 objects degraded (20.820%); 33509/320334 objects misplaced (10.461%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 66692/320334 objects degraded (20.820%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:33:51.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:51.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:51.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:51.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:51.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:51.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:51.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:51.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:51.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:51.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:51.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:52.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:52.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:33:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:33:51] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:33:52.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:52.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:52.279 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:52.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:52.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:52.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:52.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:52.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:52.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:52.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:52.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:52.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:52.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:52.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:53.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:53.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:53.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:53.151 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:53.151 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:53.151 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:52 smithi142 ceph-mon[103614]: pgmap v1177: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 66692/320334 objects degraded (20.820%); 32956/320334 objects misplaced (10.288%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:33:53.151 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:53.151 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:53.151 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:52 smithi142 ceph-mon[106729]: pgmap v1177: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 66692/320334 objects degraded (20.820%); 32956/320334 objects misplaced (10.288%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:33:53.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:53.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:53.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:53.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:33:53.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:52 smithi152 ceph-mon[93456]: pgmap v1177: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 66692/320334 objects degraded (20.820%); 32956/320334 objects misplaced (10.288%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:33:53.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:53.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:53.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:53.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:53.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:53.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:53.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.794 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:53.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:53.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:53.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:53.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:54.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:54.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:54.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:54.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:54.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:54.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:54.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:54.414 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:54.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:54.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:54.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:54.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:54.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:54.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:54.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:54.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:54.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:55.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:55.034 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:55.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:55.159 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:55.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:55.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:55.309 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:33:55.309 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:33:55.309 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:33:55.309 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:33:55.309 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:55.310 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:55.310 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:55.310 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:55.384 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:55 smithi152 ceph-mon[93456]: pgmap v1178: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 66692/320334 objects degraded (20.820%); 32956/320334 objects misplaced (10.288%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:33:55.384 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/397525993' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:55.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:55.398 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "" 2024-08-17T17:33:55.399 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:33:55.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:55.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:55.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:55 smithi142 ceph-mon[106729]: pgmap v1178: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 66692/320334 objects degraded (20.820%); 32956/320334 objects misplaced (10.288%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:33:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/397525993' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:55 smithi142 ceph-mon[103614]: pgmap v1178: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 66692/320334 objects degraded (20.820%); 32956/320334 objects misplaced (10.288%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:33:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/397525993' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:55.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:55.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:55.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:55.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:55.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:55.903 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:56.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:56.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:56.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:56.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:56.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:56.276 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:56.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:56.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:56.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:56.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:56.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:56.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:56.684 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:33:56.684 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:33:56.684 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:33:56.684 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:33:56.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:56.776 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:56.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:56.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:57.002 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:57.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:57.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:57.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:57.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:57.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:57.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:57.280 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:33:57.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:57.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:57.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:57.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:57.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:57.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:57 smithi152 ceph-mon[93456]: pgmap v1179: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 66692/320334 objects degraded (20.820%); 32660/320334 objects misplaced (10.196%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:33:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/842316878' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:57.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:57.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:57 smithi142 ceph-mon[103614]: pgmap v1179: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 66692/320334 objects degraded (20.820%); 32660/320334 objects misplaced (10.196%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:33:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/842316878' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:57 smithi142 ceph-mon[106729]: pgmap v1179: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 66692/320334 objects degraded (20.820%); 32660/320334 objects misplaced (10.196%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:33:57.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/842316878' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:57.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:57.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:57.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:57.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:57.830 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:33:57.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:57.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:57.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:58.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:58.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:58.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:58.160 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:58.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:58.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:58.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:58.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:58.497 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:58.520 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:58.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:58 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/842316878' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:58.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:58 smithi152 ceph-mon[93456]: osdmap e339: 8 total, 8 up, 6 in 2024-08-17T17:33:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:58 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/842316878' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:58 smithi142 ceph-mon[106729]: osdmap e339: 8 total, 8 up, 6 in 2024-08-17T17:33:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:58 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/842316878' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:58.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:58 smithi142 ceph-mon[103614]: osdmap e339: 8 total, 8 up, 6 in 2024-08-17T17:33:58.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:58.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:58.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:58.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:58.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:58.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:58.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:59.024 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:59.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:59.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:59.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:59.274 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:59.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:59.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:59.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:33:59.528 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:33:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:33:59 smithi152 ceph-mon[93456]: pgmap v1181: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320334 objects degraded (20.820%); 32395/320334 objects misplaced (10.113%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:33:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:33:59 smithi142 ceph-mon[106729]: pgmap v1181: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320334 objects degraded (20.820%); 32395/320334 objects misplaced (10.113%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:33:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:33:59 smithi142 ceph-mon[103614]: pgmap v1181: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320334 objects degraded (20.820%); 32395/320334 objects misplaced (10.113%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:33:59.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:59.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:33:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:59.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:59.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:59.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:59.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:59.901 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:33:59.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:00.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:00.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.274 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:00.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:34:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:34:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:00.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.644 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:34:00.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:00.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:00.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:01.018 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:01.093 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:34:01.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:34:01.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:01.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:01.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:01.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:01.266 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:01.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:01.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:01.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-mon[103614]: pgmap v1182: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320334 objects degraded (20.820%); 32395/320334 objects misplaced (10.113%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:34:01.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:01.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-mon[106729]: pgmap v1182: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320334 objects degraded (20.820%); 32395/320334 objects misplaced (10.113%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:34:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:01.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:01.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:01.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:01.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:01.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:01 smithi152 ceph-mon[93456]: pgmap v1182: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320334 objects degraded (20.820%); 32395/320334 objects misplaced (10.113%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:34:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:01.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:01.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:01.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:01.741 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:01.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:01.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:01.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:34:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:34:01] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.51.0" 2024-08-17T17:34:01.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:01.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:01.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:02.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:02.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:02.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:02.136 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:02.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:02.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:02.260 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:02.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:02.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:02.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:02.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:02.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:02.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:02.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:02 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 66692/320319 objects degraded (20.820%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:34:02.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:02 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 66692/320319 objects degraded (20.820%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:34:02.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:02.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:02.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:02.787 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:34:02.787 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:34:02.787 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:34:02.787 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:34:02.787 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:34:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:02 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 66692/320319 objects degraded (20.820%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:34:02.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:02.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:02.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:02.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:03.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.107 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:03.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.231 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:03.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:03.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.500 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:03.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:03.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:03 smithi142 ceph-mon[106729]: pgmap v1183: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31820/320319 objects misplaced (9.934%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:34:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:03 smithi142 ceph-mon[103614]: pgmap v1183: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31820/320319 objects misplaced (9.934%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:34:03.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:03.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:03.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.748 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:03.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:03 smithi152 ceph-mon[93456]: pgmap v1183: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31820/320319 objects misplaced (9.934%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:34:03.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:03.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:03.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:04.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:04.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:04.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:04.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:04.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:04.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:04.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:04.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:04.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:04.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:04.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:04.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:04.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:04.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:04.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:04.997 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:04.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:05.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:05.122 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:05.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:05.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:05.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:05.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:05.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:05.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:05.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:05 smithi142 ceph-mon[106729]: pgmap v1184: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31820/320319 objects misplaced (9.934%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:34:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:05 smithi142 ceph-mon[103614]: pgmap v1184: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31820/320319 objects misplaced (9.934%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:34:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:05.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:05.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:05 smithi152 ceph-mon[93456]: pgmap v1184: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31820/320319 objects misplaced (9.934%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:34:05.848 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:05.869 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:05.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:05.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:06.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:06.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:06.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:06.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:06.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:06.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:06.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:06.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:06.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:06.503 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:06.503 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:06.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:34:06.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:06.617 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:06.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:06.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:06.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:06.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:06.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:06.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:06.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:06.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:07.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:07.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:07.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:07.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:07.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:07.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:07.376 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:07 smithi142 ceph-mon[103614]: pgmap v1185: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31491/320319 objects misplaced (9.831%); 7.3 MiB/s, 116 objects/s recovering 2024-08-17T17:34:07.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:07.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:07.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:07.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:07.670 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:07 smithi142 ceph-mon[106729]: pgmap v1185: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31491/320319 objects misplaced (9.831%); 7.3 MiB/s, 116 objects/s recovering 2024-08-17T17:34:07.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:07.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:07 smithi152 ceph-mon[93456]: pgmap v1185: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31491/320319 objects misplaced (9.831%); 7.3 MiB/s, 116 objects/s recovering 2024-08-17T17:34:07.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:07.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:07.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:07.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:07.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:08.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:08.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:08.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:08.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:08.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:08.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:08.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:08.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:08.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:08.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:08.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:08.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:08.730 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:08.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:08.855 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:08.941 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:08.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:08.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:09.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:09.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:09.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:09.227 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:09.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:09.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:09.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:09.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:09.434 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:09 smithi142 ceph-mon[106729]: pgmap v1186: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31187/320319 objects misplaced (9.736%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T17:34:09.435 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:09 smithi142 ceph-mon[103614]: pgmap v1186: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31187/320319 objects misplaced (9.736%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T17:34:09.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:09.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:09.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:09.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:09.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:09.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:09.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:09 smithi152 ceph-mon[93456]: pgmap v1186: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31187/320319 objects misplaced (9.736%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T17:34:09.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:09.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:09.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:09.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:09.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:10.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:10.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:10.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:10.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:10.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:10.349 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:10.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:10.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:10.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:10.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:10.481 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:34:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:34:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:10.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:10.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:10.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:10.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:10.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:10.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:10.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:10.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:10.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:10.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:10.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:11.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:11.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:11.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:11.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:11.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:11.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:11.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:11.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:11.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:11.382 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:34:11.383 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:34:11.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:11.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:11.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:11.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:11 smithi152 ceph-mon[93456]: pgmap v1187: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31187/320319 objects misplaced (9.736%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:34:11.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/137093194' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:34:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:11 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:34:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1325718598' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:11 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:11.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:11.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[106729]: pgmap v1187: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31187/320319 objects misplaced (9.736%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:34:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/137093194' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:34:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:34:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1325718598' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[103614]: pgmap v1187: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 66692/320319 objects degraded (20.820%); 31187/320319 objects misplaced (9.736%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:34:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/137093194' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:34:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:34:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1325718598' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:11.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:11.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:11.964 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:12.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:12.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:34:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:34:11] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.51.0" 2024-08-17T17:34:12.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:12.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:12.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:12.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:12.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:12.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:34:12.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:12.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:12.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:12.644 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:12.644 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:34:12.644 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:12.644 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[106729]: osdmap e340: 8 total, 8 up, 6 in 2024-08-17T17:34:12.644 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:12.644 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:12.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:12.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:34:12.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:12.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[103614]: osdmap e340: 8 total, 8 up, 6 in 2024-08-17T17:34:12.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:12.645 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:12.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:12.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:12.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:12 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:34:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:12 smithi152 ceph-mon[93456]: osdmap e340: 8 total, 8 up, 6 in 2024-08-17T17:34:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:12.864 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:34:12.864 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:12.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:12.882 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:34:12.882 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:12.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:12.899 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:34:12.899 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:34:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:12.916 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:34:12.917 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:13.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:13.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:13.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:13.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:13.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:13.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:13.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:13.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:13.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:13.522 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:13.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:13.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:13.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:13.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:13.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:13.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:13 smithi152 ceph-mon[93456]: pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320319 objects degraded (20.820%); 30537/320319 objects misplaced (9.533%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T17:34:13.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:13.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:13 smithi142 ceph-mon[106729]: pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320319 objects degraded (20.820%); 30537/320319 objects misplaced (9.533%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T17:34:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:13 smithi142 ceph-mon[103614]: pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 remapped+peering, 1 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 66692/320319 objects degraded (20.820%); 30537/320319 objects misplaced (9.533%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T17:34:13.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:13.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:14.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:14.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:14.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:14.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:14.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:14.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:14.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:14.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:14.516 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:14.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:14 smithi142 ceph-mon[106729]: pgmap v1190: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 44542/319800 objects misplaced (13.928%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:34:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:14 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72601/319800 objects degraded (22.702%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T17:34:14.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:14 smithi142 ceph-mon[103614]: pgmap v1190: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 44542/319800 objects misplaced (13.928%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:34:14.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:14 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72601/319800 objects degraded (22.702%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T17:34:14.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:14.741 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:14.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:14 smithi152 ceph-mon[93456]: pgmap v1190: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 44542/319800 objects misplaced (13.928%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:34:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:14 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72601/319800 objects degraded (22.702%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T17:34:14.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:14.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:14.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:15.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:15.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:15.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:15.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:15.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:15.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:15.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:15.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:15.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:15.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:15.505 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:15.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:15.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:15.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:15.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:15.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:15.752 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:15.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:15.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:15.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:15.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:15.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:15.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:16.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:16.101 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:16.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:16.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:16.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:16.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:16.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:16.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:16.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:16.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:16.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:16.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:16.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:16.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:16.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:16.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:16.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:16.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:16.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:16.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:17.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:17.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:17.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:17.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:17.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:17.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:17.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:17.365 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:17.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:17.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:17.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:17 smithi152 ceph-mon[93456]: pgmap v1191: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 44189/319800 objects misplaced (13.818%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:34:17.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:17.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:17.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:17 smithi142 ceph-mon[106729]: pgmap v1191: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 44189/319800 objects misplaced (13.818%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:34:17.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:17 smithi142 ceph-mon[103614]: pgmap v1191: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 44189/319800 objects misplaced (13.818%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:34:17.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:17.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:17.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:17.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:17.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:17.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:17.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:18.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:18.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:18.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:18.234 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:18.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:18.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:18.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:18.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:18.482 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:18.584 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:18.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:18.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:18.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:18.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:18.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:18.856 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:18.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:18.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:19.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:19.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:19.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:19.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:19.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179059 10.16 10:6ca643db:::benchmark_data_smithi142_192453_object53311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.874961+0000", 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.36741487899997, 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1128011550000001, 2024-08-17T17:34:19.246 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179059 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874961+0000", 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874961+0000", 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874962+0000", 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.466e-06 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874968+0000", 2024-08-17T17:34:19.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7629999999999997e-06 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874981+0000", 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2499e-05 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.874988+0000", 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.875063+0000", 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5281e-05 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.875075+0000", 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1722999999999999e-05 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.977731+0000", 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.102656842 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.977994+0000", 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026225899999999998 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.978306+0000", 2024-08-17T17:34:19.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00031236099999999999 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980318+0000", 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002012167 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980412+0000", 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3225000000000003e-05 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987210+0000", 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067988019999999996 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987324+0000", 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114079 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987762+0000", 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000437792 2024-08-17T17:34:19.253 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179060 10.6 10:6267ad6a:::benchmark_data_smithi142_192453_object53312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.877493+0000", 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.36488317999999, 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0776005580000001, 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.254 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179060 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877493+0000", 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877493+0000", 2024-08-17T17:34:19.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877495+0000", 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7379999999999999e-06 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877507+0000", 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1565e-05 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877531+0000", 2024-08-17T17:34:19.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4179999999999999e-05 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877543+0000", 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1841000000000001e-05 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877631+0000", 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7466999999999995e-05 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.877643+0000", 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2224e-05 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.776800+0000", 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.89915759500000003 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.777008+0000", 2024-08-17T17:34:19.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020719900000000001 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.777249+0000", 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024190299999999999 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876915+0000", 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.099665980000000001 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876952+0000", 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6426e-05 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.954737+0000", 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.077785629999999994 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.954847+0000", 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010928400000000001 2024-08-17T17:34:19.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955093+0000", 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000246526 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179070 10.6 10:673852e0:::benchmark_data_smithi142_192453_object53322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.939158+0000", 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.30321833200003, 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.016300097, 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.262 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179070 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939158+0000", 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939158+0000", 2024-08-17T17:34:19.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939160+0000", 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.04e-06 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939165+0000", 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6280000000000002e-06 2024-08-17T17:34:19.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939180+0000", 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4820999999999999e-05 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939190+0000", 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9709999999999999e-06 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939262+0000", 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1489999999999995e-05 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.939275+0000", 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.385e-05 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779505+0000", 2024-08-17T17:34:19.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.84022981600000002 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779727+0000", 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022187800000000001 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.779969+0000", 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024200799999999999 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876965+0000", 2024-08-17T17:34:19.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.096995998 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.877076+0000", 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111371 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955159+0000", 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078082562999999994 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955254+0000", 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4983999999999997e-05 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955458+0000", 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020367900000000001 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179074 10.6 10:60355f2a:::benchmark_data_smithi142_192453_object53326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:22.959773+0000", 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.28260308100005, 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.99600560299999996, 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179074 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.271 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959773+0000", 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959773+0000", 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959775+0000", 2024-08-17T17:34:19.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7630000000000001e-06 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959780+0000", 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8270000000000003e-06 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959796+0000", 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.519e-05 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959804+0000", 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4489999999999999e-06 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959885+0000", 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0570999999999998e-05 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:22.959899+0000", 2024-08-17T17:34:19.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4100999999999999e-05 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.781785+0000", 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.82188591899999996 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.782008+0000", 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022341700000000001 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.782252+0000", 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024394299999999999 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.876976+0000", 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.094723940000000006 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.877172+0000", 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019610600000000001 2024-08-17T17:34:19.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955515+0000", 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078342785999999998 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955590+0000", 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5014999999999999e-05 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955778+0000", 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018857600000000001 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179093 10.16 10:69bfd028:::benchmark_data_smithi142_192453_object53345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:23.033659+0000", 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.20871652000005, 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95483545400000003, 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.278 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179093 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033659+0000", 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033659+0000", 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033661+0000", 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.547e-06 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.033667+0000", 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1029999999999998e-06 2024-08-17T17:34:19.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034263+0000", 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059557700000000002 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034271+0000", 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2549999999999997e-06 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034347+0000", 2024-08-17T17:34:19.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5828000000000007e-05 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.034354+0000", 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9639999999999997e-06 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980469+0000", 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.94611584100000001 2024-08-17T17:34:19.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980680+0000", 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00021035800000000001 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.980941+0000", 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026073300000000003 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987843+0000", 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0069019479999999998 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987877+0000", 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4063999999999998e-05 2024-08-17T17:34:19.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.987950+0000", 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3293000000000005e-05 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.988217+0000", 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00026665499999999997 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.988495+0000", 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00027828800000000002 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179098 10.16 10:680f5353:::benchmark_data_smithi142_192453_object53350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:23.042973+0000", 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.19940287400004, 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95446910100000004, 2024-08-17T17:34:19.286 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179098 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042973+0000", 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042973+0000", 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042975+0000", 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.823e-06 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042981+0000", 2024-08-17T17:34:19.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5740000000000001e-06 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.042997+0000", 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5534999999999999e-05 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.043008+0000", 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0831e-05 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.043072+0000", 2024-08-17T17:34:19.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3793000000000004e-05 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.043081+0000", 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9260000000000006e-06 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.982770+0000", 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93968903199999998 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.983144+0000", 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00037481999999999999 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.985041+0000", 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0018962899999999999 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.989001+0000", 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039599580000000004 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.991100+0000", 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002099007 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.997105+0000", 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060057089999999997 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.997189+0000", 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3978000000000001e-05 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.997442+0000", 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025282499999999998 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36309.0:179107 10.6 10:6057202c:::benchmark_data_smithi142_192453_object53359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:25:23.076805+0000", 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 536.16557042399995, 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.87953849799999995, 2024-08-17T17:34:19.293 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36309", 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/1467203670", 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 179107 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076805+0000", 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076805+0000", 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076807+0000", 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.139e-06 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076811+0000", 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0679999999999998e-06 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076822+0000", 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0886999999999999e-05 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076826+0000", 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7720000000000002e-06 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076884+0000", 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7207000000000003e-05 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.076893+0000", 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5610000000000004e-06 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.874948+0000", 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.79805484299999996 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.875101+0000", 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015309000000000001 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.875259+0000", 2024-08-17T17:34:19.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015764899999999999 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.877266+0000", 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020069229999999999 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.955941+0000", 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078675373000000007 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.956007+0000", 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5870999999999993e-05 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.956089+0000", 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1760000000000003e-05 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:25:23.956344+0000", 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025535500000000002 2024-08-17T17:34:19.300 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:29101 11.9 e287/283 11:91525119:::benchmark_data_smithi142_227468_object29100:head v 287'1828, mlcod=287'1825)", 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 415.15177830300001, 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26308380100000001, 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090598+0000", 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3300000000000001e-07 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090642+0000", 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3078999999999999e-05 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090653+0000", 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1972e-05 2024-08-17T17:34:19.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090661+0000", 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3549999999999999e-06 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090738+0000", 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7309000000000001e-05 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.090757+0000", 2024-08-17T17:34:19.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8936e-05 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092123+0000", 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001365478 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353681+0000", 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26155883899999999 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.305 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:29102 11.1 e287/283 11:8390c806:::benchmark_data_smithi142_227468_object29101:head v 287'1859, mlcod=287'1857)", 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.092007+0000", 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 415.15036916000003, 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26468164500000002, 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092007+0000", 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092007+0000", 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092008+0000", 2024-08-17T17:34:19.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4799999999999997e-07 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092206+0000", 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000198228 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353636+0000", 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26143067199999998 2024-08-17T17:34:19.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353654+0000", 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7646999999999999e-05 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353728+0000", 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3374000000000006e-05 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353753+0000", 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5216e-05 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.356643+0000", 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028901360000000002 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.356688+0000", 2024-08-17T17:34:19.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5524e-05 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:29103 11.9 e287/283 11:950c9362:::benchmark_data_smithi142_227468_object29102:head v 287'1829, mlcod=287'1825)", 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:27:24.092977+0000", 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 415.14939860700002, 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26484094000000002, 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.311 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092977+0000", 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092977+0000", 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.092978+0000", 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0359999999999999e-06 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.093003+0000", 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4321e-05 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353634+0000", 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.260631157 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353654+0000", 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9958000000000001e-05 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353739+0000", 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5712999999999997e-05 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.353754+0000", 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4806e-05 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357766+0000", 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040113789999999998 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:27:24.357818+0000", 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.257e-05 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.315 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:104295 11.2 e293/283 11:429583dc:::benchmark_data_smithi142_227468_object104294:head v 293'6496, mlcod=293'6494)", 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.263427+0000", 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 356.97894939299999, 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28800537100000001, 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263427+0000", 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263427+0000", 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263427+0000", 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5799999999999998e-07 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263449+0000", 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1591e-05 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533556+0000", 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.270107127 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533574+0000", 2024-08-17T17:34:19.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8289999999999999e-05 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533645+0000", 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0405000000000001e-05 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533670+0000", 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4947999999999999e-05 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.551374+0000", 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017704386999999999 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.551432+0000", 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7664999999999998e-05 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:104311 11.1 e293/283 11:8d2185cc:::benchmark_data_smithi142_227468_object104310:head v 293'6574, mlcod=293'6572)", 2024-08-17T17:34:19.320 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.277242+0000", 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 356.96513425799998, 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26855934199999998, 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277242+0000", 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277242+0000", 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277243+0000", 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5799999999999999e-06 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277287+0000", 2024-08-17T17:34:19.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3730999999999999e-05 2024-08-17T17:34:19.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533563+0000", 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25627586099999999 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533575+0000", 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1975e-05 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533814+0000", 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000239535 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.533835+0000", 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0373e-05 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.545745+0000", 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011910702 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.545801+0000", 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5584999999999997e-05 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.326 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236392 11.1 e314/283 11:80697271:::benchmark_data_smithi142_227468_object112973:head v 314'14844, mlcod=314'14841)", 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.700037+0000", 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 196.54233938100001, 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26028967400000003, 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700037+0000", 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700037+0000", 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700038+0000", 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.268e-06 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700045+0000", 2024-08-17T17:34:19.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700059+0000", 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3743e-05 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700067+0000", 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1669999999999999e-06 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700122+0000", 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4629999999999997e-05 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700142+0000", 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0307000000000001e-05 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.705124+0000", 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0049820059999999998 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960326+0000", 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25520210999999998 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236396 11.b e314/283 11:de9f86a9:::benchmark_data_smithi142_227468_object112977:head v 314'14927, mlcod=314'14925)", 2024-08-17T17:34:19.331 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.701912+0000", 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 196.540463894, 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25917337200000001, 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701912+0000", 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701912+0000", 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701913+0000", 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0499999999999999e-06 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701920+0000", 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0099999999999998e-06 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701932+0000", 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.242e-05 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701940+0000", 2024-08-17T17:34:19.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7889999999999995e-06 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701996+0000", 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5615999999999997e-05 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.702015+0000", 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9304999999999999e-05 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960459+0000", 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25844405799999998 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.961085+0000", 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00062612400000000004 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236400 11.b e314/283 11:d7d22f27:::benchmark_data_smithi142_227468_object112981:head v 314'14928, mlcod=314'14925)", 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.707407+0000", 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 196.53496922900001, 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26579524300000001, 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707407+0000", 2024-08-17T17:34:19.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707407+0000", 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707408+0000", 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.519e-06 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707416+0000", 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6890000000000004e-06 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960286+0000", 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25287000500000001 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960303+0000", 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6773e-05 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.961120+0000", 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000817061 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.961134+0000", 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4035e-05 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973137+0000", 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012003315000000001 2024-08-17T17:34:19.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973202+0000", 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4845999999999995e-05 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236411 11.1 e314/283 11:86e88955:::benchmark_data_smithi142_227468_object112992:head v 314'14845, mlcod=314'14842)", 2024-08-17T17:34:19.341 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.716916+0000", 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 196.52545981599999, 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25703896100000001, 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716916+0000", 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716916+0000", 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716917+0000", 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.373e-06 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716929+0000", 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1445e-05 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960320+0000", 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.243391145 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960329+0000", 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0110000000000006e-06 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960521+0000", 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019235199999999999 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960546+0000", 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4553999999999999e-05 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973867+0000", 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013321058 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973955+0000", 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8022999999999994e-05 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:123775 12.c 12:321760c7:::benchmark_data_smithi142_261410_object5578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:28.712782+0000", 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 50.529593554000002, 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19839264200000001, 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.346 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 123775 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712782+0000", 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712782+0000", 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712784+0000", 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.968e-06 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712792+0000", 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1310000000000006e-06 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712813+0000", 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0407999999999998e-05 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712824+0000", 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0662000000000001e-05 2024-08-17T17:34:19.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.725136+0000", 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012312498 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.725258+0000", 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012247700000000001 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.814574+0000", 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.089315806999999997 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.814682+0000", 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010725 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.814822+0000", 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0001404 2024-08-17T17:34:19.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.820109+0000", 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052871250000000002 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.850287+0000", 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030177679999999998 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.906624+0000", 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.056337320000000003 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.906667+0000", 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2571999999999998e-05 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.911175+0000", 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045083440000000001 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.353 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:123795 12.c 12:340d23f4:::benchmark_data_smithi142_261410_object5598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:28.752468+0000", 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 50.489907522000003, 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.150072178, 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.354 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 123795 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752468+0000", 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752468+0000", 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752471+0000", 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6000000000000001e-06 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752479+0000", 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2390000000000002e-06 2024-08-17T17:34:19.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752502+0000", 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3040999999999998e-05 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752512+0000", 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4609999999999997e-06 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.768199+0000", 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015687741000000002 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.768370+0000", 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170428 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.768528+0000", 2024-08-17T17:34:19.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015771700000000001 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.820044+0000", 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.051516861999999997 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.820077+0000", 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2141000000000003e-05 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.897089+0000", 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.077012238999999996 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.897143+0000", 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4277000000000002e-05 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.902541+0000", 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053974319999999998 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:125894 12.c 12:3c54b205:::benchmark_data_smithi142_261410_object7697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:33.095034+0000", 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 46.147342033999998, 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.137142285, 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.361 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 125894 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095034+0000", 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095034+0000", 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095036+0000", 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.796e-06 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095041+0000", 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.711e-06 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095056+0000", 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4293e-05 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095064+0000", 2024-08-17T17:34:19.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5709999999999998e-06 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.103028+0000", 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079640660000000005 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.103167+0000", 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013898799999999999 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.174613+0000", 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.071445896999999994 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.174706+0000", 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2662000000000002e-05 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:34:19.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.174916+0000", 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000210476 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.203685+0000", 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.028769045 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.203942+0000", 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025621599999999997 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.228054+0000", 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024112858000000001 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.228154+0000", 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9561000000000004e-05 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.232176+0000", 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040221450000000004 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.368 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:129215 12.c 12:35a663db:::benchmark_data_smithi142_261410_object11018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:39.247687+0000", 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 39.994688539000002, 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.14726345299999999, 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 129215 2024-08-17T17:34:19.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247687+0000", 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247687+0000", 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247690+0000", 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4480000000000001e-06 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247698+0000", 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9279999999999993e-06 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247721+0000", 2024-08-17T17:34:19.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3683e-05 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247731+0000", 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8519999999999999e-06 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.266425+0000", 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018693624999999998 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.266559+0000", 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013370499999999999 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.344578+0000", 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078019644999999999 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.344892+0000", 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00031342499999999999 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.345349+0000", 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045739100000000003 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.357632+0000", 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01228248 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.388347+0000", 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030715884999999998 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.388424+0000", 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6260000000000005e-05 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.388481+0000", 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7651000000000001e-05 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:19.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.394951+0000", 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064694749999999997 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:19.376 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:34:19.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:19.377 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:19 smithi142 ceph-mon[106729]: pgmap v1192: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 43865/319800 objects misplaced (13.716%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:34:19.377 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:19 smithi142 ceph-mon[103614]: pgmap v1192: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 43865/319800 objects misplaced (13.716%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:34:19.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:19.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:19.524 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:19.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:19.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:19 smithi152 ceph-mon[93456]: pgmap v1192: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 43865/319800 objects misplaced (13.716%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:34:19.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:19.648 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:19.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:19.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:19.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:19.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:19.895 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:19.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:19.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:20.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:20.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:20.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:20.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:20.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:20.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:20.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:20.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:20.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:20.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:34:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:34:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:20.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:20.640 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:20.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:34:20.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:20.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:20.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:20.889 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:20.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:20.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:21.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:21.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:21.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:34:21.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:21.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:21.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:21.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:21 smithi142 ceph-mon[103614]: pgmap v1193: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 43865/319800 objects misplaced (13.716%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:34:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:21 smithi142 ceph-mon[106729]: pgmap v1193: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 43865/319800 objects misplaced (13.716%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:34:21.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:21.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:21.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:21.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:21 smithi152 ceph-mon[93456]: pgmap v1193: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 17 op/s; 72601/319800 objects degraded (22.702%); 43865/319800 objects misplaced (13.716%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:34:21.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:21.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:21.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:21.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:21.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:34:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:34:21] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.51.0" 2024-08-17T17:34:21.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.955 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:21.968 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:21.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:22.021 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:22.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:22.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:22.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:22.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:22.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:22.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:22.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:22.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:22 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72601/319800 objects degraded (22.702%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:22.586 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:22 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72601/319800 objects degraded (22.702%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:22.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:22.644 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:22.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:22.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:22 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72601/319800 objects degraded (22.702%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:22.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:22.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:22.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:23.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:23.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:23.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:23.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:23.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:23.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:23.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:23.314 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:23.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:23.388 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:23.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:23.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:23.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:34:23.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:23.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:23 smithi142 ceph-mon[106729]: pgmap v1194: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 15 op/s; 72601/319800 objects degraded (22.702%); 43135/319800 objects misplaced (13.488%); 9.9 MiB/s, 159 objects/s recovering 2024-08-17T17:34:23.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:23 smithi142 ceph-mon[103614]: pgmap v1194: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 15 op/s; 72601/319800 objects degraded (22.702%); 43135/319800 objects misplaced (13.488%); 9.9 MiB/s, 159 objects/s recovering 2024-08-17T17:34:23.645 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:23.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:23.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:23 smithi152 ceph-mon[93456]: pgmap v1194: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 15 op/s; 72601/319800 objects degraded (22.702%); 43135/319800 objects misplaced (13.488%); 9.9 MiB/s, 159 objects/s recovering 2024-08-17T17:34:23.871 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:23.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:23.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.020 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:24.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:24.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:24.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:24.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:24.393 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:24.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:24.644 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:24.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:24.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:24.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:24.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:24.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:25.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:25.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:25.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:25.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:25.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:25.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:25.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:25.394 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:25.434 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:34:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:34:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:34:25.434 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:34:25.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:25.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:25.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:25 smithi142 ceph-mon[106729]: pgmap v1195: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 14 op/s; 72601/319800 objects degraded (22.702%); 43135/319800 objects misplaced (13.488%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:34:25.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:25 smithi142 ceph-mon[103614]: pgmap v1195: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 14 op/s; 72601/319800 objects degraded (22.702%); 43135/319800 objects misplaced (13.488%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:34:25.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:25.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:25.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:25 smithi152 ceph-mon[93456]: pgmap v1195: 89 pgs: 1 activating+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 14 op/s; 72601/319800 objects degraded (22.702%); 43135/319800 objects misplaced (13.488%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:34:25.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:25.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:25.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:26.018 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:26.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:26.142 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:26.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:26.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:26.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:26.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:26.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:26.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:26.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:26.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:26.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:26.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:26.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:26.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:26.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:26.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:26.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:27.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:27.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:26 smithi152 ceph-mon[93456]: pgmap v1196: 89 pgs: 1 active+degraded+remapped+laggy, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 72601/319770 objects degraded (22.704%); 42758/319770 objects misplaced (13.371%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:34:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:27.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:26 smithi142 ceph-mon[106729]: pgmap v1196: 89 pgs: 1 active+degraded+remapped+laggy, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 72601/319770 objects degraded (22.704%); 42758/319770 objects misplaced (13.371%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:34:27.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:27.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:26 smithi142 ceph-mon[103614]: pgmap v1196: 89 pgs: 1 active+degraded+remapped+laggy, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 72601/319770 objects degraded (22.704%); 42758/319770 objects misplaced (13.371%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:34:27.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:27.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:27.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:27.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:27.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:27.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:27.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:27.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:27.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:27.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:27.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:27.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:27.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:27.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:27.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:27.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:27.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:28.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:28.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:28.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:28.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:28.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:28.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:28.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:28.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:28.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:28.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:28.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:28.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:28.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:28.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:28 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72596/319566 objects degraded (22.717%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:28 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72596/319566 objects degraded (22.717%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:28.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:28.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:28.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:28.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:28 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72596/319566 objects degraded (22.717%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:28.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:28.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:29.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:29.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:29.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:29.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:29.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:29.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:29.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:29.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:29.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:29.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:29.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:29 smithi142 ceph-mon[103614]: pgmap v1197: 89 pgs: 1 active+degraded+remapped+laggy, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 72596/319566 objects degraded (22.717%); 42386/319566 objects misplaced (13.264%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:34:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:29 smithi142 ceph-mon[106729]: pgmap v1197: 89 pgs: 1 active+degraded+remapped+laggy, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 72596/319566 objects degraded (22.717%); 42386/319566 objects misplaced (13.264%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:34:29.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:29.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:29 smithi152 ceph-mon[93456]: pgmap v1197: 89 pgs: 1 active+degraded+remapped+laggy, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 72596/319566 objects degraded (22.717%); 42386/319566 objects misplaced (13.264%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T17:34:29.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:29.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:29.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:29.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:29.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:29.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:30.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:30.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:30.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:30.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:30.252 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:30.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:30.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.378 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:30.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:30.479 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:30.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:34:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:34:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:30.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:30.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:30.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:30.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:30.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:30.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:30.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:30.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:31.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:31.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:31.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:31.226 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:31.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:31.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:31.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:31.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:31.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:34:31.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:31.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:31 smithi152 ceph-mon[93456]: pgmap v1198: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 9 op/s; 72575/319467 objects degraded (22.718%); 42348/319467 objects misplaced (13.256%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T17:34:31.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:31 smithi142 ceph-mon[103614]: pgmap v1198: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 9 op/s; 72575/319467 objects degraded (22.718%); 42348/319467 objects misplaced (13.256%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T17:34:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:31 smithi142 ceph-mon[106729]: pgmap v1198: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 9 op/s; 72575/319467 objects degraded (22.718%); 42348/319467 objects misplaced (13.256%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T17:34:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:31.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:31.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:31.746 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:31.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:31.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:31.912 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:31.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:31.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:32.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:32.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:32.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:34:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:34:31] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-08-17T17:34:32.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:32.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:32.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:32.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:32.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:32.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:32.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:32.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:32.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:32.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:32.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:32.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:32.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:32.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:32.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:32.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:33.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:33.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:33.103 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:33.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:33.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:33.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:33.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:33.377 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:33.417 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:33 smithi142 ceph-mon[106729]: pgmap v1199: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 72564/319089 objects degraded (22.741%); 41530/319089 objects misplaced (13.015%); 12 MiB/s, 188 objects/s recovering 2024-08-17T17:34:33.418 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:33 smithi142 ceph-mon[103614]: pgmap v1199: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 72564/319089 objects degraded (22.741%); 41530/319089 objects misplaced (13.015%); 12 MiB/s, 188 objects/s recovering 2024-08-17T17:34:33.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:33.501 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:33.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:33.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:33.625 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:33.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:33.749 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:33.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:33 smithi152 ceph-mon[93456]: pgmap v1199: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 72564/319089 objects degraded (22.741%); 41530/319089 objects misplaced (13.015%); 12 MiB/s, 188 objects/s recovering 2024-08-17T17:34:33.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:33.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:33.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:33.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:33.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:34.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:34.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:34.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:34.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:34.339 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:34:34.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:34.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:34.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:34.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:34.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3379569070' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:34.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3379569070' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:34.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:34.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:34.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:34.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3379569070' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:34.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:34.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:34.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:35.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:35.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:35.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:35.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:35.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:35.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:35.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:35.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:35.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:35.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:35.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:35.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:35.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[106729]: pgmap v1200: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 72564/318978 objects degraded (22.749%); 41530/318978 objects misplaced (13.020%); 8.0 MiB/s, 127 objects/s recovering 2024-08-17T17:34:35.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72564/318978 objects degraded (22.749%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:35.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:35.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3379569070' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:35.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[106729]: osdmap e341: 8 total, 8 up, 6 in 2024-08-17T17:34:35.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[103614]: pgmap v1200: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 72564/318978 objects degraded (22.749%); 41530/318978 objects misplaced (13.020%); 8.0 MiB/s, 127 objects/s recovering 2024-08-17T17:34:35.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72564/318978 objects degraded (22.749%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:35.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:35.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3379569070' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:35.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:35 smithi142 ceph-mon[103614]: osdmap e341: 8 total, 8 up, 6 in 2024-08-17T17:34:35.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:35.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:35.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:35.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:35.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:35 smithi152 ceph-mon[93456]: pgmap v1200: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 72564/318978 objects degraded (22.749%); 41530/318978 objects misplaced (13.020%); 8.0 MiB/s, 127 objects/s recovering 2024-08-17T17:34:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:35 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72564/318978 objects degraded (22.749%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:35 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3379569070' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:35 smithi152 ceph-mon[93456]: osdmap e341: 8 total, 8 up, 6 in 2024-08-17T17:34:35.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:35.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:35.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:35.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:35.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:36.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:36.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:36.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:36.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:36.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:36.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:36.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:36.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:36.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:36.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:36.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:36.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:36.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:36.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:36.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:36.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:36.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:37.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:37.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:37.244 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:37.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:37.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:37.368 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:37.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:37.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:37.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:37 smithi142 ceph-mon[106729]: pgmap v1202: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 33 op/s; 72537/318753 objects degraded (22.756%); 41076/318753 objects misplaced (12.886%); 10 MiB/s, 161 objects/s recovering 2024-08-17T17:34:37.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:37 smithi142 ceph-mon[103614]: pgmap v1202: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 33 op/s; 72537/318753 objects degraded (22.756%); 41076/318753 objects misplaced (12.886%); 10 MiB/s, 161 objects/s recovering 2024-08-17T17:34:37.720 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:37.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:37.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:37 smithi152 ceph-mon[93456]: pgmap v1202: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 33 op/s; 72537/318753 objects degraded (22.756%); 41076/318753 objects misplaced (12.886%); 10 MiB/s, 161 objects/s recovering 2024-08-17T17:34:37.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:37.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:37.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:38.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:38.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:38.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:38.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:38.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:38.362 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:38.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:38.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:38.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:38.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:38.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:38.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:38.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:38.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:38.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:38.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:38.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:38.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:38.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:39.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:39.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:39.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:39.243 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:39.272 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:39.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:39.368 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:39.449 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:39 smithi142 ceph-mon[106729]: pgmap v1203: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 72527/318363 objects degraded (22.781%); 40644/318363 objects misplaced (12.767%); 10 MiB/s, 166 objects/s recovering 2024-08-17T17:34:39.449 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:39 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:39.449 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:39 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:39.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:39 smithi142 ceph-mon[103614]: pgmap v1203: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 72527/318363 objects degraded (22.781%); 40644/318363 objects misplaced (12.767%); 10 MiB/s, 166 objects/s recovering 2024-08-17T17:34:39.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:39 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:39.449 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:39 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:39.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:39.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:39.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:39.557 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:34:39.558 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:39.579 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:34:39.579 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:39.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:39.601 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:34:39.601 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:39.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:39.629 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:34:39.629 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:39.655 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:34:39.655 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:39.675 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:34:39.675 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:39.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:34:39.695 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:34:39.716 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:34:39.716 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:39.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:39.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:39 smithi152 ceph-mon[93456]: pgmap v1203: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 72527/318363 objects degraded (22.781%); 40644/318363 objects misplaced (12.767%); 10 MiB/s, 166 objects/s recovering 2024-08-17T17:34:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:39 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:39 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:39.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:39.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:39.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:39.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:40.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:40.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:40.103 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:40.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:40.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:40.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:40.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:40.349 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:34:40.356 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:40.382 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:40.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:40.507 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:34:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:34:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1868896621' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:34:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2679100086' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:40.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:34:40.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:34:40.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:40.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1868896621' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:34:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2679100086' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:34:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:34:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1868896621' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:34:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2679100086' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:34:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:34:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:40.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:40.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:40.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:34:40.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:40.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:40.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:41.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:41.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:41.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:41.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:41.251 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:41.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:41.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:41.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:41.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[106729]: pgmap v1204: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 72497/318243 objects degraded (22.780%); 40616/318243 objects misplaced (12.763%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1868896621' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[106729]: osdmap e342: 8 total, 8 up, 6 in 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72527/318363 objects degraded (22.781%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[103614]: pgmap v1204: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 72497/318243 objects degraded (22.780%); 40616/318243 objects misplaced (12.763%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1868896621' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[103614]: osdmap e342: 8 total, 8 up, 6 in 2024-08-17T17:34:41.507 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72527/318363 objects degraded (22.781%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:41.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:41.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:41.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:41.750 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:41.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:41 smithi152 ceph-mon[93456]: pgmap v1204: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 72497/318243 objects degraded (22.780%); 40616/318243 objects misplaced (12.763%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:34:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1868896621' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:34:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:41 smithi152 ceph-mon[93456]: osdmap e342: 8 total, 8 up, 6 in 2024-08-17T17:34:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72527/318363 objects degraded (22.781%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:41.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:34:41.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:41.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:41.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:41.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:42.002 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:42.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:34:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:34:41] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.51.0" 2024-08-17T17:34:42.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:42.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:42.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:42.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:42.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:42.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:42.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:42.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:42.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:42.525 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:42.525 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:42.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:42.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:42.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:42.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:42.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:42.777 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:42.788 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:34:42.788 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:34:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:42.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:42.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:42.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:43.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:43.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:43.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:43.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:43.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:43 smithi142 ceph-mon[106729]: pgmap v1206: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 72485/317820 objects degraded (22.807%); 39670/317820 objects misplaced (12.482%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:34:43.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:43 smithi142 ceph-mon[103614]: pgmap v1206: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 72485/317820 objects degraded (22.807%); 39670/317820 objects misplaced (12.482%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:34:43.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:43.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:43.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:43.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:43.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:34:43.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:43.633 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:43.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:43.666 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:43.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:43.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:43 smithi152 ceph-mon[93456]: pgmap v1206: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 72485/317820 objects degraded (22.807%); 39670/317820 objects misplaced (12.482%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:34:43.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:43.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:43.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:44.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:44.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:44.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:44.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:44.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:34:44.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:44.388 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:44.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:44.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:44.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:44.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:44.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:44.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:44.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:44.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:44.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:44.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:44.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:45.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:45.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:45.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:45.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:45.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:45.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:45.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:45.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:45.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:45.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:45.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:45.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:45.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:45.618 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:45 smithi142 ceph-mon[106729]: pgmap v1207: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 43 op/s; 72485/317679 objects degraded (22.817%); 39670/317679 objects misplaced (12.487%); 11 MiB/s, 182 objects/s recovering 2024-08-17T17:34:45.619 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:45 smithi142 ceph-mon[103614]: pgmap v1207: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 43 op/s; 72485/317679 objects degraded (22.817%); 39670/317679 objects misplaced (12.487%); 11 MiB/s, 182 objects/s recovering 2024-08-17T17:34:45.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:45.687 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:45.688 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:45.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:45.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:45 smithi152 ceph-mon[93456]: pgmap v1207: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 43 op/s; 72485/317679 objects degraded (22.817%); 39670/317679 objects misplaced (12.487%); 11 MiB/s, 182 objects/s recovering 2024-08-17T17:34:45.862 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:45.885 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:45.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:45.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:46.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:46.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:46.135 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:46.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:46.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:46.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:46.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:46.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:46.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:46.483 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72485/317679 objects degraded (22.817%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:46.484 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72485/317679 objects degraded (22.817%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:46.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:46.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:46.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:46.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:46.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:46.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72485/317679 objects degraded (22.817%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:46.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:46.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:46.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:47.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:47.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:47.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:47.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:47.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:47.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:47.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:47.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:47.374 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:34:47.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:47.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:47.512 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:47.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:47 smithi142 ceph-mon[106729]: pgmap v1208: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 72443/317394 objects degraded (22.824%); 39100/317394 objects misplaced (12.319%); 12 MiB/s, 192 objects/s recovering 2024-08-17T17:34:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3571382713' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:47 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:47 smithi142 ceph-mon[103614]: pgmap v1208: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 72443/317394 objects degraded (22.824%); 39100/317394 objects misplaced (12.319%); 12 MiB/s, 192 objects/s recovering 2024-08-17T17:34:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3571382713' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:47.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:47 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:47.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:47.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:47.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:47 smithi152 ceph-mon[93456]: pgmap v1208: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 72443/317394 objects degraded (22.824%); 39100/317394 objects misplaced (12.319%); 12 MiB/s, 192 objects/s recovering 2024-08-17T17:34:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3571382713' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:47 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:47.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:47.885 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:47.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:48.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:48.135 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:48.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:48.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:48.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:48.381 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:34:48.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:48.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:48.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:48.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:48.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:48.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:48.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:48.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:48.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:48 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:34:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:48 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:48 smithi152 ceph-mon[93456]: osdmap e343: 8 total, 8 up, 6 in 2024-08-17T17:34:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3280382158' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:34:48.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:48 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:34:48.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:34:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[106729]: osdmap e343: 8 total, 8 up, 6 in 2024-08-17T17:34:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3280382158' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:34:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:34:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:34:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[103614]: osdmap e343: 8 total, 8 up, 6 in 2024-08-17T17:34:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3280382158' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:34:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:48 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:34:48.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:48.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:49.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:49.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:49.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:49.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:49.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:49.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:49.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:49.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:49.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:49.617 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:49.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:49.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:49.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:49 smithi152 ceph-mon[93456]: pgmap v1210: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 55 op/s; 72428/316902 objects degraded (22.855%); 38614/316902 objects misplaced (12.185%); 15 MiB/s, 244 objects/s recovering 2024-08-17T17:34:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:34:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:49 smithi152 ceph-mon[93456]: osdmap e344: 8 total, 8 up, 6 in 2024-08-17T17:34:49.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:49 smithi142 ceph-mon[106729]: pgmap v1210: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 55 op/s; 72428/316902 objects degraded (22.855%); 38614/316902 objects misplaced (12.185%); 15 MiB/s, 244 objects/s recovering 2024-08-17T17:34:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:34:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:49 smithi142 ceph-mon[106729]: osdmap e344: 8 total, 8 up, 6 in 2024-08-17T17:34:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:49 smithi142 ceph-mon[103614]: pgmap v1210: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 55 op/s; 72428/316902 objects degraded (22.855%); 38614/316902 objects misplaced (12.185%); 15 MiB/s, 244 objects/s recovering 2024-08-17T17:34:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:34:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:49 smithi142 ceph-mon[103614]: osdmap e344: 8 total, 8 up, 6 in 2024-08-17T17:34:49.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:49.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:49.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:50.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:50.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:50.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:50.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:50.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:50.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:50.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:50.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:50.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:50.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:50.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:50.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:34:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:34:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:50.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:50.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:50.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:50.759 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:50.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:50.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:50.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:50.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:50.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:34:50.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:51.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:51.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:51.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:51.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:51.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:51.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:51.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:51.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:34:51.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:51.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:51.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:51.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:51.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:51.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:51.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:51 smithi142 ceph-mon[106729]: pgmap v1212: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 43 op/s; 72395/316770 objects degraded (22.854%); 38587/316770 objects misplaced (12.181%); 8.4 MiB/s, 134 objects/s recovering 2024-08-17T17:34:51.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72395/316770 objects degraded (22.854%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:51.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:51 smithi142 ceph-mon[103614]: pgmap v1212: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 43 op/s; 72395/316770 objects degraded (22.854%); 38587/316770 objects misplaced (12.181%); 8.4 MiB/s, 134 objects/s recovering 2024-08-17T17:34:51.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72395/316770 objects degraded (22.854%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:51 smithi152 ceph-mon[93456]: pgmap v1212: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 43 op/s; 72395/316770 objects degraded (22.854%); 38587/316770 objects misplaced (12.181%); 8.4 MiB/s, 134 objects/s recovering 2024-08-17T17:34:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72395/316770 objects degraded (22.854%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:51.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:51.883 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:51.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:52.072 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:34:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:34:51] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.51.0" 2024-08-17T17:34:52.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:52.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:52.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:52.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:52.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:52.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:52.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:52.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:52.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:52.638 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:34:52.638 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:34:52.638 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:52.638 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:52.648 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:34:52.649 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:52.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.650 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:52.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:52.670 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:34:52.670 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:52.699 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:34:52.699 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:52.718 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:34:52.718 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:52.734 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:34:52.735 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:34:52.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:52.752 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:34:52.753 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:52.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:52.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:52.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:53.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:53.028 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:53.130 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:53.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:53.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:53.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:53.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:53.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:53.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:53.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:53.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:53.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:53.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:53.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:53.760 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:53.782 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:53.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:53 smithi152 ceph-mon[93456]: pgmap v1213: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 53 op/s; 72385/316398 objects degraded (22.878%); 37369/316398 objects misplaced (11.811%); 18 MiB/s, 283 objects/s recovering 2024-08-17T17:34:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1924686961' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:53 smithi142 ceph-mon[103614]: pgmap v1213: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 53 op/s; 72385/316398 objects degraded (22.878%); 37369/316398 objects misplaced (11.811%); 18 MiB/s, 283 objects/s recovering 2024-08-17T17:34:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1924686961' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:53 smithi142 ceph-mon[106729]: pgmap v1213: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 53 op/s; 72385/316398 objects degraded (22.878%); 37369/316398 objects misplaced (11.811%); 18 MiB/s, 283 objects/s recovering 2024-08-17T17:34:53.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1924686961' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:53.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:53.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:53.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:54.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:54.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:54.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:54.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:54.249 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:34:54.249 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:56985 12.4 e325/319 12:257f5782:::benchmark_data_smithi142_261410_object56984:head v 325'3469, mlcod=325'3467)", 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:32:16.009422+0000", 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 156.912931886, 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33730865399999999, 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009422+0000", 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009422+0000", 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.251 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009423+0000", 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2249999999999999e-06 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009466+0000", 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2206000000000002e-05 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009510+0000", 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3992999999999997e-05 2024-08-17T17:34:54.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009533+0000", 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3152000000000001e-05 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009619+0000", 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6065999999999999e-05 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.009637+0000", 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8085e-05 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.346476+0000", 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33683961200000001 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.346731+0000", 2024-08-17T17:34:54.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025431500000000002 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:56992 12.7 e325/319 12:e742fdef:::benchmark_data_smithi142_261410_object56991:head v 325'3596, mlcod=325'3594)", 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:32:16.010464+0000", 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 156.91188990500001, 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33649907400000001, 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.255 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.010464+0000", 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.010464+0000", 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.010466+0000", 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.542e-06 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.010507+0000", 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1221999999999999e-05 2024-08-17T17:34:54.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.341156+0000", 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33064861000000001 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.341164+0000", 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7059999999999993e-06 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.341222+0000", 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7602999999999997e-05 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.341250+0000", 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8334999999999999e-05 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.346908+0000", 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0056576370000000001 2024-08-17T17:34:54.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:32:16.346963+0000", 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5418999999999999e-05 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:112415 12.4 e330/319 12:226a051b:::benchmark_data_smithi142_261410_object112414:head v 330'6931, mlcod=330'6928)", 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.971707+0000", 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.950646703, 2024-08-17T17:34:54.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34399211899999999, 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971707+0000", 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971707+0000", 2024-08-17T17:34:54.261 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971708+0000", 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0950000000000001e-06 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971781+0000", 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2865000000000001e-05 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971793+0000", 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1956999999999999e-05 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971800+0000", 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1509999999999998e-06 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971860+0000", 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9917999999999998e-05 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971883+0000", 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2957000000000001e-05 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315643+0000", 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34376013399999999 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315699+0000", 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6042000000000003e-05 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112420 12.6 12:61ca06b2:::benchmark_data_smithi142_261410_object112419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.974860+0000", 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.947494315, 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34479163299999999, 2024-08-17T17:34:54.265 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112420 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974860+0000", 2024-08-17T17:34:54.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974860+0000", 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974862+0000", 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.582e-06 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974895+0000", 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2369999999999997e-05 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974911+0000", 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6665000000000001e-05 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974921+0000", 2024-08-17T17:34:54.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7599999999999997e-06 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976156+0000", 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001235014 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976372+0000", 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000216145 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976513+0000", 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014044799999999999 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319515+0000", 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34300181600000001 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319525+0000", 2024-08-17T17:34:54.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0505e-05 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319558+0000", 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2926000000000003e-05 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319607+0000", 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9308999999999998e-05 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319651+0000", 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4092999999999999e-05 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:112418 12.7 e330/319 12:e0d90dce:::benchmark_data_smithi142_261410_object112417:head v 330'7009, mlcod=330'7006)", 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.975051+0000", 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.947303332, 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33773811199999998, 2024-08-17T17:34:54.272 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975051+0000", 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975051+0000", 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975052+0000", 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.082e-06 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975138+0000", 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6225999999999997e-05 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975153+0000", 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4523e-05 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975160+0000", 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2710000000000003e-06 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975262+0000", 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010202800000000001 2024-08-17T17:34:54.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975283+0000", 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0791000000000001e-05 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312735+0000", 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33745272399999998 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312789+0000", 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3467000000000002e-05 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:112419 12.4 e330/319 12:213f767a:::benchmark_data_smithi142_261410_object112418:head v 330'6932, mlcod=330'6928)", 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.975189+0000", 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.947165, 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34409234900000002, 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.277 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975189+0000", 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975189+0000", 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975190+0000", 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9999999999999997e-07 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975245+0000", 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5454000000000001e-05 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975256+0000", 2024-08-17T17:34:54.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0832000000000001e-05 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975263+0000", 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312755+0000", 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33749242299999999 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312789+0000", 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3392999999999997e-05 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319234+0000", 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064450030000000004 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319281+0000", 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7772000000000003e-05 2024-08-17T17:34:54.281 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112421 12.6 12:6aa59c45:::benchmark_data_smithi142_261410_object112420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.975254+0000", 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.94710017200001, 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35309264099999998, 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.282 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112421 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975254+0000", 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975254+0000", 2024-08-17T17:34:54.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975255+0000", 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5199999999999995e-07 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975276+0000", 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1350999999999999e-05 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975288+0000", 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1435e-05 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975294+0000", 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7320000000000001e-06 2024-08-17T17:34:54.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315710+0000", 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34041555600000001 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315859+0000", 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149598 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.316011+0000", 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000151278 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328095+0000", 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012084594000000001 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328130+0000", 2024-08-17T17:34:54.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5156e-05 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328248+0000", 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011713 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328300+0000", 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2113000000000001e-05 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328347+0000", 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6845999999999998e-05 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112424 12.6 12:645c2f85:::benchmark_data_smithi142_261410_object112423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.976252+0000", 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.946102115, 2024-08-17T17:34:54.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35217241199999999, 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112424 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976252+0000", 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976252+0000", 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976253+0000", 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.09e-07 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976272+0000", 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9545e-05 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976281+0000", 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6240000000000008e-06 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976287+0000", 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0290000000000002e-06 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319683+0000", 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34339546700000001 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319831+0000", 2024-08-17T17:34:54.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014813599999999999 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.320195+0000", 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036470099999999999 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328189+0000", 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079938449999999994 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328238+0000", 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8869999999999998e-05 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328354+0000", 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116296 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328395+0000", 2024-08-17T17:34:54.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0342000000000001e-05 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328424+0000", 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9648000000000001e-05 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112426 12.6 12:6a0dc15b:::benchmark_data_smithi142_261410_object112425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:34:54.296 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.979459+0000", 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.942895327, 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36174886099999998, 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112426 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979459+0000", 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979459+0000", 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979460+0000", 2024-08-17T17:34:54.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.765e-06 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979485+0000", 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4813000000000001e-05 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312692+0000", 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33320643500000002 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312710+0000", 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7818000000000001e-05 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.323866+0000", 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011156919 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.324047+0000", 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000180514 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.324182+0000", 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013546000000000001 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.340995+0000", 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016812420000000002 2024-08-17T17:34:54.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341058+0000", 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3223000000000001e-05 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341106+0000", 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7602000000000003e-05 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341164+0000", 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8125999999999999e-05 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341208+0000", 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3766000000000001e-05 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.303 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:112420 12.6 e330/319 ondisk, result = 0)", 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.980350+0000", 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.942003738, 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33917538400000002, 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980350+0000", 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980350+0000", 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980351+0000", 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.032e-06 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980356+0000", 2024-08-17T17:34:54.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9729999999999997e-06 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312695+0000", 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.332338945 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312708+0000", 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2778e-05 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319502+0000", 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067940639999999998 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319509+0000", 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4869999999999999e-06 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319526+0000", 2024-08-17T17:34:54.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7105e-05 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124351 12.6 12:637584c3:::benchmark_data_smithi142_261410_object6154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.812962+0000", 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 83.109392130000003, 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36987189399999998, 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.308 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124351 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812962+0000", 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812962+0000", 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812963+0000", 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.39e-06 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812968+0000", 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.561e-06 2024-08-17T17:34:54.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812978+0000", 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0335e-05 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812984+0000", 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7740000000000001e-06 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.813573+0000", 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058854999999999997 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.813702+0000", 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012960900000000001 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.813820+0000", 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117624 2024-08-17T17:34:54.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818616+0000", 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0047958339999999997 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.822350+0000", 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037340160000000002 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182730+0000", 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36038021100000001 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182784+0000", 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4246999999999997e-05 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182834+0000", 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9743000000000001e-05 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.314 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124355 12.6 12:6a89de93:::benchmark_data_smithi142_261410_object6158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.818633+0000", 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 83.103720576000001, 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36461828400000001, 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.315 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124355 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818633+0000", 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818633+0000", 2024-08-17T17:34:54.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818636+0000", 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1849999999999998e-06 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818641+0000", 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4960000000000004e-06 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818654+0000", 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3348e-05 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818663+0000", 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6689999999999995e-06 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818718+0000", 2024-08-17T17:34:54.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4656e-05 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818847+0000", 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012894499999999999 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818974+0000", 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000127284 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.822364+0000", 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003390311 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.826234+0000", 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038695719999999999 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182872+0000", 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35663769899999997 2024-08-17T17:34:54.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183211+0000", 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00033976399999999999 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183252+0000", 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0355000000000002e-05 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPush(12.a 335/332 [PushOp(12:540a96b7:::benchmark_data_smithi142_261410_object6169:head, version: 323'409, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:540a96b7:::benchmark_data_smithi142_261410_object6169:head@323'409, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.830075+0000", 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 83.092278992000004, 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35255843799999997, 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830075+0000", 2024-08-17T17:34:54.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830075+0000", 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830076+0000", 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2410000000000001e-06 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830106+0000", 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9629e-05 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830126+0000", 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9961000000000001e-05 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830140+0000", 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4003e-05 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.831209+0000", 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010689160000000001 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.831220+0000", 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0869e-05 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182633+0000", 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35141381900000002 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:124358 12.7 e335/332 12:ee40e7c7:::benchmark_data_smithi142_261410_object6161:head v 335'7760, mlcod=322'128)", 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.834295+0000", 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 83.088058857999997, 2024-08-17T17:34:54.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36059230599999997, 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834295+0000", 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834295+0000", 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834296+0000", 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.145e-06 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834306+0000", 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1819999999999995e-06 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834321+0000", 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5634999999999999e-05 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834330+0000", 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2949999999999999e-06 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182530+0000", 2024-08-17T17:34:54.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34819973700000001 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182567+0000", 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7032000000000001e-05 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194840+0000", 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012272857 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194887+0000", 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7423000000000003e-05 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:124351 12.6 e335/319 ondisk, result = 0)", 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.839700+0000", 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 83.082654228999999, 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.343139741, 2024-08-17T17:34:54.331 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839700+0000", 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839700+0000", 2024-08-17T17:34:54.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839701+0000", 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.06e-06 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839707+0000", 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8340000000000004e-06 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182642+0000", 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.342935406 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182655+0000", 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3193999999999999e-05 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182716+0000", 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0647999999999999e-05 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182724+0000", 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8290000000000008e-06 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182840+0000", 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011577 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124369 12.6 12:6623c19c:::benchmark_data_smithi142_261410_object6172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.843595+0000", 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 83.078759409, 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35119447599999998, 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.336 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124369 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843595+0000", 2024-08-17T17:34:54.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843595+0000", 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843596+0000", 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843604+0000", 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3780000000000003e-06 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182660+0000", 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.339056248 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182673+0000", 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2995e-05 2024-08-17T17:34:54.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183281+0000", 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00060766499999999996 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183414+0000", 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013326000000000001 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:54.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183536+0000", 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012200499999999999 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.191414+0000", 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078783220000000001 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192455+0000", 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010408139999999999 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194685+0000", 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0022300499999999999 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194741+0000", 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6051000000000003e-05 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194789+0000", 2024-08-17T17:34:54.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7958999999999999e-05 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:130294 12.d 12:b6fb4791:::benchmark_data_smithi142_261410_object12097:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:36.228291+0000", 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 16.694062821999999, 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.022097851000000002, 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.343 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 130294 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228291+0000", 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228291+0000", 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228293+0000", 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.95e-06 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228305+0000", 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.168e-05 2024-08-17T17:34:54.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228322+0000", 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6872e-05 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228332+0000", 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9869999999999994e-06 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.228405+0000", 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3393999999999996e-05 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.229114+0000", 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00070862500000000005 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.239202+0000", 2024-08-17T17:34:54.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010087909000000001 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.239317+0000", 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115715 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.239480+0000", 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016253 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.245513+0000", 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060328029999999998 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.245553+0000", 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0683000000000003e-05 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.245570+0000", 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6229999999999999e-05 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.246248+0000", 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000678592 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.246288+0000", 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9990999999999997e-05 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.250389+0000", 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041008900000000003 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:130308 12.d 12:b55ee6ed:::benchmark_data_smithi142_261410_object12111:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:36.526218+0000", 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 16.396135615999999, 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.020429236, 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.351 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 130308 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526218+0000", 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526218+0000", 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526220+0000", 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526227+0000", 2024-08-17T17:34:54.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0840000000000003e-06 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526242+0000", 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5243e-05 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526250+0000", 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4939999999999997e-06 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526304+0000", 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4005000000000002e-05 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.526743+0000", 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000439097 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.535392+0000", 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0086489800000000006 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.535465+0000", 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2949999999999998e-05 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.535556+0000", 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1484999999999996e-05 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.540719+0000", 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051624189999999997 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.540758+0000", 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8807000000000003e-05 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.540775+0000", 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7342000000000001e-05 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.542825+0000", 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020499759999999998 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.542866+0000", 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0725000000000001e-05 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:36.546648+0000", 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037818999999999999 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:130397 12.d 12:b0b364ca:::benchmark_data_smithi142_261410_object12200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:38.472357+0000", 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 14.449996901, 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.022989209, 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.359 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 130397 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472357+0000", 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472357+0000", 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472359+0000", 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.437e-06 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472364+0000", 2024-08-17T17:34:54.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.271e-06 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472376+0000", 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2198000000000001e-05 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472382+0000", 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3400000000000003e-06 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472442+0000", 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9599999999999999e-05 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.472989+0000", 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00054703 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.482712+0000", 2024-08-17T17:34:54.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0097225639999999995 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.482782+0000", 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0129999999999994e-05 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.482908+0000", 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000125831 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.488207+0000", 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00529899 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.488269+0000", 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2088000000000006e-05 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.488332+0000", 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3201999999999996e-05 2024-08-17T17:34:54.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.490768+0000", 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024366330000000001 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.490835+0000", 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6148000000000004e-05 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:38.495346+0000", 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.004511747 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:130431 12.d 12:bea283a5:::benchmark_data_smithi142_261410_object12234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:39.206450+0000", 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 13.715903908, 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024459798000000001, 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:34:54.367 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:54.368 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:34:54.368 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:34:54.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:34:54.369 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 130431 2024-08-17T17:34:54.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.369 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:34:54.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.369 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206450+0000", 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206450+0000", 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206452+0000", 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.595e-06 2024-08-17T17:34:54.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206460+0000", 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.011e-06 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206476+0000", 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6328999999999999e-05 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:34:54.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206484+0000", 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5789999999999998e-06 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.206537+0000", 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2890999999999999e-05 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.207121+0000", 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058410999999999999 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.216676+0000", 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0095550429999999992 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.216785+0000", 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010898799999999999 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:34:54.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.216948+0000", 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016310200000000001 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.222941+0000", 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059932309999999999 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.222956+0000", 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5231999999999999e-05 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.222998+0000", 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1313000000000002e-05 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.225927+0000", 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029291159999999998 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.226009+0000", 2024-08-17T17:34:54.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1887999999999999e-05 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:39.230910+0000", 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0049013700000000004 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:34:54.376 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:34:54.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:54.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:54.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:54.524 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:54.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:54.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:54.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:54.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:54.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:54.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:54.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:54.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:54.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:34:54.939 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:54.940 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:55.000 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:55.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:55.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:55.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:55.147 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:55.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:55.272 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:55.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:55.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:55.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:55.514 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:55 smithi142 ceph-mon[106729]: pgmap v1214: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 46 op/s; 72385/316272 objects degraded (22.887%); 37369/316272 objects misplaced (11.815%); 13 MiB/s, 211 objects/s recovering 2024-08-17T17:34:55.514 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:55 smithi142 ceph-mon[103614]: pgmap v1214: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 46 op/s; 72385/316272 objects degraded (22.887%); 37369/316272 objects misplaced (11.815%); 13 MiB/s, 211 objects/s recovering 2024-08-17T17:34:55.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:55.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:55.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:55.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:55.778 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:55.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:55 smithi152 ceph-mon[93456]: pgmap v1214: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 46 op/s; 72385/316272 objects degraded (22.887%); 37369/316272 objects misplaced (11.815%); 13 MiB/s, 211 objects/s recovering 2024-08-17T17:34:55.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:55.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:56.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:56.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:56.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:56.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:34:56.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:56.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:56.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:56.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:56.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:56.378 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:56.407 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:56.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:56.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:56.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:56.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:56.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:56.781 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 72385/316272 objects degraded (22.887%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 72385/316272 objects degraded (22.887%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 72385/316272 objects degraded (22.887%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:56.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:56.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:57.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:57.041 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:57.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:57.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:57.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:57.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:57.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:57.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:57.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:57.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:57 smithi142 ceph-mon[106729]: pgmap v1215: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 39 op/s; 72354/316002 objects degraded (22.897%); 36544/316002 objects misplaced (11.564%); 14 MiB/s, 230 objects/s recovering 2024-08-17T17:34:57.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:57.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:57.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:57 smithi142 ceph-mon[103614]: pgmap v1215: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 39 op/s; 72354/316002 objects degraded (22.897%); 36544/316002 objects misplaced (11.564%); 14 MiB/s, 230 objects/s recovering 2024-08-17T17:34:57.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:57.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:57.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:57.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:57.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.782 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:57.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:57 smithi152 ceph-mon[93456]: pgmap v1215: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 39 op/s; 72354/316002 objects degraded (22.897%); 36544/316002 objects misplaced (11.564%); 14 MiB/s, 230 objects/s recovering 2024-08-17T17:34:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:34:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:57.883 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:58.007 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:58.029 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:58.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:58.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:58.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:58.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:58.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:58.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:58.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:58.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:58.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:58.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:58.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:58.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.782 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:58.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:58.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:34:58.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:34:59.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:59.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:59.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:59.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:59.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:59.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:59.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:59.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:59.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:34:59.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:59.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:59.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:59.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:59.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:59.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:59.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:34:59 smithi142 ceph-mon[103614]: pgmap v1216: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 47 op/s; 72336/315474 objects degraded (22.929%); 35873/315474 objects misplaced (11.371%); 17 MiB/s, 268 objects/s recovering 2024-08-17T17:34:59.769 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:34:59 smithi142 ceph-mon[106729]: pgmap v1216: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 47 op/s; 72336/315474 objects degraded (22.929%); 35873/315474 objects misplaced (11.371%); 17 MiB/s, 268 objects/s recovering 2024-08-17T17:34:59.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:59.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:59.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:34:59 smithi152 ceph-mon[93456]: pgmap v1216: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 47 op/s; 72336/315474 objects degraded (22.929%); 35873/315474 objects misplaced (11.371%); 17 MiB/s, 268 objects/s recovering 2024-08-17T17:34:59.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:59.914 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:34:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:00.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:00.039 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:00.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:00.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:00.164 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:00.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:00.287 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:00.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:00.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:00.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:00.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:00.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:00 smithi152 ceph-mon[93456]: osdmap e345: 8 total, 8 up, 6 in 2024-08-17T17:35:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/169899188' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:35:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:00 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:35:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:35:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:35:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:00.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:00.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:00.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:00.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:00 smithi142 ceph-mon[106729]: osdmap e345: 8 total, 8 up, 6 in 2024-08-17T17:35:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/169899188' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:35:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:00 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:35:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:00 smithi142 ceph-mon[103614]: osdmap e345: 8 total, 8 up, 6 in 2024-08-17T17:35:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/169899188' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:35:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:00 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:35:00.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:00.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:00.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:01.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:01.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:01.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:01.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:01.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:01.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:01.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:01.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:01.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.424 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:01.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:01.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:01.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:01.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:01.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[106729]: pgmap v1218: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 47 op/s; 65783/315336 objects degraded (20.861%); 42363/315336 objects misplaced (13.434%); 17 MiB/s, 268 objects/s recovering 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[106729]: osdmap e346: 8 total, 8 up, 6 in 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 65783/315336 objects degraded (20.861%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[103614]: pgmap v1218: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 47 op/s; 65783/315336 objects degraded (20.861%); 42363/315336 objects misplaced (13.434%); 17 MiB/s, 268 objects/s recovering 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[103614]: osdmap e346: 8 total, 8 up, 6 in 2024-08-17T17:35:01.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 65783/315336 objects degraded (20.861%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:01.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:01.795 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:01 smithi152 ceph-mon[93456]: pgmap v1218: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 47 op/s; 65783/315336 objects degraded (20.861%); 42363/315336 objects misplaced (13.434%); 17 MiB/s, 268 objects/s recovering 2024-08-17T17:35:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:01 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:35:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:01 smithi152 ceph-mon[93456]: osdmap e346: 8 total, 8 up, 6 in 2024-08-17T17:35:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 65783/315336 objects degraded (20.861%), 13 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:01.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:01.919 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:02.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:02.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:35:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:35:01] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-08-17T17:35:02.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:02.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:02.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:02.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:02.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:02.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:02.420 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:02.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:02.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:02.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:02.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:02.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:02.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:02.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:02.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:02.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:03.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:03.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:03.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:03.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:03.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:03.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:03.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:03.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:03.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:03.424 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:03.526 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:03.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:03.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:03.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:03.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:03 smithi152 ceph-mon[93456]: pgmap v1220: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 58469/314919 objects degraded (18.566%); 47628/314919 objects misplaced (15.124%); 22 MiB/s, 435 objects/s recovering 2024-08-17T17:35:03.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:03 smithi142 ceph-mon[106729]: pgmap v1220: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 58469/314919 objects degraded (18.566%); 47628/314919 objects misplaced (15.124%); 22 MiB/s, 435 objects/s recovering 2024-08-17T17:35:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:03 smithi142 ceph-mon[103614]: pgmap v1220: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 58469/314919 objects degraded (18.566%); 47628/314919 objects misplaced (15.124%); 22 MiB/s, 435 objects/s recovering 2024-08-17T17:35:03.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:03.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:03.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:04.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:04.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:04.107 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:04.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:04.231 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:04.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:04.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:04.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:04.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:04.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:04.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:04.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:04.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:04.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:04.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:04.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:04.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:04.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:04.974 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:35:04.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:05.002 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:05.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:05.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:05.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:05.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:05.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:05.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:05.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:05.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:05.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:05.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:05.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:05.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:05.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:05 smithi152 ceph-mon[93456]: pgmap v1221: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 51 op/s; 58469/314769 objects degraded (18.575%); 47628/314769 objects misplaced (15.131%); 15 MiB/s, 333 objects/s recovering 2024-08-17T17:35:05.848 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:05.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:05 smithi142 ceph-mon[106729]: pgmap v1221: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 51 op/s; 58469/314769 objects degraded (18.575%); 47628/314769 objects misplaced (15.131%); 15 MiB/s, 333 objects/s recovering 2024-08-17T17:35:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:05 smithi142 ceph-mon[103614]: pgmap v1221: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 51 op/s; 58469/314769 objects degraded (18.575%); 47628/314769 objects misplaced (15.131%); 15 MiB/s, 333 objects/s recovering 2024-08-17T17:35:05.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:05.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:06.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:06.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:06.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:06.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:06.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:06.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:06.473 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:06.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:06.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:06.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:06.745 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:06.809 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:06 smithi142 ceph-mon[103614]: osdmap e347: 8 total, 8 up, 6 in 2024-08-17T17:35:06.809 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 58469/314769 objects degraded (18.575%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:06.809 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:06 smithi142 ceph-mon[103614]: pgmap v1223: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 55 op/s; 58448/314334 objects degraded (18.594%); 47287/314334 objects misplaced (15.044%); 15 MiB/s, 342 objects/s recovering 2024-08-17T17:35:06.809 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:06 smithi142 ceph-mon[106729]: osdmap e347: 8 total, 8 up, 6 in 2024-08-17T17:35:06.809 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 58469/314769 objects degraded (18.575%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:06.810 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:06 smithi142 ceph-mon[106729]: pgmap v1223: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 55 op/s; 58448/314334 objects degraded (18.594%); 47287/314334 objects misplaced (15.044%); 15 MiB/s, 342 objects/s recovering 2024-08-17T17:35:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:06 smithi152 ceph-mon[93456]: osdmap e347: 8 total, 8 up, 6 in 2024-08-17T17:35:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 58469/314769 objects degraded (18.575%), 12 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:06 smithi152 ceph-mon[93456]: pgmap v1223: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 55 op/s; 58448/314334 objects degraded (18.594%); 47287/314334 objects misplaced (15.044%); 15 MiB/s, 342 objects/s recovering 2024-08-17T17:35:06.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:06.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:06.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:06.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:07.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:07.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:07.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:07.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:07.241 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:07.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:07.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:07.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:07.493 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:07.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:07.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:07.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:07.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:07 smithi152 ceph-mon[93456]: osdmap e348: 8 total, 8 up, 6 in 2024-08-17T17:35:07.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:07.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:07 smithi142 ceph-mon[106729]: osdmap e348: 8 total, 8 up, 6 in 2024-08-17T17:35:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:07 smithi142 ceph-mon[103614]: osdmap e348: 8 total, 8 up, 6 in 2024-08-17T17:35:07.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:07.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:07.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:08.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:08.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:08.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:08.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:08.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:08.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:08.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:08.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:08.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:08.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:08.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:08.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:08.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:08.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:08.733 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:08.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:08 smithi152 ceph-mon[93456]: pgmap v1225: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57823/313632 objects degraded (18.437%); 47065/313632 objects misplaced (15.006%); 14 MiB/s, 406 objects/s recovering 2024-08-17T17:35:08.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:08.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:08.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:08.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:08 smithi142 ceph-mon[106729]: pgmap v1225: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57823/313632 objects degraded (18.437%); 47065/313632 objects misplaced (15.006%); 14 MiB/s, 406 objects/s recovering 2024-08-17T17:35:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:08 smithi142 ceph-mon[103614]: pgmap v1225: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57823/313632 objects degraded (18.437%); 47065/313632 objects misplaced (15.006%); 14 MiB/s, 406 objects/s recovering 2024-08-17T17:35:08.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:08.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:09.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:09.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:09.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:09.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:09.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:09.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:09.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:09.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:09.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:09.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:09.584 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:35:09.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:09.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:09.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:09.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:09.850 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:09.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:09.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:09.975 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:10.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:10.078 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:10.102 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:10.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:10.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:10.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:10.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:10.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:10.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:10.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:35:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:35:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:10.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:10.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:10.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:10.724 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:10.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:10.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:10.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:10.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:10.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:11.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:11.098 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:11.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:11.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:11.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:11.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:11.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:11.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:11.523 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:11 smithi142 ceph-mon[106729]: pgmap v1226: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 57795/313437 objects degraded (18.439%); 47003/313437 objects misplaced (14.996%); 4.1 MiB/s, 153 objects/s recovering 2024-08-17T17:35:11.524 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57795/313437 objects degraded (18.439%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:11.524 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:11 smithi142 ceph-mon[103614]: pgmap v1226: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 57795/313437 objects degraded (18.439%); 47003/313437 objects misplaced (14.996%); 4.1 MiB/s, 153 objects/s recovering 2024-08-17T17:35:11.524 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57795/313437 objects degraded (18.439%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:11.528 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:35:11.528 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:35:11.528 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:35:11.528 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:35:11.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:11 smithi152 ceph-mon[93456]: pgmap v1226: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 57795/313437 objects degraded (18.439%); 47003/313437 objects misplaced (14.996%); 4.1 MiB/s, 153 objects/s recovering 2024-08-17T17:35:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57795/313437 objects degraded (18.439%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:11.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:11.603 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:11.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:11.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:11.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:11.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:11.860 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:35:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:35:11] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:35:11.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:11.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:11.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:12.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:12.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:12.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:12.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:12.300 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:35:12.329 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:12.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:12.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:12.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:12.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:12.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:12.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/276522957' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:12.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:12.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:12.568 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/276522957' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:12.569 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:12.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:12.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/276522957' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:12.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:12.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:12.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:12.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:12.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:12.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:12.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:12.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:12.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:12.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:12.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:12.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:13.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:13.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:13.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:13.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:13.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:13.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:13.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:13.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:13.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:13.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:13 smithi152 ceph-mon[93456]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57764/312678 objects degraded (18.474%); 46623/312678 objects misplaced (14.911%); 6.6 MiB/s, 193 objects/s recovering 2024-08-17T17:35:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:13 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:13 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:13 smithi152 ceph-mon[93456]: osdmap e349: 8 total, 8 up, 6 in 2024-08-17T17:35:13.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[106729]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57764/312678 objects degraded (18.474%); 46623/312678 objects misplaced (14.911%); 6.6 MiB/s, 193 objects/s recovering 2024-08-17T17:35:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[106729]: osdmap e349: 8 total, 8 up, 6 in 2024-08-17T17:35:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[103614]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57764/312678 objects degraded (18.474%); 46623/312678 objects misplaced (14.911%); 6.6 MiB/s, 193 objects/s recovering 2024-08-17T17:35:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:13.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:13 smithi142 ceph-mon[103614]: osdmap e349: 8 total, 8 up, 6 in 2024-08-17T17:35:13.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:13.719 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:13.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:13.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:13.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:13.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:13.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:14.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:14.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:14.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:14.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:14.280 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:14.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:35:14.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:14.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:14.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:14.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:14.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:14.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:14.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:14.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:14.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:14.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:14.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:14.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:14.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.966 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:14.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:15.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:15.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:15.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:15.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:15.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:15.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:15.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:15.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:15.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:15.550 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:15 smithi142 ceph-mon[106729]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 57764/312429 objects degraded (18.489%); 46623/312429 objects misplaced (14.923%); 4.2 MiB/s, 154 objects/s recovering 2024-08-17T17:35:15.550 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:15 smithi142 ceph-mon[103614]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 57764/312429 objects degraded (18.489%); 46623/312429 objects misplaced (14.923%); 4.2 MiB/s, 154 objects/s recovering 2024-08-17T17:35:15.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:15 smithi152 ceph-mon[93456]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 57764/312429 objects degraded (18.489%); 46623/312429 objects misplaced (14.923%); 4.2 MiB/s, 154 objects/s recovering 2024-08-17T17:35:15.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:15.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:15.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:15.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:35:15.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:15.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:15.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:16.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:16.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:16.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:16.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:16.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:16.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:16.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:16.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:16.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:16.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/107976729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:16.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57764/312429 objects degraded (18.489%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:16.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/107976729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57764/312429 objects degraded (18.489%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:16.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/107976729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:16.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57764/312429 objects degraded (18.489%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:16.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:16.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:16.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:16.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:16.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:16.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:16.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:17.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:17.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:17.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:17.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:17.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:17.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:17.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:17.462 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:17.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:17.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:17 smithi152 ceph-mon[93456]: pgmap v1230: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 57739/312057 objects degraded (18.503%); 46254/312057 objects misplaced (14.822%); 5.6 MiB/s, 161 objects/s recovering 2024-08-17T17:35:17.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:17 smithi142 ceph-mon[106729]: pgmap v1230: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 57739/312057 objects degraded (18.503%); 46254/312057 objects misplaced (14.822%); 5.6 MiB/s, 161 objects/s recovering 2024-08-17T17:35:17.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:17 smithi142 ceph-mon[103614]: pgmap v1230: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 57739/312057 objects degraded (18.503%); 46254/312057 objects misplaced (14.822%); 5.6 MiB/s, 161 objects/s recovering 2024-08-17T17:35:17.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:17.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:35:17.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:17.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:17.812 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:17.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:17.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:17.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:18.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:18.082 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:18.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:18.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:18.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:18.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:18.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:18.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:18.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:18.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:18.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:18.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:18.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:18.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:18.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:18.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:18.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:19.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:19.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:19.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:19.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:19.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:19.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:19 smithi152 ceph-mon[93456]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 57726/311481 objects degraded (18.533%); 46198/311481 objects misplaced (14.832%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:35:19.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:19 smithi142 ceph-mon[106729]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 57726/311481 objects degraded (18.533%); 46198/311481 objects misplaced (14.832%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:35:19.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:19 smithi142 ceph-mon[103614]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 57726/311481 objects degraded (18.533%); 46198/311481 objects misplaced (14.832%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:35:19.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:19.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:19.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:19.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:19.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:19.952 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:20.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:20.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:20.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:20.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:20.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:20.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:20.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:20.426 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:20.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:20.550 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:20.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:35:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:35:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:20.674 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:20.686 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:20.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:20.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:20.748 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:20.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:20.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:20.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:20.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:20.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:20.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:21.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:21.081 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:21.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:21.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:21.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:21.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:21 smithi142 ceph-mon[106729]: pgmap v1232: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57700/311247 objects degraded (18.538%); 46120/311247 objects misplaced (14.818%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:35:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57700/311247 objects degraded (18.538%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:21 smithi142 ceph-mon[103614]: pgmap v1232: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57700/311247 objects degraded (18.538%); 46120/311247 objects misplaced (14.818%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:35:21.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57700/311247 objects degraded (18.538%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:21.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:21.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:21.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:21.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:21 smithi152 ceph-mon[93456]: pgmap v1232: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57700/311247 objects degraded (18.538%); 46120/311247 objects misplaced (14.818%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:35:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57700/311247 objects degraded (18.538%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:21.575 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:21.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:21.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:21.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:21.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:21.831 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:35:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:35:21] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:35:21.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:21.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:22.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:22.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:22.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:22.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:22.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:35:22.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:22.325 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:22.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:22.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:22.450 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:22.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:22.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:22.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:22.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:22.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:22.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:22.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:22.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:22.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:23.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:23.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:23.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:23.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:23.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:23.315 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:35:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:23.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:23.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:23.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:23.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:23 smithi152 ceph-mon[93456]: pgmap v1233: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 57677/310590 objects degraded (18.570%); 45741/310590 objects misplaced (14.727%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:35:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3340812348' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:35:23.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:23 smithi142 ceph-mon[106729]: pgmap v1233: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 57677/310590 objects degraded (18.570%); 45741/310590 objects misplaced (14.727%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:35:23.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3340812348' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:35:23.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:23 smithi142 ceph-mon[103614]: pgmap v1233: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 57677/310590 objects degraded (18.570%); 45741/310590 objects misplaced (14.727%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:35:23.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3340812348' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:35:23.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:23.696 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:23.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:23.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:23.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:23.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:24.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:24.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:24.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:24.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.450 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:24.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3340812348' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:35:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:24 smithi142 ceph-mon[106729]: osdmap e350: 8 total, 8 up, 6 in 2024-08-17T17:35:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3340812348' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:35:24.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:24 smithi142 ceph-mon[103614]: osdmap e350: 8 total, 8 up, 6 in 2024-08-17T17:35:24.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:24.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:24.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3340812348' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:35:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:24 smithi152 ceph-mon[93456]: osdmap e350: 8 total, 8 up, 6 in 2024-08-17T17:35:24.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:24.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:24.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:24.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:25.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:25.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:25.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:25.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:25.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:25.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:25.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:25.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:25.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:25.568 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:25 smithi142 ceph-mon[106729]: pgmap v1235: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57677/310425 objects degraded (18.580%); 45741/310425 objects misplaced (14.735%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:35:25.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:25 smithi142 ceph-mon[103614]: pgmap v1235: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57677/310425 objects degraded (18.580%); 45741/310425 objects misplaced (14.735%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:35:25.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:25.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:25.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:25.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:25 smithi152 ceph-mon[93456]: pgmap v1235: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57677/310425 objects degraded (18.580%); 45741/310425 objects misplaced (14.735%); 4.7 MiB/s, 74 objects/s recovering 2024-08-17T17:35:25.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:25.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:26.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:26.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:26.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:26.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:26.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:26.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:26.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:26.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:26.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:26.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:26.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:26.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:26.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:26.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57677/310425 objects degraded (18.580%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57677/310425 objects degraded (18.580%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:26.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:26.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:26.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:26.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:35:26.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:26.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:26.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57677/310425 objects degraded (18.580%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:26.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:26.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:27.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:27.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:27.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:27.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:27.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:27.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:27.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:27.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:27.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:27.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:27.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:27 smithi142 ceph-mon[106729]: pgmap v1236: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 57649/310125 objects degraded (18.589%); 45349/310125 objects misplaced (14.623%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:35:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:27 smithi142 ceph-mon[103614]: pgmap v1236: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 57649/310125 objects degraded (18.589%); 45349/310125 objects misplaced (14.623%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:35:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:27.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:27.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:27.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:27.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:27 smithi152 ceph-mon[93456]: pgmap v1236: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 57649/310125 objects degraded (18.589%); 45349/310125 objects misplaced (14.623%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:35:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:27.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:27.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:28.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:28.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:28.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:28.193 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:28.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:28.317 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:28.381 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:28.382 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:28.419 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:28.446 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:28.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:28.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:28.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:28.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:28.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:28.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:28.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:28.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:28.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:28.941 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:29.043 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.192 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:29.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:29.319 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:29.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:29.439 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:35:29.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:29.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:29 smithi142 ceph-mon[106729]: pgmap v1237: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 57620/309423 objects degraded (18.622%); 45281/309423 objects misplaced (14.634%); 5.0 MiB/s, 79 objects/s recovering 2024-08-17T17:35:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/883740206' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:29.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:29 smithi142 ceph-mon[103614]: pgmap v1237: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 57620/309423 objects degraded (18.622%); 45281/309423 objects misplaced (14.634%); 5.0 MiB/s, 79 objects/s recovering 2024-08-17T17:35:29.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/883740206' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:29.668 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:29.690 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:29.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:35:29.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:29.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:29 smithi152 ceph-mon[93456]: pgmap v1237: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 57620/309423 objects degraded (18.622%); 45281/309423 objects misplaced (14.634%); 5.0 MiB/s, 79 objects/s recovering 2024-08-17T17:35:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/883740206' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:29.899 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:29.902 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:29.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:29.938 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:30.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:30.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:30.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:30.188 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:30.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:30.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:30.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:30.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:30.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:30.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:30.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:30.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:30 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/883740206' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:30 smithi152 ceph-mon[93456]: osdmap e351: 8 total, 8 up, 6 in 2024-08-17T17:35:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:35:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:35:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:30.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:30.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:30.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:30.728 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:30 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:30.728 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/883740206' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:30.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:30 smithi142 ceph-mon[106729]: osdmap e351: 8 total, 8 up, 6 in 2024-08-17T17:35:30.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:30 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:30.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/883740206' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:30.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:30 smithi142 ceph-mon[103614]: osdmap e351: 8 total, 8 up, 6 in 2024-08-17T17:35:30.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:30.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:30.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:30.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.035 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:31.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:31.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:31.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:31.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:31.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:31.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:31.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:31.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:31.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:31.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:31.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:31.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:31 smithi142 ceph-mon[106729]: pgmap v1239: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 57592/309222 objects degraded (18.625%); 45216/309222 objects misplaced (14.623%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:35:31.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57592/309222 objects degraded (18.625%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:31.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:31 smithi142 ceph-mon[103614]: pgmap v1239: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 57592/309222 objects degraded (18.625%); 45216/309222 objects misplaced (14.623%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:35:31.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57592/309222 objects degraded (18.625%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:31.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:31.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:31 smithi152 ceph-mon[93456]: pgmap v1239: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 57592/309222 objects degraded (18.625%); 45216/309222 objects misplaced (14.623%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:35:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57592/309222 objects degraded (18.625%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:31.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:31.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:32.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:32.055 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:32.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:32.065 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:35:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:35:31] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:35:32.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:32.282 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:32.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:32.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:32.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:32.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:32.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:35:32.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:32.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:32.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:32.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:32.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:32.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:32.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:32 smithi152 ceph-mon[93456]: pgmap v1240: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 67 op/s; 57566/308610 objects degraded (18.653%); 44821/308610 objects misplaced (14.524%); 5.7 MiB/s, 90 objects/s recovering 2024-08-17T17:35:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:32 smithi142 ceph-mon[106729]: pgmap v1240: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 67 op/s; 57566/308610 objects degraded (18.653%); 44821/308610 objects misplaced (14.524%); 5.7 MiB/s, 90 objects/s recovering 2024-08-17T17:35:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:32 smithi142 ceph-mon[103614]: pgmap v1240: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 67 op/s; 57566/308610 objects degraded (18.653%); 44821/308610 objects misplaced (14.524%); 5.7 MiB/s, 90 objects/s recovering 2024-08-17T17:35:32.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:32.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:33.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:33.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:33.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:33.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:33.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:33.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:33.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:33.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:33.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:33.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:33.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:33.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:33.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:33.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:33.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:33.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:33.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:33.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:34.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:34.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:34.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:34.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:34.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:34.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:34.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:34.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:34.546 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:34.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:34.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:34.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:34.793 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:34.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:34.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:34.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:35.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:35.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:35.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:35.163 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:35.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:35.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:35.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:35.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:35.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:35.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:35.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:35 smithi152 ceph-mon[93456]: pgmap v1241: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57566/308430 objects degraded (18.664%); 44821/308430 objects misplaced (14.532%); 5.1 MiB/s, 80 objects/s recovering 2024-08-17T17:35:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:35 smithi142 ceph-mon[106729]: pgmap v1241: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57566/308430 objects degraded (18.664%); 44821/308430 objects misplaced (14.532%); 5.1 MiB/s, 80 objects/s recovering 2024-08-17T17:35:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:35 smithi142 ceph-mon[103614]: pgmap v1241: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57566/308430 objects degraded (18.664%); 44821/308430 objects misplaced (14.532%); 5.1 MiB/s, 80 objects/s recovering 2024-08-17T17:35:35.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.659 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:35.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:35.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.785 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:35.887 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:35.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:35.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:36.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.039 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:36.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:36.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:36.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:36.389 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:36.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:36.513 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:36.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57566/308430 objects degraded (18.664%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:36.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57566/308430 objects degraded (18.664%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:36.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57566/308430 objects degraded (18.664%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:36.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:36.660 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:36.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:36.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:36.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:37.010 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:37.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.164 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.513 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:37 smithi152 ceph-mon[93456]: pgmap v1242: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 57539/308001 objects degraded (18.681%); 44388/308001 objects misplaced (14.412%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:35:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:37 smithi142 ceph-mon[106729]: pgmap v1242: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 57539/308001 objects degraded (18.681%); 44388/308001 objects misplaced (14.412%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:35:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:37 smithi142 ceph-mon[103614]: pgmap v1242: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 57539/308001 objects degraded (18.681%); 44388/308001 objects misplaced (14.412%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:35:37.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:37.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:37.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:37.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:37.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:37.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:37.972 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:38.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:38.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:38.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:38.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:38.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:38.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:38.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:38.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:38.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:38.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:38.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:38.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:38.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:38.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:38.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:38.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:38.912 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:39.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:39.036 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:39.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:39.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:39.205 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:39.208 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:35:39.208 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:39.232 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:35:39.232 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:39.250 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:35:39.250 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:39.267 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:35:39.267 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:39.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:39.283 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:35:39.284 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:39.290 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:39.300 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:35:39.300 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:39.305 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:35:39.326 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:35:39.327 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:39.347 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:39 smithi142 ceph-mon[106729]: pgmap v1243: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57523/307254 objects degraded (18.722%); 44326/307254 objects misplaced (14.427%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T17:35:39.347 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2333322318' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:35:39.347 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1938957231' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:39.347 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:35:39.348 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:39.348 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:39 smithi142 ceph-mon[103614]: pgmap v1243: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57523/307254 objects degraded (18.722%); 44326/307254 objects misplaced (14.427%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T17:35:39.348 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2333322318' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:35:39.348 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1938957231' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:39.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:39.417 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:39.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:39.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:39.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:39 smithi152 ceph-mon[93456]: pgmap v1243: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57523/307254 objects degraded (18.722%); 44326/307254 objects misplaced (14.427%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T17:35:39.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2333322318' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:35:39.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1938957231' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:39.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:39.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:39.772 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:39.794 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:39.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:39.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:39.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:40.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:40.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:40.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:40.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:40.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:40.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:40.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:40.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:40.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:40.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:40.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:40.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:40.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:40.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2333322318' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:35:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:40 smithi152 ceph-mon[93456]: osdmap e352: 8 total, 8 up, 6 in 2024-08-17T17:35:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:35:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:35:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:35:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2333322318' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:35:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:40 smithi142 ceph-mon[106729]: osdmap e352: 8 total, 8 up, 6 in 2024-08-17T17:35:40.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:35:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2333322318' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:35:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:40 smithi142 ceph-mon[103614]: osdmap e352: 8 total, 8 up, 6 in 2024-08-17T17:35:40.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:35:40.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:40.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:40.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:40.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:40.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:40.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:40.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:41.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:41.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:41.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:41.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:41.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:41.296 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:41.345 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:41 smithi142 ceph-mon[106729]: pgmap v1245: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 57523/307254 objects degraded (18.722%); 44326/307254 objects misplaced (14.427%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:35:41.345 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57523/307254 objects degraded (18.722%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:41.345 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:41 smithi142 ceph-mon[103614]: pgmap v1245: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 57523/307254 objects degraded (18.722%); 44326/307254 objects misplaced (14.427%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:35:41.345 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57523/307254 objects degraded (18.722%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:41.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:41.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:41.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:41.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:41.544 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:41.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:41 smithi152 ceph-mon[93456]: pgmap v1245: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 57523/307254 objects degraded (18.722%); 44326/307254 objects misplaced (14.427%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:35:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57523/307254 objects degraded (18.722%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:41.610 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:35:41.610 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:35:41.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e340/337 cost 1049576)", 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:28.516579+0000", 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 73.079584800999996, 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00065220199999999995, 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:35:41.611 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516579+0000", 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.612 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516579+0000", 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516580+0000", 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8810000000000001e-06 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.613 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516596+0000", 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5826999999999999e-05 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516615+0000", 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9130000000000001e-05 2024-08-17T17:35:41.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516623+0000", 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0339999999999997e-06 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.516692+0000", 2024-08-17T17:35:41.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8096999999999995e-05 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:28.517231+0000", 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00053923299999999996 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e340/337 cost 1049576)", 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:31.394400+0000", 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 70.201762909999999, 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00068497399999999998, 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:35:41.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394400+0000", 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394400+0000", 2024-08-17T17:35:41.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394402+0000", 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2619999999999999e-06 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:35:41.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394424+0000", 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2271000000000001e-05 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394446+0000", 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2242000000000002e-05 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394455+0000", 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2240000000000001e-06 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.394536+0000", 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.085e-05 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:31.395085+0000", 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00054912500000000003 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e348/337 cost 1049576)", 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:35:08.036913+0000", 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 33.559250556999999, 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00065308399999999996, 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:35:41.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.036913+0000", 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.036913+0000", 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.036914+0000", 2024-08-17T17:35:41.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4100000000000001e-06 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.036931+0000", 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6616000000000002e-05 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.036947+0000", 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6520000000000001e-05 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:35:41.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.036955+0000", 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1589999999999993e-06 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.037024+0000", 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8693000000000001e-05 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:08.037566+0000", 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00054168599999999999 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.626 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e348/337 cost 1049576)", 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:35:09.012274+0000", 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 32.583889378000002, 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00063541200000000002, 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:35:41.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012274+0000", 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012274+0000", 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012276+0000", 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.685e-06 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012291+0000", 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5586e-05 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012309+0000", 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7910999999999998e-05 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012319+0000", 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5719999999999992e-06 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012388+0000", 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9540000000000002e-05 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:09.012909+0000", 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00052111799999999999 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e352/337 cost 1049576)", 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:35:41.277483+0000", 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.31868077500000003, 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046190100000000002, 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277483+0000", 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277483+0000", 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277484+0000", 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2920000000000001e-06 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277500+0000", 2024-08-17T17:35:41.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6022e-05 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277516+0000", 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6104999999999999e-05 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277523+0000", 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4599999999999997e-06 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:35:41.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277588+0000", 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4290999999999998e-05 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:41.277944+0000", 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000356731 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:35:41.634 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:35:41.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:41.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:41.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:41.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:41.865 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:35:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:35:41] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:35:41.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:41.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:42.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:42.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:42.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:42.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:42.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:42.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:42.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:42.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:42.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:42.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:42.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:42.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:42.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:42.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:42.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:42.736 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:35:42.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:35:42.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:42.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:42.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:42.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:43.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:43.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:43.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:35:43.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:43.183 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:43.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:43.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:43.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:43.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:43.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:43.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:43.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:43 smithi142 ceph-mon[106729]: pgmap v1246: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57474/305994 objects degraded (18.783%); 43789/305994 objects misplaced (14.310%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:35:43.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:43 smithi142 ceph-mon[103614]: pgmap v1246: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57474/305994 objects degraded (18.783%); 43789/305994 objects misplaced (14.310%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:35:43.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:43.677 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:43.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:43.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:43 smithi152 ceph-mon[93456]: pgmap v1246: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57474/305994 objects degraded (18.783%); 43789/305994 objects misplaced (14.310%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:35:43.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:43.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:44.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:44.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:44.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:44.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:44.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:44.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:44.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:44.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:44.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:44.422 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:44.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:44.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:44.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:44.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:44.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:44.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:44.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:44.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:44.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:44.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:45.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:45.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:45.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:45.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:45.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:45.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:45.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:45.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:45.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:45.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:45.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:45.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:45 smithi142 ceph-mon[103614]: pgmap v1247: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 57474/305718 objects degraded (18.800%); 43789/305718 objects misplaced (14.323%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:35:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:45 smithi142 ceph-mon[106729]: pgmap v1247: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 57474/305718 objects degraded (18.800%); 43789/305718 objects misplaced (14.323%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:35:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:45.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:45.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:45.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:45 smithi152 ceph-mon[93456]: pgmap v1247: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 57474/305718 objects degraded (18.800%); 43789/305718 objects misplaced (14.323%); 5.4 MiB/s, 86 objects/s recovering 2024-08-17T17:35:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:45.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:45.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:45.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:45.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:46.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:46.100 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:46.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:46.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:46.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:46.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:46.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:46.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:46.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57474/305718 objects degraded (18.800%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:46 smithi152 ceph-mon[93456]: pgmap v1248: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 57451/305337 objects degraded (18.816%); 43321/305337 objects misplaced (14.188%); 5.6 MiB/s, 89 objects/s recovering 2024-08-17T17:35:46.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:46.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:46.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:46.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:46.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:46.848 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:46.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57474/305718 objects degraded (18.800%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[106729]: pgmap v1248: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 57451/305337 objects degraded (18.816%); 43321/305337 objects misplaced (14.188%); 5.6 MiB/s, 89 objects/s recovering 2024-08-17T17:35:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57474/305718 objects degraded (18.800%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:46.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:46 smithi142 ceph-mon[103614]: pgmap v1248: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 57451/305337 objects degraded (18.816%); 43321/305337 objects misplaced (14.188%); 5.6 MiB/s, 89 objects/s recovering 2024-08-17T17:35:46.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:46.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:47.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:47.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:47.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:47.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:47.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:47.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:47.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:47.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:47.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:47.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:47.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:47.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:35:47.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:47.724 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:47.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:47.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:47.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:47.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:47.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:47.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:48.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:48.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:48.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:48.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:48.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:48.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:48.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:48.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:48.476 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:48.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:48.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:48.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:48.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:48.837 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:48.852 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:48.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:48.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:48.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:49.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:49.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:49.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:49.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:49.302 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:35:49.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:49.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:49.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:49.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:49.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:49 smithi152 ceph-mon[93456]: pgmap v1249: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 57430/304644 objects degraded (18.852%); 43250/304644 objects misplaced (14.197%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:35:49.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/407309814' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:49.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:49.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:49 smithi142 ceph-mon[106729]: pgmap v1249: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 57430/304644 objects degraded (18.852%); 43250/304644 objects misplaced (14.197%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:35:49.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/407309814' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:49 smithi142 ceph-mon[103614]: pgmap v1249: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 57430/304644 objects degraded (18.852%); 43250/304644 objects misplaced (14.197%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:35:49.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/407309814' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:49.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:49.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:49.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:49.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:49.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:49.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:49.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:49.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:50.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:50.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:50.220 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:50.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:50.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:50.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:50.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:50.468 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:50.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:50.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:50 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/407309814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:50 smithi152 ceph-mon[93456]: osdmap e353: 8 total, 8 up, 6 in 2024-08-17T17:35:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:50.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:50 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:35:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:35:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:50.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:50.615 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:50.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/407309814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[106729]: osdmap e353: 8 total, 8 up, 6 in 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/407309814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[103614]: osdmap e353: 8 total, 8 up, 6 in 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:50.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:50 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:35:50.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:50.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:50.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:50.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:50.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:50.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:50.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:50.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:51.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:51.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:51.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:51.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:51.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:51.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:51 smithi142 ceph-mon[106729]: pgmap v1251: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 57430/304644 objects degraded (18.852%); 43250/304644 objects misplaced (14.197%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:35:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57430/304644 objects degraded (18.852%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:51 smithi142 ceph-mon[103614]: pgmap v1251: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 57430/304644 objects degraded (18.852%); 43250/304644 objects misplaced (14.197%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:35:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57430/304644 objects degraded (18.852%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:51.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:51 smithi152 ceph-mon[93456]: pgmap v1251: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 57430/304644 objects degraded (18.852%); 43250/304644 objects misplaced (14.197%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T17:35:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57430/304644 objects degraded (18.852%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:51.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:51.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.910 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:35:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:35:51] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:35:51.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:51.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:52.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:52.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:52.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:52.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:52.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:52.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:52.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:52.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:52.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:52.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:52.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:52.617 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:52.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:52.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:52.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:52.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:52.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:52.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:52.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:52.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:53.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:53.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.328 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:53.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:53.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:53.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:53.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:53.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:53.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:53 smithi142 ceph-mon[106729]: pgmap v1252: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 57382/303693 objects degraded (18.895%); 42665/303693 objects misplaced (14.049%); 6.1 MiB/s, 96 objects/s recovering 2024-08-17T17:35:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:53 smithi142 ceph-mon[103614]: pgmap v1252: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 57382/303693 objects degraded (18.895%); 42665/303693 objects misplaced (14.049%); 6.1 MiB/s, 96 objects/s recovering 2024-08-17T17:35:53.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:53.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:53.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:53 smithi152 ceph-mon[93456]: pgmap v1252: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 57382/303693 objects degraded (18.895%); 42665/303693 objects misplaced (14.049%); 6.1 MiB/s, 96 objects/s recovering 2024-08-17T17:35:53.838 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:53.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:53.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:53.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:54.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:54.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:54.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:54.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:54.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:54.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:54.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:54.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:54.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:54.482 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:54.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:54.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:54.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:54.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:54.730 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:54.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:54.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:54.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:54.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:54.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:55.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:55.105 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:55.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:55.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:55.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:55.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:35:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:55.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:55.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:55.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:55.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:55.605 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:55.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:55 smithi142 ceph-mon[106729]: pgmap v1253: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 57382/303468 objects degraded (18.909%); 42665/303468 objects misplaced (14.059%); 6.1 MiB/s, 96 objects/s recovering 2024-08-17T17:35:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:55 smithi142 ceph-mon[103614]: pgmap v1253: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 57382/303468 objects degraded (18.909%); 42665/303468 objects misplaced (14.059%); 6.1 MiB/s, 96 objects/s recovering 2024-08-17T17:35:55.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:55.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:55 smithi152 ceph-mon[93456]: pgmap v1253: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 57382/303468 objects degraded (18.909%); 42665/303468 objects misplaced (14.059%); 6.1 MiB/s, 96 objects/s recovering 2024-08-17T17:35:55.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:55.855 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:55.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:55.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:56.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:56.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:56.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:56.227 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:56.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:56.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:56.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:56.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:56.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:56.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57382/303468 objects degraded (18.909%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57382/303468 objects degraded (18.909%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:56.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:56.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:56.769 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:35:56.769 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:35:56.769 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:35:56.769 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:35:56.799 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:35:56.800 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:35:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57382/303468 objects degraded (18.909%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:56.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:56.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:56.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:56.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:56.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:56.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:57.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:57.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:57.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:57.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:57.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:57.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:57.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:57.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:57.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:57.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:57 smithi142 ceph-mon[106729]: pgmap v1254: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 57354/303006 objects degraded (18.928%); 42099/303006 objects misplaced (13.894%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:35:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:57.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:57 smithi142 ceph-mon[103614]: pgmap v1254: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 57354/303006 objects degraded (18.928%); 42099/303006 objects misplaced (13.894%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:35:57.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:57.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:57.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:57 smithi152 ceph-mon[93456]: pgmap v1254: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 57354/303006 objects degraded (18.928%); 42099/303006 objects misplaced (13.894%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:35:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:57.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:57.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:57.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:57.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:58.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:58.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:58.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:58.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:58.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:58.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:58.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:58.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:58.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.552 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:58 smithi142 ceph-mon[106729]: from='client.46937 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:35:58.553 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:58 smithi142 ceph-mon[103614]: from='client.46937 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:35:58.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:58.580 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1254,"stamp":"2024-08-17T17:35:56.249391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6618093541,"num_objects":101002,"num_object_clones":0,"num_object_copies":303006,"num_objects_missing_on_primary":6932,"num_objects_missing":6932,"num_objects_degraded":57354,"num_objects_misplaced":42099,"num_objects_unfound":0,"num_objects_dirty":101002,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":135692,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20347,"num_bytes_recovered":1239491214,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104840,"ondisk_log_size":104840,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18628824,"kb_used_data":13555172,"kb_used_omap":27,"kb_used_meta":5073572,"kb_avail":543842088,"statfs":{"total":575970213888,"available":556894298112,"internally_reserved":0,"allocated":13880496128,"data_stored":13869461897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5195338472},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1,2,1,1,2,3,2,2,4,5],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-50921472,"num_objects":-777,"num_object_clones":0,"num_object_copies":-2331,"num_objects_missing_on_primary":-54,"num_objects_missing":-54,"num_objects_degraded":-97,"num_objects_misplaced":-1222,"num_objects_unfound":0,"num_objects_dirty":-777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":777,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1060,"num_bytes_recovered":69468160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002707"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":546,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306673+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:35:49.306673+0000","last_peered":"2024-08-17T17:35:49.306673+0000","last_clean":"2024-08-17T17:35:49.306673+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:35:49.306673+0000","last_undegraded":"2024-08-17T17:35:49.306673+0000","last_fullsized":"2024-08-17T17:35:49.306673+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":62,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362546+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:35:39.362546+0000","last_peered":"2024-08-17T17:35:39.362546+0000","last_clean":"2024-08-17T17:35:39.362546+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:35:39.362546+0000","last_undegraded":"2024-08-17T17:35:39.362546+0000","last_fullsized":"2024-08-17T17:35:39.362546+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":256,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398831+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:35:39.398831+0000","last_peered":"2024-08-17T17:35:39.398831+0000","last_clean":"2024-08-17T17:35:39.398831+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:35:39.398831+0000","last_undegraded":"2024-08-17T17:35:39.398831+0000","last_fullsized":"2024-08-17T17:35:39.398831+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312937+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:35:49.312937+0000","last_peered":"2024-08-17T17:35:49.312937+0000","last_clean":"2024-08-17T17:35:49.312937+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:35:49.312937+0000","last_undegraded":"2024-08-17T17:35:49.312937+0000","last_fullsized":"2024-08-17T17:35:49.312937+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":422,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.310252+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:35:49.310252+0000","last_peered":"2024-08-17T17:35:49.310252+0000","last_clean":"2024-08-17T17:35:49.310252+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:35:49.310252+0000","last_undegraded":"2024-08-17T17:35:49.310252+0000","last_fullsized":"2024-08-17T17:35:49.310252+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":205,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362158+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:35:39.362158+0000","last_peered":"2024-08-17T17:35:39.362158+0000","last_clean":"2024-08-17T17:35:39.362158+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:35:39.362158+0000","last_undegraded":"2024-08-17T17:35:39.362158+0000","last_fullsized":"2024-08-17T17:35:39.362158+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":515,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306667+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:35:49.306667+0000","last_peered":"2024-08-17T17:35:49.306667+0000","last_clean":"2024-08-17T17:35:49.306667+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:35:49.306667+0000","last_undegraded":"2024-08-17T17:35:49.306667+0000","last_fullsized":"2024-08-17T17:35:49.306667+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":373,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.381791+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:35:39.381791+0000","last_peered":"2024-08-17T17:35:39.381791+0000","last_clean":"2024-08-17T17:35:39.381791+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:35:39.381791+0000","last_undegraded":"2024-08-17T17:35:39.381791+0000","last_fullsized":"2024-08-17T17:35:39.381791+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"353'8443","reported_seq":9232,"reported_epoch":353,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:35:55.513774+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:35:55.513774+0000","last_peered":"2024-08-17T17:35:55.513774+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:35:55.513774+0000","last_undegraded":"2024-08-17T17:35:55.513774+0000","last_fullsized":"2024-08-17T17:35:55.513774+0000","mapping_epoch":337,"log_start":"324'1563","ondisk_log_start":"324'1563","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410320896,"num_objects":6261,"num_object_clones":0,"num_object_copies":18783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6261,"num_objects_unfound":0,"num_objects_dirty":6261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8443,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":362,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":6261}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":58,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.408621+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:35:39.408621+0000","last_peered":"2024-08-17T17:35:39.408621+0000","last_clean":"2024-08-17T17:35:39.408621+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:35:39.408621+0000","last_undegraded":"2024-08-17T17:35:39.408621+0000","last_fullsized":"2024-08-17T17:35:39.408621+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.399647+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:35:39.399647+0000","last_peered":"2024-08-17T17:35:39.399647+0000","last_clean":"2024-08-17T17:35:39.399647+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:35:39.399647+0000","last_undegraded":"2024-08-17T17:35:39.399647+0000","last_fullsized":"2024-08-17T17:35:39.399647+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"353'8672","reported_seq":8723,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:35:51.706444+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:35:51.706444+0000","last_peered":"2024-08-17T17:35:51.706444+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:35:51.706444+0000","last_undegraded":"2024-08-17T17:35:51.706444+0000","last_fullsized":"2024-08-17T17:35:51.706444+0000","mapping_epoch":337,"log_start":"324'2082","ondisk_log_start":"324'2082","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6590,"log_dups_size":0,"ondisk_log_size":6590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419561472,"num_objects":6402,"num_object_clones":0,"num_object_copies":19206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6402,"num_objects_unfound":0,"num_objects_dirty":6402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8672,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":6402}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":60,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.365041+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:35:39.365041+0000","last_peered":"2024-08-17T17:35:39.365041+0000","last_clean":"2024-08-17T17:35:39.365041+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:35:39.365041+0000","last_undegraded":"2024-08-17T17:35:39.365041+0000","last_fullsized":"2024-08-17T17:35:39.365041+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":324,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362101+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:35:39.362101+0000","last_peered":"2024-08-17T17:35:39.362101+0000","last_clean":"2024-08-17T17:35:39.362101+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:35:39.362101+0000","last_undegraded":"2024-08-17T17:35:39.362101+0000","last_fullsized":"2024-08-17T17:35:39.362101+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"353'8421","reported_seq":8475,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:35:55.217167+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:35:55.217167+0000","last_peered":"2024-08-17T17:35:55.217167+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:35:55.217167+0000","last_undegraded":"2024-08-17T17:35:55.217167+0000","last_fullsized":"2024-08-17T17:35:55.217167+0000","mapping_epoch":337,"log_start":"324'1553","ondisk_log_start":"324'1553","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6868,"log_dups_size":0,"ondisk_log_size":6868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413728768,"num_objects":6313,"num_object_clones":0,"num_object_copies":18939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6313,"num_objects_unfound":0,"num_objects_dirty":6313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8421,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":6313}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398759+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:35:39.398759+0000","last_peered":"2024-08-17T17:35:39.398759+0000","last_clean":"2024-08-17T17:35:39.398759+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:35:39.398759+0000","last_undegraded":"2024-08-17T17:35:39.398759+0000","last_fullsized":"2024-08-17T17:35:39.398759+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":334,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.381689+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:35:39.381689+0000","last_peered":"2024-08-17T17:35:39.381689+0000","last_clean":"2024-08-17T17:35:39.381689+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:35:39.381689+0000","last_undegraded":"2024-08-17T17:35:39.381689+0000","last_fullsized":"2024-08-17T17:35:39.381689+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"353'8558","reported_seq":8642,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:52.739973+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:35:52.739973+0000","last_peered":"2024-08-17T17:35:52.739973+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:35:52.739973+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'1513","ondisk_log_start":"324'1513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7045,"log_dups_size":0,"ondisk_log_size":7045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419430400,"num_objects":6400,"num_object_clones":0,"num_object_copies":19200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8558,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-369},{"shards":"0,3,7","objects":6400}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":458,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.307015+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:35:49.307015+0000","last_peered":"2024-08-17T17:35:49.307015+0000","last_clean":"2024-08-17T17:35:49.307015+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:35:49.307015+0000","last_undegraded":"2024-08-17T17:35:49.307015+0000","last_fullsized":"2024-08-17T17:35:49.307015+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.333428+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:35:49.333428+0000","last_peered":"2024-08-17T17:35:49.333428+0000","last_clean":"2024-08-17T17:35:49.333428+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:35:49.333428+0000","last_undegraded":"2024-08-17T17:35:49.333428+0000","last_fullsized":"2024-08-17T17:35:49.333428+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"353'8351","reported_seq":8393,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:52.125771+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:35:52.125771+0000","last_peered":"2024-08-17T17:35:52.125771+0000","last_clean":"2024-08-17T17:35:52.125771+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:35:52.125771+0000","last_undegraded":"2024-08-17T17:35:52.125771+0000","last_fullsized":"2024-08-17T17:35:52.125771+0000","mapping_epoch":319,"log_start":"324'1774","ondisk_log_start":"324'1774","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409141248,"num_objects":6243,"num_object_clones":0,"num_object_copies":18729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8351,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":548,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306478+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:35:49.306478+0000","last_peered":"2024-08-17T17:35:49.306478+0000","last_clean":"2024-08-17T17:35:49.306478+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:35:49.306478+0000","last_undegraded":"2024-08-17T17:35:49.306478+0000","last_fullsized":"2024-08-17T17:35:49.306478+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362923+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:35:39.362923+0000","last_peered":"2024-08-17T17:35:39.362923+0000","last_clean":"2024-08-17T17:35:39.362923+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:35:39.362923+0000","last_undegraded":"2024-08-17T17:35:39.362923+0000","last_fullsized":"2024-08-17T17:35:39.362923+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"353'8499","reported_seq":8551,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:52.056490+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:35:52.056490+0000","last_peered":"2024-08-17T17:35:52.056490+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:35:52.056490+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'1935","ondisk_log_start":"324'1935","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6564,"log_dups_size":0,"ondisk_log_size":6564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421593088,"num_objects":6433,"num_object_clones":0,"num_object_copies":19299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8499,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":6433},{"shards":"1,6","objects":-933}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":308,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.381833+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:35:39.381833+0000","last_peered":"2024-08-17T17:35:39.381833+0000","last_clean":"2024-08-17T17:35:39.381833+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:35:39.381833+0000","last_undegraded":"2024-08-17T17:35:39.381833+0000","last_fullsized":"2024-08-17T17:35:39.381833+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":334,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362602+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:35:39.362602+0000","last_peered":"2024-08-17T17:35:39.362602+0000","last_clean":"2024-08-17T17:35:39.362602+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:35:39.362602+0000","last_undegraded":"2024-08-17T17:35:39.362602+0000","last_fullsized":"2024-08-17T17:35:39.362602+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"353'8568","reported_seq":8652,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:35:51.836389+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:35:51.836389+0000","last_peered":"2024-08-17T17:35:51.836389+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:35:51.836389+0000","last_undegraded":"2024-08-17T17:35:51.836389+0000","last_fullsized":"2024-08-17T17:35:51.836389+0000","mapping_epoch":337,"log_start":"325'2961","ondisk_log_start":"325'2961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5607,"log_dups_size":0,"ondisk_log_size":5607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422182912,"num_objects":6442,"num_object_clones":0,"num_object_copies":19326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6442,"num_objects_unfound":0,"num_objects_dirty":6442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8568,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":6442}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":345,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362238+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:35:39.362238+0000","last_peered":"2024-08-17T17:35:39.362238+0000","last_clean":"2024-08-17T17:35:39.362238+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:35:39.362238+0000","last_undegraded":"2024-08-17T17:35:39.362238+0000","last_fullsized":"2024-08-17T17:35:39.362238+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.365120+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:35:39.365120+0000","last_peered":"2024-08-17T17:35:39.365120+0000","last_clean":"2024-08-17T17:35:39.365120+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:35:39.365120+0000","last_undegraded":"2024-08-17T17:35:39.365120+0000","last_fullsized":"2024-08-17T17:35:39.365120+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"353'8465","reported_seq":9350,"reported_epoch":353,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:35:55.513095+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:35:55.513095+0000","last_peered":"2024-08-17T17:35:55.513095+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:35:55.513095+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:35:55.513095+0000","mapping_epoch":337,"log_start":"324'1641","ondisk_log_start":"324'1641","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408879104,"num_objects":6239,"num_object_clones":0,"num_object_copies":18717,"num_objects_missing_on_primary":6932,"num_objects_missing":6932,"num_objects_degraded":6239,"num_objects_misplaced":6932,"num_objects_unfound":0,"num_objects_dirty":6239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8465,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":26607616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":6226}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":517,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309025+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:35:49.309025+0000","last_peered":"2024-08-17T17:35:49.309025+0000","last_clean":"2024-08-17T17:35:49.309025+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:35:49.309025+0000","last_undegraded":"2024-08-17T17:35:49.309025+0000","last_fullsized":"2024-08-17T17:35:49.309025+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":454,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306754+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:35:49.306754+0000","last_peered":"2024-08-17T17:35:49.306754+0000","last_clean":"2024-08-17T17:35:49.306754+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:35:49.306754+0000","last_undegraded":"2024-08-17T17:35:49.306754+0000","last_fullsized":"2024-08-17T17:35:49.306754+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"353'8528","reported_seq":9261,"reported_epoch":353,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:35:51.811411+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:35:51.811411+0000","last_peered":"2024-08-17T17:35:51.811411+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:35:51.811411+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:35:51.811411+0000","mapping_epoch":337,"log_start":"325'2944","ondisk_log_start":"325'2944","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417267736,"num_objects":6368,"num_object_clones":0,"num_object_copies":19104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6368,"num_objects_misplaced":7118,"num_objects_unfound":0,"num_objects_dirty":6368,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8528,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":330,"num_bytes_recovered":21561344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":6368}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":648,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:49.325001+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:35:49.325001+0000","last_peered":"2024-08-17T17:35:49.325001+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:35:49.325001+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":282,"reported_epoch":352,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:35:39.398668+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:35:39.398668+0000","last_peered":"2024-08-17T17:35:39.398668+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:35:39.398668+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:35:39.398668+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":497,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312813+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:35:49.312813+0000","last_peered":"2024-08-17T17:35:49.312813+0000","last_clean":"2024-08-17T17:35:49.312813+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:35:49.312813+0000","last_undegraded":"2024-08-17T17:35:49.312813+0000","last_fullsized":"2024-08-17T17:35:49.312813+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":317,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382275+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:35:39.382275+0000","last_peered":"2024-08-17T17:35:39.382275+0000","last_clean":"2024-08-17T17:35:39.382275+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:35:39.382275+0000","last_undegraded":"2024-08-17T17:35:39.382275+0000","last_fullsized":"2024-08-17T17:35:39.382275+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"353'8461","reported_seq":8547,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:52.778876+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:35:52.778876+0000","last_peered":"2024-08-17T17:35:52.778876+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:35:52.778876+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1421","ondisk_log_start":"324'1421","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411762688,"num_objects":6283,"num_object_clones":0,"num_object_copies":18849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8461,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-364},{"shards":"0,4,7","objects":6283}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":313,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.311726+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:35:49.311726+0000","last_peered":"2024-08-17T17:35:49.311726+0000","last_clean":"2024-08-17T17:35:49.311726+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:35:49.311726+0000","last_undegraded":"2024-08-17T17:35:49.311726+0000","last_fullsized":"2024-08-17T17:35:49.311726+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":289,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362029+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:35:39.362029+0000","last_peered":"2024-08-17T17:35:39.362029+0000","last_clean":"2024-08-17T17:35:39.362029+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:35:39.362029+0000","last_undegraded":"2024-08-17T17:35:39.362029+0000","last_fullsized":"2024-08-17T17:35:39.362029+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":239,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.325552+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:35:49.325552+0000","last_peered":"2024-08-17T17:35:49.325552+0000","last_clean":"2024-08-17T17:35:49.325552+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:35:49.325552+0000","last_undegraded":"2024-08-17T17:35:49.325552+0000","last_fullsized":"2024-08-17T17:35:49.325552+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"353'8469","reported_seq":8521,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:52.030332+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:35:52.030332+0000","last_peered":"2024-08-17T17:35:52.030332+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:35:52.030332+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'1917","ondisk_log_start":"324'1917","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414646272,"num_objects":6327,"num_object_clones":0,"num_object_copies":18981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8469,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":6327},{"shards":"5,6","objects":-969}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":405,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309820+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:35:49.309820+0000","last_peered":"2024-08-17T17:35:49.309820+0000","last_clean":"2024-08-17T17:35:49.309820+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:35:49.309820+0000","last_undegraded":"2024-08-17T17:35:49.309820+0000","last_fullsized":"2024-08-17T17:35:49.309820+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382004+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:35:39.382004+0000","last_peered":"2024-08-17T17:35:39.382004+0000","last_clean":"2024-08-17T17:35:39.382004+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:35:39.382004+0000","last_undegraded":"2024-08-17T17:35:39.382004+0000","last_fullsized":"2024-08-17T17:35:39.382004+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":589,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.313430+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:35:49.313430+0000","last_peered":"2024-08-17T17:35:49.313430+0000","last_clean":"2024-08-17T17:35:49.313430+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:35:49.313430+0000","last_undegraded":"2024-08-17T17:35:49.313430+0000","last_fullsized":"2024-08-17T17:35:49.313430+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"353'8370","reported_seq":9180,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:52.604660+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:35:52.604660+0000","last_peered":"2024-08-17T17:35:52.604660+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:35:52.604660+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1316","ondisk_log_start":"324'1316","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411303936,"num_objects":6276,"num_object_clones":0,"num_object_copies":18828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8370,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":6276},{"shards":"0,5","objects":-378}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":381,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312366+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:35:49.312366+0000","last_peered":"2024-08-17T17:35:49.312366+0000","last_clean":"2024-08-17T17:35:49.312366+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:35:49.312366+0000","last_undegraded":"2024-08-17T17:35:49.312366+0000","last_fullsized":"2024-08-17T17:35:49.312366+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":328,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362834+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:35:39.362834+0000","last_peered":"2024-08-17T17:35:39.362834+0000","last_clean":"2024-08-17T17:35:39.362834+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:35:39.362834+0000","last_undegraded":"2024-08-17T17:35:39.362834+0000","last_fullsized":"2024-08-17T17:35:39.362834+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":176,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306604+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:35:49.306604+0000","last_peered":"2024-08-17T17:35:49.306604+0000","last_clean":"2024-08-17T17:35:49.306604+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:35:49.306604+0000","last_undegraded":"2024-08-17T17:35:49.306604+0000","last_fullsized":"2024-08-17T17:35:49.306604+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"353'8510","reported_seq":17288,"reported_epoch":353,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:55.513258+0000","last_change":"2024-08-17T17:34:59.006485+0000","last_active":"2024-08-17T17:35:55.513258+0000","last_peered":"2024-08-17T17:35:55.513258+0000","last_clean":"2024-08-17T17:33:19.259922+0000","last_became_active":"2024-08-17T17:33:21.566016+0000","last_became_peered":"2024-08-17T17:33:21.566016+0000","last_unstale":"2024-08-17T17:35:55.513258+0000","last_undegraded":"2024-08-17T17:33:21.324561+0000","last_fullsized":"2024-08-17T17:33:21.272318+0000","mapping_epoch":332,"log_start":"324'2230","ondisk_log_start":"324'2230","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6280,"log_dups_size":0,"ondisk_log_size":6280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414580736,"num_objects":6326,"num_object_clones":0,"num_object_copies":18978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":325,"num_objects_misplaced":2631,"num_objects_unfound":0,"num_objects_dirty":6326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8510,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4346,"num_bytes_recovered":284753920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":2631},{"shards":"0,3","objects":3370}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":267,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398677+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:35:39.398677+0000","last_peered":"2024-08-17T17:35:39.398677+0000","last_clean":"2024-08-17T17:35:39.398677+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:35:39.398677+0000","last_undegraded":"2024-08-17T17:35:39.398677+0000","last_fullsized":"2024-08-17T17:35:39.398677+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":45,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306516+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:35:49.306516+0000","last_peered":"2024-08-17T17:35:49.306516+0000","last_clean":"2024-08-17T17:35:49.306516+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:35:49.306516+0000","last_undegraded":"2024-08-17T17:35:49.306516+0000","last_fullsized":"2024-08-17T17:35:49.306516+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":540,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312759+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:35:49.312759+0000","last_peered":"2024-08-17T17:35:49.312759+0000","last_clean":"2024-08-17T17:35:49.312759+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:35:49.312759+0000","last_undegraded":"2024-08-17T17:35:49.312759+0000","last_fullsized":"2024-08-17T17:35:49.312759+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"353'8256","reported_seq":22753,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:55.238182+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:35:55.238182+0000","last_peered":"2024-08-17T17:35:55.238182+0000","last_clean":"2024-08-17T17:35:55.238182+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:35:55.238182+0000","last_undegraded":"2024-08-17T17:35:55.238182+0000","last_fullsized":"2024-08-17T17:35:55.238182+0000","mapping_epoch":345,"log_start":"324'1972","ondisk_log_start":"324'1972","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8256,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":263,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.399002+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:35:39.399002+0000","last_peered":"2024-08-17T17:35:39.399002+0000","last_clean":"2024-08-17T17:35:39.399002+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:35:39.399002+0000","last_undegraded":"2024-08-17T17:35:39.399002+0000","last_fullsized":"2024-08-17T17:35:39.399002+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":296,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.324768+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:35:49.324768+0000","last_peered":"2024-08-17T17:35:49.324768+0000","last_clean":"2024-08-17T17:35:49.324768+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:35:49.324768+0000","last_undegraded":"2024-08-17T17:35:49.324768+0000","last_fullsized":"2024-08-17T17:35:49.324768+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":405,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398591+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:35:39.398591+0000","last_peered":"2024-08-17T17:35:39.398591+0000","last_clean":"2024-08-17T17:35:39.398591+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:35:39.398591+0000","last_undegraded":"2024-08-17T17:35:39.398591+0000","last_fullsized":"2024-08-17T17:35:39.398591+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"353'8494","reported_seq":8599,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:55.234020+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:35:55.234020+0000","last_peered":"2024-08-17T17:35:55.234020+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:35:55.234020+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2187","ondisk_log_start":"324'2187","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412221440,"num_objects":6290,"num_object_clones":0,"num_object_copies":18870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8494,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":6290},{"shards":"0,3","objects":-958}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":570,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306908+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:35:49.306908+0000","last_peered":"2024-08-17T17:35:49.306908+0000","last_clean":"2024-08-17T17:35:49.306908+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:35:49.306908+0000","last_undegraded":"2024-08-17T17:35:49.306908+0000","last_fullsized":"2024-08-17T17:35:49.306908+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":538,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.314050+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:35:49.314050+0000","last_peered":"2024-08-17T17:35:49.314050+0000","last_clean":"2024-08-17T17:35:49.314050+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:35:49.314050+0000","last_undegraded":"2024-08-17T17:35:49.314050+0000","last_fullsized":"2024-08-17T17:35:49.314050+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":556,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.308828+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:35:49.308828+0000","last_peered":"2024-08-17T17:35:49.308828+0000","last_clean":"2024-08-17T17:35:49.308828+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:35:49.308828+0000","last_undegraded":"2024-08-17T17:35:49.308828+0000","last_fullsized":"2024-08-17T17:35:49.308828+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"353'8349","reported_seq":22999,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:51.926708+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:35:51.926708+0000","last_peered":"2024-08-17T17:35:51.926708+0000","last_clean":"2024-08-17T17:35:51.926708+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:35:51.926708+0000","last_undegraded":"2024-08-17T17:35:51.926708+0000","last_fullsized":"2024-08-17T17:35:51.926708+0000","mapping_epoch":347,"log_start":"324'1772","ondisk_log_start":"324'1772","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407961600,"num_objects":6225,"num_object_clones":0,"num_object_copies":18675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8349,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":229,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.325429+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:35:49.325429+0000","last_peered":"2024-08-17T17:35:49.325429+0000","last_clean":"2024-08-17T17:35:49.325429+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:35:49.325429+0000","last_undegraded":"2024-08-17T17:35:49.325429+0000","last_fullsized":"2024-08-17T17:35:49.325429+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":357,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382393+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:35:39.382393+0000","last_peered":"2024-08-17T17:35:39.382393+0000","last_clean":"2024-08-17T17:35:39.382393+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:35:39.382393+0000","last_undegraded":"2024-08-17T17:35:39.382393+0000","last_fullsized":"2024-08-17T17:35:39.382393+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362198+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:35:39.362198+0000","last_peered":"2024-08-17T17:35:39.362198+0000","last_clean":"2024-08-17T17:35:39.362198+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:35:39.362198+0000","last_undegraded":"2024-08-17T17:35:39.362198+0000","last_fullsized":"2024-08-17T17:35:39.362198+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362776+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:35:39.362776+0000","last_peered":"2024-08-17T17:35:39.362776+0000","last_clean":"2024-08-17T17:35:39.362776+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:35:39.362776+0000","last_undegraded":"2024-08-17T17:35:39.362776+0000","last_fullsized":"2024-08-17T17:35:39.362776+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":350,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.363034+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:35:39.363034+0000","last_peered":"2024-08-17T17:35:39.363034+0000","last_clean":"2024-08-17T17:35:39.363034+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:35:39.363034+0000","last_undegraded":"2024-08-17T17:35:39.363034+0000","last_fullsized":"2024-08-17T17:35:39.363034+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":546,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312525+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:35:49.312525+0000","last_peered":"2024-08-17T17:35:49.312525+0000","last_clean":"2024-08-17T17:35:49.312525+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:35:49.312525+0000","last_undegraded":"2024-08-17T17:35:49.312525+0000","last_fullsized":"2024-08-17T17:35:49.312525+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":85,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.322716+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:35:49.322716+0000","last_peered":"2024-08-17T17:35:49.322716+0000","last_clean":"2024-08-17T17:35:49.322716+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:35:49.322716+0000","last_undegraded":"2024-08-17T17:35:49.322716+0000","last_fullsized":"2024-08-17T17:35:49.322716+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398613+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:35:39.398613+0000","last_peered":"2024-08-17T17:35:39.398613+0000","last_clean":"2024-08-17T17:35:39.398613+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:35:39.398613+0000","last_undegraded":"2024-08-17T17:35:39.398613+0000","last_fullsized":"2024-08-17T17:35:39.398613+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":503,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309185+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:35:49.309185+0000","last_peered":"2024-08-17T17:35:49.309185+0000","last_clean":"2024-08-17T17:35:49.309185+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:35:49.309185+0000","last_undegraded":"2024-08-17T17:35:49.309185+0000","last_fullsized":"2024-08-17T17:35:49.309185+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":401,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.399028+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:35:39.399028+0000","last_peered":"2024-08-17T17:35:39.399028+0000","last_clean":"2024-08-17T17:35:39.399028+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:35:39.399028+0000","last_undegraded":"2024-08-17T17:35:39.399028+0000","last_fullsized":"2024-08-17T17:35:39.399028+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":497,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.307436+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:35:49.307436+0000","last_peered":"2024-08-17T17:35:49.307436+0000","last_clean":"2024-08-17T17:35:49.307436+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:35:49.307436+0000","last_undegraded":"2024-08-17T17:35:49.307436+0000","last_fullsized":"2024-08-17T17:35:49.307436+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":324,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382327+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:35:39.382327+0000","last_peered":"2024-08-17T17:35:39.382327+0000","last_clean":"2024-08-17T17:35:39.382327+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:35:39.382327+0000","last_undegraded":"2024-08-17T17:35:39.382327+0000","last_fullsized":"2024-08-17T17:35:39.382327+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":529,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.308934+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:35:49.308934+0000","last_peered":"2024-08-17T17:35:49.308934+0000","last_clean":"2024-08-17T17:35:49.308934+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:35:49.308934+0000","last_undegraded":"2024-08-17T17:35:49.308934+0000","last_fullsized":"2024-08-17T17:35:49.308934+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":245,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.310116+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:35:49.310116+0000","last_peered":"2024-08-17T17:35:49.310116+0000","last_clean":"2024-08-17T17:35:49.310116+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:35:49.310116+0000","last_undegraded":"2024-08-17T17:35:49.310116+0000","last_fullsized":"2024-08-17T17:35:49.310116+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":537,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312136+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:35:49.312136+0000","last_peered":"2024-08-17T17:35:49.312136+0000","last_clean":"2024-08-17T17:35:49.312136+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:35:49.312136+0000","last_undegraded":"2024-08-17T17:35:49.312136+0000","last_fullsized":"2024-08-17T17:35:49.312136+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":497,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.322119+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:35:49.322119+0000","last_peered":"2024-08-17T17:35:49.322119+0000","last_clean":"2024-08-17T17:35:49.322119+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:35:49.322119+0000","last_undegraded":"2024-08-17T17:35:49.322119+0000","last_fullsized":"2024-08-17T17:35:49.322119+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":336,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362399+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:35:39.362399+0000","last_peered":"2024-08-17T17:35:39.362399+0000","last_clean":"2024-08-17T17:35:39.362399+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:35:39.362399+0000","last_undegraded":"2024-08-17T17:35:39.362399+0000","last_fullsized":"2024-08-17T17:35:39.362399+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306789+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:35:49.306789+0000","last_peered":"2024-08-17T17:35:49.306789+0000","last_clean":"2024-08-17T17:35:49.306789+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:35:49.306789+0000","last_undegraded":"2024-08-17T17:35:49.306789+0000","last_fullsized":"2024-08-17T17:35:49.306789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":242,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.324754+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:35:49.324754+0000","last_peered":"2024-08-17T17:35:49.324754+0000","last_clean":"2024-08-17T17:35:49.324754+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:35:49.324754+0000","last_undegraded":"2024-08-17T17:35:49.324754+0000","last_fullsized":"2024-08-17T17:35:49.324754+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":344,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362705+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:35:39.362705+0000","last_peered":"2024-08-17T17:35:39.362705+0000","last_clean":"2024-08-17T17:35:39.362705+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:35:39.362705+0000","last_undegraded":"2024-08-17T17:35:39.362705+0000","last_fullsized":"2024-08-17T17:35:39.362705+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":207,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362966+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:35:39.362966+0000","last_peered":"2024-08-17T17:35:39.362966+0000","last_clean":"2024-08-17T17:35:39.362966+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:35:39.362966+0000","last_undegraded":"2024-08-17T17:35:39.362966+0000","last_fullsized":"2024-08-17T17:35:39.362966+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":344,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362279+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:35:39.362279+0000","last_peered":"2024-08-17T17:35:39.362279+0000","last_clean":"2024-08-17T17:35:39.362279+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:35:39.362279+0000","last_undegraded":"2024-08-17T17:35:39.362279+0000","last_fullsized":"2024-08-17T17:35:39.362279+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":207,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362481+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:35:39.362481+0000","last_peered":"2024-08-17T17:35:39.362481+0000","last_clean":"2024-08-17T17:35:39.362481+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:35:39.362481+0000","last_undegraded":"2024-08-17T17:35:39.362481+0000","last_fullsized":"2024-08-17T17:35:39.362481+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":549,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309912+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:35:49.309912+0000","last_peered":"2024-08-17T17:35:49.309912+0000","last_clean":"2024-08-17T17:35:49.309912+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:35:49.309912+0000","last_undegraded":"2024-08-17T17:35:49.309912+0000","last_fullsized":"2024-08-17T17:35:49.309912+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":522,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.310298+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:35:49.310298+0000","last_peered":"2024-08-17T17:35:49.310298+0000","last_clean":"2024-08-17T17:35:49.310298+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:35:49.310298+0000","last_undegraded":"2024-08-17T17:35:49.310298+0000","last_fullsized":"2024-08-17T17:35:49.310298+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.365625+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:35:39.365625+0000","last_peered":"2024-08-17T17:35:39.365625+0000","last_clean":"2024-08-17T17:35:39.365625+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:35:39.365625+0000","last_undegraded":"2024-08-17T17:35:39.365625+0000","last_fullsized":"2024-08-17T17:35:39.365625+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":244,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.325136+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:35:49.325136+0000","last_peered":"2024-08-17T17:35:49.325136+0000","last_clean":"2024-08-17T17:35:49.325136+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:35:49.325136+0000","last_undegraded":"2024-08-17T17:35:49.325136+0000","last_fullsized":"2024-08-17T17:35:49.325136+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6617497624,"num_objects":100976,"num_object_clones":0,"num_object_copies":302928,"num_objects_missing_on_primary":6932,"num_objects_missing":6932,"num_objects_degraded":57351,"num_objects_misplaced":42099,"num_objects_unfound":0,"num_objects_dirty":100976,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135414,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20306,"num_bytes_recovered":1237123072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21360685056,"data_stored":21360672840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104633,"ondisk_log_size":104633,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170034,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463403,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3720360,"kb_used_data":2826656,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90024792,"statfs":{"total":95995035648,"available":92185387008,"internally_reserved":0,"allocated":2894495744,"data_stored":2892644510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492078,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1197644,"kb_used_data":810628,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92547508,"statfs":{"total":95995035648,"available":94768648192,"internally_reserved":0,"allocated":830083072,"data_stored":828254138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986670,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2498844,"kb_used_data":1606228,"kb_used_omap":1,"kb_used_meta":892606,"kb_avail":91246308,"statfs":{"total":95995035648,"available":93436219392,"internally_reserved":0,"allocated":1644777472,"data_stored":1642937178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914028888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,1,1,3,2,2,4,5],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225497,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4126668,"kb_used_data":3224836,"kb_used_omap":4,"kb_used_meta":901819,"kb_avail":89618484,"statfs":{"total":95995035648,"available":91769327616,"internally_reserved":0,"allocated":3302232064,"data_stored":3300403428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":923463104},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075822,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533714,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3658132,"kb_used_data":2646664,"kb_used_omap":5,"kb_used_meta":1011450,"kb_avail":90087020,"statfs":{"total":95995035648,"available":92249108480,"internally_reserved":0,"allocated":2710183936,"data_stored":2708349978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1035725040},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089691,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3427176,"kb_used_data":2440160,"kb_used_omap":5,"kb_used_meta":987002,"kb_avail":90317976,"statfs":{"total":95995035648,"available":92485607424,"internally_reserved":0,"allocated":2498723840,"data_stored":2496872665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1010690107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2492071936,"data_stored":2492071936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2708541440,"data_stored":2708537368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4552986624,"data_stored":4552982552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3295936512,"data_stored":3295936512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1638858752,"data_stored":1638858752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":824967168,"data_stored":824967168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2888630272,"data_stored":2888630272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2958692352,"data_stored":2958688280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:35:58.584 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:35:58.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:58.649 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:35:58.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:58.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:58.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:58 smithi152 ceph-mon[93456]: from='client.46937 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:35:58.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:58.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:58.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:58.990 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:58.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:59.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:59.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:59.131 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:35:59.131 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:59.153 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:35:59.153 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:59.177 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:35:59.178 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:59.201 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:35:59.201 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:59.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:59.221 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:35:59.221 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:59.245 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:35:59.245 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:59.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:59.271 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:35:59.271 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:59.291 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:35:59.291 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:59.308 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:59.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:35:59.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:35:59.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:59.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:59.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:59.513 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:59.535 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:59.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:59.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:59.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:59.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:59 smithi142 ceph-mon[106729]: pgmap v1255: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57329/302463 objects degraded (18.954%); 42043/302463 objects misplaced (13.900%); 6.6 MiB/s, 106 objects/s recovering 2024-08-17T17:35:59.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:35:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/620855993' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:59 smithi142 ceph-mon[103614]: pgmap v1255: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57329/302463 objects degraded (18.954%); 42043/302463 objects misplaced (13.900%); 6.6 MiB/s, 106 objects/s recovering 2024-08-17T17:35:59.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:35:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/620855993' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:59.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:59.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:59 smithi152 ceph-mon[93456]: pgmap v1255: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 57329/302463 objects degraded (18.954%); 42043/302463 objects misplaced (13.900%); 6.6 MiB/s, 106 objects/s recovering 2024-08-17T17:35:59.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:35:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/620855993' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:59.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:35:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:59.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:59.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.875 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:35:59.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:59.976 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:00.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:00.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:00.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:00.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:00.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:00.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:00.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:00.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:36:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:36:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:00.602 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:00.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:00.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:00.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:00.749 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:00.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:00.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:00.997 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:01.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:01.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:01.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:01.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:01.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:01.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:01.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:01.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:01.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:01.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:01 smithi152 ceph-mon[93456]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57329/302463 objects degraded (18.954%); 42043/302463 objects misplaced (13.900%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:36:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57329/302463 objects degraded (18.954%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:01.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:01.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:01 smithi142 ceph-mon[106729]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57329/302463 objects degraded (18.954%); 42043/302463 objects misplaced (13.900%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:36:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57329/302463 objects degraded (18.954%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:01.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:01 smithi142 ceph-mon[103614]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 57329/302463 objects degraded (18.954%); 42043/302463 objects misplaced (13.900%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:36:01.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57329/302463 objects degraded (18.954%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:01.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:01.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:01.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:01.871 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:01.920 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:36:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:36:01] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:36:01.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:01.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:01.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:02.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:02.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:02.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:02.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:02.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:02.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:02.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:02.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:02.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:02.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:02.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:02.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:02.846 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:02.868 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:02.970 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:02.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:03.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:03.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:03.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:03.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:03.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:03.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:03.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:03 smithi142 ceph-mon[106729]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57272/301500 objects degraded (18.996%); 41378/301500 objects misplaced (13.724%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:36:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:03 smithi142 ceph-mon[103614]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57272/301500 objects degraded (18.996%); 41378/301500 objects misplaced (13.724%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:36:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:03.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:03.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:03.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:03 smithi152 ceph-mon[93456]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 57272/301500 objects degraded (18.996%); 41378/301500 objects misplaced (13.724%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:36:03.848 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:03.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:03.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:04.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:04.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:04.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:04.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:04.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:04.365 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:04.365 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:04.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:04.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:04.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:04.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:04.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:04.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:04.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:04.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:04.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:04.749 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:04.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:04.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:04.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:04.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:04.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:05.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:05.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:36:05.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:05.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:05.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:05.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:05.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:05.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:05 smithi142 ceph-mon[106729]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 67 op/s; 57272/301266 objects degraded (19.010%); 41378/301266 objects misplaced (13.735%); 6.0 MiB/s, 96 objects/s recovering 2024-08-17T17:36:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:05 smithi142 ceph-mon[103614]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 67 op/s; 57272/301266 objects degraded (19.010%); 41378/301266 objects misplaced (13.735%); 6.0 MiB/s, 96 objects/s recovering 2024-08-17T17:36:05.674 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:05.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:05.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:05.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:05.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:05 smithi152 ceph-mon[93456]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 67 op/s; 57272/301266 objects degraded (19.010%); 41378/301266 objects misplaced (13.735%); 6.0 MiB/s, 96 objects/s recovering 2024-08-17T17:36:05.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:05.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:06.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:06.074 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:06.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:06.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:06.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:06.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:06.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:06.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:06.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:06.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:06.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:06.621 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:06.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57272/301266 objects degraded (19.010%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:06.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57272/301266 objects degraded (19.010%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:06.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:06.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:06.722 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:36:06.722 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:36:06.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:06.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57272/301266 objects degraded (19.010%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:06.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:06.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:07.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:07.072 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:07.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:07.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:07.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:07.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:07.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:07.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:07.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:07.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:07.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:07 smithi142 ceph-mon[103614]: pgmap v1259: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 57245/300759 objects degraded (19.034%); 40667/300759 objects misplaced (13.521%); 9.6 MiB/s, 153 objects/s recovering 2024-08-17T17:36:07.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:07 smithi142 ceph-mon[106729]: pgmap v1259: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 57245/300759 objects degraded (19.034%); 40667/300759 objects misplaced (13.521%); 9.6 MiB/s, 153 objects/s recovering 2024-08-17T17:36:07.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:07.695 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:07.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:07 smithi152 ceph-mon[93456]: pgmap v1259: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 57245/300759 objects degraded (19.034%); 40667/300759 objects misplaced (13.521%); 9.6 MiB/s, 153 objects/s recovering 2024-08-17T17:36:07.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:07.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:07.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:08.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:08.074 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:08.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:08.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:08.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:08.240 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1259,"stamp":"2024-08-17T17:36:06.252307+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6569007077,"num_objects":100253,"num_object_clones":0,"num_object_copies":300759,"num_objects_missing_on_primary":6878,"num_objects_missing":6878,"num_objects_degraded":57245,"num_objects_misplaced":40667,"num_objects_unfound":0,"num_objects_dirty":100253,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":136441,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21646,"num_bytes_recovered":1324622478,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104889,"ondisk_log_size":104889,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18584340,"kb_used_data":13514852,"kb_used_omap":27,"kb_used_meta":5069412,"kb_avail":543886572,"statfs":{"total":575970213888,"available":556939849728,"internally_reserved":0,"allocated":13839208448,"data_stored":13828174217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5191078632},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-59179008,"num_objects":-903,"num_object_clones":0,"num_object_copies":-2709,"num_objects_missing_on_primary":-81,"num_objects_missing":-81,"num_objects_degraded":-137,"num_objects_misplaced":-1998,"num_objects_unfound":0,"num_objects_dirty":-903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":903,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1839,"num_bytes_recovered":120520704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003354"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":547,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306705+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:35:49.306705+0000","last_peered":"2024-08-17T17:35:49.306705+0000","last_clean":"2024-08-17T17:35:49.306705+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:35:49.306705+0000","last_undegraded":"2024-08-17T17:35:49.306705+0000","last_fullsized":"2024-08-17T17:35:49.306705+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":62,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362546+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:35:39.362546+0000","last_peered":"2024-08-17T17:35:39.362546+0000","last_clean":"2024-08-17T17:35:39.362546+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:35:39.362546+0000","last_undegraded":"2024-08-17T17:35:39.362546+0000","last_fullsized":"2024-08-17T17:35:39.362546+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":256,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398831+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:35:39.398831+0000","last_peered":"2024-08-17T17:35:39.398831+0000","last_clean":"2024-08-17T17:35:39.398831+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:35:39.398831+0000","last_undegraded":"2024-08-17T17:35:39.398831+0000","last_fullsized":"2024-08-17T17:35:39.398831+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312977+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:35:49.312977+0000","last_peered":"2024-08-17T17:35:49.312977+0000","last_clean":"2024-08-17T17:35:49.312977+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:35:49.312977+0000","last_undegraded":"2024-08-17T17:35:49.312977+0000","last_fullsized":"2024-08-17T17:35:49.312977+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":422,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.310252+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:35:49.310252+0000","last_peered":"2024-08-17T17:35:49.310252+0000","last_clean":"2024-08-17T17:35:49.310252+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:35:49.310252+0000","last_undegraded":"2024-08-17T17:35:49.310252+0000","last_fullsized":"2024-08-17T17:35:49.310252+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":205,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362158+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:35:39.362158+0000","last_peered":"2024-08-17T17:35:39.362158+0000","last_clean":"2024-08-17T17:35:39.362158+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:35:39.362158+0000","last_undegraded":"2024-08-17T17:35:39.362158+0000","last_fullsized":"2024-08-17T17:35:39.362158+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":515,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306667+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:35:49.306667+0000","last_peered":"2024-08-17T17:35:49.306667+0000","last_clean":"2024-08-17T17:35:49.306667+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:35:49.306667+0000","last_undegraded":"2024-08-17T17:35:49.306667+0000","last_fullsized":"2024-08-17T17:35:49.306667+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":373,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.381791+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:35:39.381791+0000","last_peered":"2024-08-17T17:35:39.381791+0000","last_clean":"2024-08-17T17:35:39.381791+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:35:39.381791+0000","last_undegraded":"2024-08-17T17:35:39.381791+0000","last_fullsized":"2024-08-17T17:35:39.381791+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"353'8501","reported_seq":9406,"reported_epoch":353,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:36:05.267309+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:36:05.267309+0000","last_peered":"2024-08-17T17:36:05.267309+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:36:05.267309+0000","last_undegraded":"2024-08-17T17:36:05.267309+0000","last_fullsized":"2024-08-17T17:36:05.267309+0000","mapping_epoch":337,"log_start":"324'1663","ondisk_log_start":"324'1663","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6838,"log_dups_size":0,"ondisk_log_size":6838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406519808,"num_objects":6203,"num_object_clones":0,"num_object_copies":18609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6203,"num_objects_unfound":0,"num_objects_dirty":6203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8501,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":27525120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":6203}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":58,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.408621+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:35:39.408621+0000","last_peered":"2024-08-17T17:35:39.408621+0000","last_clean":"2024-08-17T17:35:39.408621+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:35:39.408621+0000","last_undegraded":"2024-08-17T17:35:39.408621+0000","last_fullsized":"2024-08-17T17:35:39.408621+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.399647+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:35:39.399647+0000","last_peered":"2024-08-17T17:35:39.399647+0000","last_clean":"2024-08-17T17:35:39.399647+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:35:39.399647+0000","last_undegraded":"2024-08-17T17:35:39.399647+0000","last_fullsized":"2024-08-17T17:35:39.399647+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"353'8704","reported_seq":8755,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:02.060275+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:36:02.060275+0000","last_peered":"2024-08-17T17:36:02.060275+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:36:02.060275+0000","last_undegraded":"2024-08-17T17:36:02.060275+0000","last_fullsized":"2024-08-17T17:36:02.060275+0000","mapping_epoch":337,"log_start":"324'2182","ondisk_log_start":"324'2182","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417464320,"num_objects":6370,"num_object_clones":0,"num_object_copies":19110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6370,"num_objects_unfound":0,"num_objects_dirty":6370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8704,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":6370}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":60,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.365041+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:35:39.365041+0000","last_peered":"2024-08-17T17:35:39.365041+0000","last_clean":"2024-08-17T17:35:39.365041+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:35:39.365041+0000","last_undegraded":"2024-08-17T17:35:39.365041+0000","last_fullsized":"2024-08-17T17:35:39.365041+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":324,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362101+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:35:39.362101+0000","last_peered":"2024-08-17T17:35:39.362101+0000","last_clean":"2024-08-17T17:35:39.362101+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:35:39.362101+0000","last_undegraded":"2024-08-17T17:35:39.362101+0000","last_fullsized":"2024-08-17T17:35:39.362101+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"353'8476","reported_seq":8530,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:05.775992+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:36:05.775992+0000","last_peered":"2024-08-17T17:36:05.775992+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:36:05.775992+0000","last_undegraded":"2024-08-17T17:36:05.775992+0000","last_fullsized":"2024-08-17T17:36:05.775992+0000","mapping_epoch":337,"log_start":"324'1653","ondisk_log_start":"324'1653","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410124288,"num_objects":6258,"num_object_clones":0,"num_object_copies":18774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6258,"num_objects_unfound":0,"num_objects_dirty":6258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8476,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":6258}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398759+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:35:39.398759+0000","last_peered":"2024-08-17T17:35:39.398759+0000","last_clean":"2024-08-17T17:35:39.398759+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:35:39.398759+0000","last_undegraded":"2024-08-17T17:35:39.398759+0000","last_fullsized":"2024-08-17T17:35:39.398759+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":334,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.381689+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:35:39.381689+0000","last_peered":"2024-08-17T17:35:39.381689+0000","last_clean":"2024-08-17T17:35:39.381689+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:35:39.381689+0000","last_undegraded":"2024-08-17T17:35:39.381689+0000","last_fullsized":"2024-08-17T17:35:39.381689+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"353'8605","reported_seq":8689,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:02.599636+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:36:02.599636+0000","last_peered":"2024-08-17T17:36:02.599636+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:36:02.599636+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'1613","ondisk_log_start":"324'1613","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416350208,"num_objects":6353,"num_object_clones":0,"num_object_copies":19059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8605,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-416},{"shards":"0,3,7","objects":6353}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":458,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.307015+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:35:49.307015+0000","last_peered":"2024-08-17T17:35:49.307015+0000","last_clean":"2024-08-17T17:35:49.307015+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:35:49.307015+0000","last_undegraded":"2024-08-17T17:35:49.307015+0000","last_fullsized":"2024-08-17T17:35:49.307015+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.333428+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:35:49.333428+0000","last_peered":"2024-08-17T17:35:49.333428+0000","last_clean":"2024-08-17T17:35:49.333428+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:35:49.333428+0000","last_undegraded":"2024-08-17T17:35:49.333428+0000","last_fullsized":"2024-08-17T17:35:49.333428+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"353'8385","reported_seq":8427,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:36:01.877967+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:36:01.877967+0000","last_peered":"2024-08-17T17:36:01.877967+0000","last_clean":"2024-08-17T17:36:01.877967+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:36:01.877967+0000","last_undegraded":"2024-08-17T17:36:01.877967+0000","last_fullsized":"2024-08-17T17:36:01.877967+0000","mapping_epoch":319,"log_start":"324'1774","ondisk_log_start":"324'1774","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406913024,"num_objects":6209,"num_object_clones":0,"num_object_copies":18627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8385,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":548,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306478+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:35:49.306478+0000","last_peered":"2024-08-17T17:35:49.306478+0000","last_clean":"2024-08-17T17:35:49.306478+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:35:49.306478+0000","last_undegraded":"2024-08-17T17:35:49.306478+0000","last_fullsized":"2024-08-17T17:35:49.306478+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362923+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:35:39.362923+0000","last_peered":"2024-08-17T17:35:39.362923+0000","last_clean":"2024-08-17T17:35:39.362923+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:35:39.362923+0000","last_undegraded":"2024-08-17T17:35:39.362923+0000","last_fullsized":"2024-08-17T17:35:39.362923+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"353'8539","reported_seq":8591,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:01.822777+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:36:01.822777+0000","last_peered":"2024-08-17T17:36:01.822777+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:36:01.822777+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'1935","ondisk_log_start":"324'1935","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418971648,"num_objects":6393,"num_object_clones":0,"num_object_copies":19179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8539,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":6393},{"shards":"1,6","objects":-973}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":308,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.381833+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:35:39.381833+0000","last_peered":"2024-08-17T17:35:39.381833+0000","last_clean":"2024-08-17T17:35:39.381833+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:35:39.381833+0000","last_undegraded":"2024-08-17T17:35:39.381833+0000","last_fullsized":"2024-08-17T17:35:39.381833+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":334,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362602+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:35:39.362602+0000","last_peered":"2024-08-17T17:35:39.362602+0000","last_clean":"2024-08-17T17:35:39.362602+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:35:39.362602+0000","last_undegraded":"2024-08-17T17:35:39.362602+0000","last_fullsized":"2024-08-17T17:35:39.362602+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"353'8614","reported_seq":8699,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:01.725545+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:36:01.725545+0000","last_peered":"2024-08-17T17:36:01.725545+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:36:01.725545+0000","last_undegraded":"2024-08-17T17:36:01.725545+0000","last_fullsized":"2024-08-17T17:36:01.725545+0000","mapping_epoch":337,"log_start":"325'2961","ondisk_log_start":"325'2961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5653,"log_dups_size":0,"ondisk_log_size":5653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419168256,"num_objects":6396,"num_object_clones":0,"num_object_copies":19188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6396,"num_objects_unfound":0,"num_objects_dirty":6396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8614,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":6396}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":345,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362238+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:35:39.362238+0000","last_peered":"2024-08-17T17:35:39.362238+0000","last_clean":"2024-08-17T17:35:39.362238+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:35:39.362238+0000","last_undegraded":"2024-08-17T17:35:39.362238+0000","last_fullsized":"2024-08-17T17:35:39.362238+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.365120+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:35:39.365120+0000","last_peered":"2024-08-17T17:35:39.365120+0000","last_clean":"2024-08-17T17:35:39.365120+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:35:39.365120+0000","last_undegraded":"2024-08-17T17:35:39.365120+0000","last_fullsized":"2024-08-17T17:35:39.365120+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"353'8519","reported_seq":9512,"reported_epoch":353,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:05.725711+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:36:05.725711+0000","last_peered":"2024-08-17T17:36:05.725711+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:36:05.725711+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:36:05.725711+0000","mapping_epoch":337,"log_start":"324'1641","ondisk_log_start":"324'1641","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405340160,"num_objects":6185,"num_object_clones":0,"num_object_copies":18555,"num_objects_missing_on_primary":6878,"num_objects_missing":6878,"num_objects_degraded":6185,"num_objects_misplaced":6878,"num_objects_unfound":0,"num_objects_dirty":6185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8519,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":461,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":6172}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":517,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309025+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:35:49.309025+0000","last_peered":"2024-08-17T17:35:49.309025+0000","last_clean":"2024-08-17T17:35:49.309025+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:35:49.309025+0000","last_undegraded":"2024-08-17T17:35:49.309025+0000","last_fullsized":"2024-08-17T17:35:49.309025+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":454,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306754+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:35:49.306754+0000","last_peered":"2024-08-17T17:35:49.306754+0000","last_clean":"2024-08-17T17:35:49.306754+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:35:49.306754+0000","last_undegraded":"2024-08-17T17:35:49.306754+0000","last_fullsized":"2024-08-17T17:35:49.306754+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"353'8583","reported_seq":9427,"reported_epoch":353,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:01.858409+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:36:01.858409+0000","last_peered":"2024-08-17T17:36:01.858409+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:36:01.858409+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:36:01.858409+0000","mapping_epoch":337,"log_start":"325'2944","ondisk_log_start":"325'2944","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413663256,"num_objects":6313,"num_object_clones":0,"num_object_copies":18939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6313,"num_objects_misplaced":7063,"num_objects_unfound":0,"num_objects_dirty":6313,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8583,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":385,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":6313}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":648,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:35:49.325001+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:35:49.325001+0000","last_peered":"2024-08-17T17:35:49.325001+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:35:49.325001+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":282,"reported_epoch":352,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:35:39.398668+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:35:39.398668+0000","last_peered":"2024-08-17T17:35:39.398668+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:35:39.398668+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:35:39.398668+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":497,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312813+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:35:49.312813+0000","last_peered":"2024-08-17T17:35:49.312813+0000","last_clean":"2024-08-17T17:35:49.312813+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:35:49.312813+0000","last_undegraded":"2024-08-17T17:35:49.312813+0000","last_fullsized":"2024-08-17T17:35:49.312813+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":317,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382275+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:35:39.382275+0000","last_peered":"2024-08-17T17:35:39.382275+0000","last_clean":"2024-08-17T17:35:39.382275+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:35:39.382275+0000","last_undegraded":"2024-08-17T17:35:39.382275+0000","last_fullsized":"2024-08-17T17:35:39.382275+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"353'8508","reported_seq":8594,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:02.902606+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:36:02.902606+0000","last_peered":"2024-08-17T17:36:02.902606+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:36:02.902606+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1521","ondisk_log_start":"324'1521","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408682496,"num_objects":6236,"num_object_clones":0,"num_object_copies":18708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8508,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-411},{"shards":"0,4,7","objects":6236}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":313,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.311726+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:35:49.311726+0000","last_peered":"2024-08-17T17:35:49.311726+0000","last_clean":"2024-08-17T17:35:49.311726+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:35:49.311726+0000","last_undegraded":"2024-08-17T17:35:49.311726+0000","last_fullsized":"2024-08-17T17:35:49.311726+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":289,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362029+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:35:39.362029+0000","last_peered":"2024-08-17T17:35:39.362029+0000","last_clean":"2024-08-17T17:35:39.362029+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:35:39.362029+0000","last_undegraded":"2024-08-17T17:35:39.362029+0000","last_fullsized":"2024-08-17T17:35:39.362029+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":239,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.325552+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:35:49.325552+0000","last_peered":"2024-08-17T17:35:49.325552+0000","last_clean":"2024-08-17T17:35:49.325552+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:35:49.325552+0000","last_undegraded":"2024-08-17T17:35:49.325552+0000","last_fullsized":"2024-08-17T17:35:49.325552+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"353'8509","reported_seq":8561,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:02.055605+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:36:02.055605+0000","last_peered":"2024-08-17T17:36:02.055605+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:36:02.055605+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'1917","ondisk_log_start":"324'1917","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6592,"log_dups_size":0,"ondisk_log_size":6592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412024832,"num_objects":6287,"num_object_clones":0,"num_object_copies":18861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":6287},{"shards":"5,6","objects":-1009}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":405,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309820+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:35:49.309820+0000","last_peered":"2024-08-17T17:35:49.309820+0000","last_clean":"2024-08-17T17:35:49.309820+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:35:49.309820+0000","last_undegraded":"2024-08-17T17:35:49.309820+0000","last_fullsized":"2024-08-17T17:35:49.309820+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382004+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:35:39.382004+0000","last_peered":"2024-08-17T17:35:39.382004+0000","last_clean":"2024-08-17T17:35:39.382004+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:35:39.382004+0000","last_undegraded":"2024-08-17T17:35:39.382004+0000","last_fullsized":"2024-08-17T17:35:39.382004+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":590,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.313467+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:35:49.313467+0000","last_peered":"2024-08-17T17:35:49.313467+0000","last_clean":"2024-08-17T17:35:49.313467+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:35:49.313467+0000","last_undegraded":"2024-08-17T17:35:49.313467+0000","last_fullsized":"2024-08-17T17:35:49.313467+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"353'8408","reported_seq":9218,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:02.014832+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:36:02.014832+0000","last_peered":"2024-08-17T17:36:02.014832+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:36:02.014832+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1416","ondisk_log_start":"324'1416","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408813568,"num_objects":6238,"num_object_clones":0,"num_object_copies":18714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8408,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":6238},{"shards":"0,5","objects":-416}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":381,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312366+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:35:49.312366+0000","last_peered":"2024-08-17T17:35:49.312366+0000","last_clean":"2024-08-17T17:35:49.312366+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:35:49.312366+0000","last_undegraded":"2024-08-17T17:35:49.312366+0000","last_fullsized":"2024-08-17T17:35:49.312366+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":328,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362834+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:35:39.362834+0000","last_peered":"2024-08-17T17:35:39.362834+0000","last_clean":"2024-08-17T17:35:39.362834+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:35:39.362834+0000","last_undegraded":"2024-08-17T17:35:39.362834+0000","last_fullsized":"2024-08-17T17:35:39.362834+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":176,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306604+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:35:49.306604+0000","last_peered":"2024-08-17T17:35:49.306604+0000","last_clean":"2024-08-17T17:35:49.306604+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:35:49.306604+0000","last_undegraded":"2024-08-17T17:35:49.306604+0000","last_fullsized":"2024-08-17T17:35:49.306604+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"353'8562","reported_seq":19604,"reported_epoch":353,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:05.527785+0000","last_change":"2024-08-17T17:34:59.006485+0000","last_active":"2024-08-17T17:36:05.527785+0000","last_peered":"2024-08-17T17:36:05.527785+0000","last_clean":"2024-08-17T17:33:19.259922+0000","last_became_active":"2024-08-17T17:33:21.566016+0000","last_became_peered":"2024-08-17T17:33:21.566016+0000","last_unstale":"2024-08-17T17:36:05.527785+0000","last_undegraded":"2024-08-17T17:33:21.324561+0000","last_fullsized":"2024-08-17T17:33:21.272318+0000","mapping_epoch":332,"log_start":"324'2230","ondisk_log_start":"324'2230","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6332,"log_dups_size":0,"ondisk_log_size":6332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411172864,"num_objects":6274,"num_object_clones":0,"num_object_copies":18822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":325,"num_objects_misplaced":1499,"num_objects_unfound":0,"num_objects_dirty":6274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8562,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5478,"num_bytes_recovered":358940672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1499},{"shards":"0,3","objects":4450}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":267,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398677+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:35:39.398677+0000","last_peered":"2024-08-17T17:35:39.398677+0000","last_clean":"2024-08-17T17:35:39.398677+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:35:39.398677+0000","last_undegraded":"2024-08-17T17:35:39.398677+0000","last_fullsized":"2024-08-17T17:35:39.398677+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":45,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306516+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:35:49.306516+0000","last_peered":"2024-08-17T17:35:49.306516+0000","last_clean":"2024-08-17T17:35:49.306516+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:35:49.306516+0000","last_undegraded":"2024-08-17T17:35:49.306516+0000","last_fullsized":"2024-08-17T17:35:49.306516+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":540,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312759+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:35:49.312759+0000","last_peered":"2024-08-17T17:35:49.312759+0000","last_clean":"2024-08-17T17:35:49.312759+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:35:49.312759+0000","last_undegraded":"2024-08-17T17:35:49.312759+0000","last_fullsized":"2024-08-17T17:35:49.312759+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"353'8301","reported_seq":22798,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:36:05.186562+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:36:05.186562+0000","last_peered":"2024-08-17T17:36:05.186562+0000","last_clean":"2024-08-17T17:36:05.186562+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:36:05.186562+0000","last_undegraded":"2024-08-17T17:36:05.186562+0000","last_fullsized":"2024-08-17T17:36:05.186562+0000","mapping_epoch":345,"log_start":"324'1972","ondisk_log_start":"324'1972","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399966208,"num_objects":6103,"num_object_clones":0,"num_object_copies":18309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8301,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":263,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.399002+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:35:39.399002+0000","last_peered":"2024-08-17T17:35:39.399002+0000","last_clean":"2024-08-17T17:35:39.399002+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:35:39.399002+0000","last_undegraded":"2024-08-17T17:35:39.399002+0000","last_fullsized":"2024-08-17T17:35:39.399002+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":296,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.324768+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:35:49.324768+0000","last_peered":"2024-08-17T17:35:49.324768+0000","last_clean":"2024-08-17T17:35:49.324768+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:35:49.324768+0000","last_undegraded":"2024-08-17T17:35:49.324768+0000","last_fullsized":"2024-08-17T17:35:49.324768+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":405,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398591+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:35:39.398591+0000","last_peered":"2024-08-17T17:35:39.398591+0000","last_clean":"2024-08-17T17:35:39.398591+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:35:39.398591+0000","last_undegraded":"2024-08-17T17:35:39.398591+0000","last_fullsized":"2024-08-17T17:35:39.398591+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"353'8560","reported_seq":8666,"reported_epoch":353,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:05.308245+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:36:05.308245+0000","last_peered":"2024-08-17T17:36:05.308245+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:36:05.308245+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2287","ondisk_log_start":"324'2287","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407896064,"num_objects":6224,"num_object_clones":0,"num_object_copies":18672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8560,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":6224},{"shards":"0,3","objects":-1024}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":570,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306908+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:35:49.306908+0000","last_peered":"2024-08-17T17:35:49.306908+0000","last_clean":"2024-08-17T17:35:49.306908+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:35:49.306908+0000","last_undegraded":"2024-08-17T17:35:49.306908+0000","last_fullsized":"2024-08-17T17:35:49.306908+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":539,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.314087+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:35:49.314087+0000","last_peered":"2024-08-17T17:35:49.314087+0000","last_clean":"2024-08-17T17:35:49.314087+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:35:49.314087+0000","last_undegraded":"2024-08-17T17:35:49.314087+0000","last_fullsized":"2024-08-17T17:35:49.314087+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":556,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.308828+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:35:49.308828+0000","last_peered":"2024-08-17T17:35:49.308828+0000","last_clean":"2024-08-17T17:35:49.308828+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:35:49.308828+0000","last_undegraded":"2024-08-17T17:35:49.308828+0000","last_fullsized":"2024-08-17T17:35:49.308828+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"353'8389","reported_seq":23039,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:36:01.830075+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:36:01.830075+0000","last_peered":"2024-08-17T17:36:01.830075+0000","last_clean":"2024-08-17T17:36:01.830075+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:36:01.830075+0000","last_undegraded":"2024-08-17T17:36:01.830075+0000","last_fullsized":"2024-08-17T17:36:01.830075+0000","mapping_epoch":347,"log_start":"324'1772","ondisk_log_start":"324'1772","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405340160,"num_objects":6185,"num_object_clones":0,"num_object_copies":18555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8389,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":229,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.325429+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:35:49.325429+0000","last_peered":"2024-08-17T17:35:49.325429+0000","last_clean":"2024-08-17T17:35:49.325429+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:35:49.325429+0000","last_undegraded":"2024-08-17T17:35:49.325429+0000","last_fullsized":"2024-08-17T17:35:49.325429+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":357,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382393+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:35:39.382393+0000","last_peered":"2024-08-17T17:35:39.382393+0000","last_clean":"2024-08-17T17:35:39.382393+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:35:39.382393+0000","last_undegraded":"2024-08-17T17:35:39.382393+0000","last_fullsized":"2024-08-17T17:35:39.382393+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362198+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:35:39.362198+0000","last_peered":"2024-08-17T17:35:39.362198+0000","last_clean":"2024-08-17T17:35:39.362198+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:35:39.362198+0000","last_undegraded":"2024-08-17T17:35:39.362198+0000","last_fullsized":"2024-08-17T17:35:39.362198+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362776+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:35:39.362776+0000","last_peered":"2024-08-17T17:35:39.362776+0000","last_clean":"2024-08-17T17:35:39.362776+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:35:39.362776+0000","last_undegraded":"2024-08-17T17:35:39.362776+0000","last_fullsized":"2024-08-17T17:35:39.362776+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":350,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.363034+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:35:39.363034+0000","last_peered":"2024-08-17T17:35:39.363034+0000","last_clean":"2024-08-17T17:35:39.363034+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:35:39.363034+0000","last_undegraded":"2024-08-17T17:35:39.363034+0000","last_fullsized":"2024-08-17T17:35:39.363034+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":546,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312525+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:35:49.312525+0000","last_peered":"2024-08-17T17:35:49.312525+0000","last_clean":"2024-08-17T17:35:49.312525+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:35:49.312525+0000","last_undegraded":"2024-08-17T17:35:49.312525+0000","last_fullsized":"2024-08-17T17:35:49.312525+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":86,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.322765+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:35:49.322765+0000","last_peered":"2024-08-17T17:35:49.322765+0000","last_clean":"2024-08-17T17:35:49.322765+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:35:49.322765+0000","last_undegraded":"2024-08-17T17:35:49.322765+0000","last_fullsized":"2024-08-17T17:35:49.322765+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.398613+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:35:39.398613+0000","last_peered":"2024-08-17T17:35:39.398613+0000","last_clean":"2024-08-17T17:35:39.398613+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:35:39.398613+0000","last_undegraded":"2024-08-17T17:35:39.398613+0000","last_fullsized":"2024-08-17T17:35:39.398613+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":504,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309220+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:35:49.309220+0000","last_peered":"2024-08-17T17:35:49.309220+0000","last_clean":"2024-08-17T17:35:49.309220+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:35:49.309220+0000","last_undegraded":"2024-08-17T17:35:49.309220+0000","last_fullsized":"2024-08-17T17:35:49.309220+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":401,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.399028+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:35:39.399028+0000","last_peered":"2024-08-17T17:35:39.399028+0000","last_clean":"2024-08-17T17:35:39.399028+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:35:39.399028+0000","last_undegraded":"2024-08-17T17:35:39.399028+0000","last_fullsized":"2024-08-17T17:35:39.399028+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":498,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.307479+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:35:49.307479+0000","last_peered":"2024-08-17T17:35:49.307479+0000","last_clean":"2024-08-17T17:35:49.307479+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:35:49.307479+0000","last_undegraded":"2024-08-17T17:35:49.307479+0000","last_fullsized":"2024-08-17T17:35:49.307479+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":324,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.382327+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:35:39.382327+0000","last_peered":"2024-08-17T17:35:39.382327+0000","last_clean":"2024-08-17T17:35:39.382327+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:35:39.382327+0000","last_undegraded":"2024-08-17T17:35:39.382327+0000","last_fullsized":"2024-08-17T17:35:39.382327+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":529,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.308934+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:35:49.308934+0000","last_peered":"2024-08-17T17:35:49.308934+0000","last_clean":"2024-08-17T17:35:49.308934+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:35:49.308934+0000","last_undegraded":"2024-08-17T17:35:49.308934+0000","last_fullsized":"2024-08-17T17:35:49.308934+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":245,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.310116+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:35:49.310116+0000","last_peered":"2024-08-17T17:35:49.310116+0000","last_clean":"2024-08-17T17:35:49.310116+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:35:49.310116+0000","last_undegraded":"2024-08-17T17:35:49.310116+0000","last_fullsized":"2024-08-17T17:35:49.310116+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":537,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.312136+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:35:49.312136+0000","last_peered":"2024-08-17T17:35:49.312136+0000","last_clean":"2024-08-17T17:35:49.312136+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:35:49.312136+0000","last_undegraded":"2024-08-17T17:35:49.312136+0000","last_fullsized":"2024-08-17T17:35:49.312136+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":497,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.322119+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:35:49.322119+0000","last_peered":"2024-08-17T17:35:49.322119+0000","last_clean":"2024-08-17T17:35:49.322119+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:35:49.322119+0000","last_undegraded":"2024-08-17T17:35:49.322119+0000","last_fullsized":"2024-08-17T17:35:49.322119+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":336,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362399+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:35:39.362399+0000","last_peered":"2024-08-17T17:35:39.362399+0000","last_clean":"2024-08-17T17:35:39.362399+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:35:39.362399+0000","last_undegraded":"2024-08-17T17:35:39.362399+0000","last_fullsized":"2024-08-17T17:35:39.362399+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.306789+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:35:49.306789+0000","last_peered":"2024-08-17T17:35:49.306789+0000","last_clean":"2024-08-17T17:35:49.306789+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:35:49.306789+0000","last_undegraded":"2024-08-17T17:35:49.306789+0000","last_fullsized":"2024-08-17T17:35:49.306789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":242,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.324754+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:35:49.324754+0000","last_peered":"2024-08-17T17:35:49.324754+0000","last_clean":"2024-08-17T17:35:49.324754+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:35:49.324754+0000","last_undegraded":"2024-08-17T17:35:49.324754+0000","last_fullsized":"2024-08-17T17:35:49.324754+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":344,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362705+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:35:39.362705+0000","last_peered":"2024-08-17T17:35:39.362705+0000","last_clean":"2024-08-17T17:35:39.362705+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:35:39.362705+0000","last_undegraded":"2024-08-17T17:35:39.362705+0000","last_fullsized":"2024-08-17T17:35:39.362705+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":207,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362966+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:35:39.362966+0000","last_peered":"2024-08-17T17:35:39.362966+0000","last_clean":"2024-08-17T17:35:39.362966+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:35:39.362966+0000","last_undegraded":"2024-08-17T17:35:39.362966+0000","last_fullsized":"2024-08-17T17:35:39.362966+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":344,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362279+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:35:39.362279+0000","last_peered":"2024-08-17T17:35:39.362279+0000","last_clean":"2024-08-17T17:35:39.362279+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:35:39.362279+0000","last_undegraded":"2024-08-17T17:35:39.362279+0000","last_fullsized":"2024-08-17T17:35:39.362279+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":207,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.362481+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:35:39.362481+0000","last_peered":"2024-08-17T17:35:39.362481+0000","last_clean":"2024-08-17T17:35:39.362481+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:35:39.362481+0000","last_undegraded":"2024-08-17T17:35:39.362481+0000","last_fullsized":"2024-08-17T17:35:39.362481+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":549,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.309912+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:35:49.309912+0000","last_peered":"2024-08-17T17:35:49.309912+0000","last_clean":"2024-08-17T17:35:49.309912+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:35:49.309912+0000","last_undegraded":"2024-08-17T17:35:49.309912+0000","last_fullsized":"2024-08-17T17:35:49.309912+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":522,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.310298+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:35:49.310298+0000","last_peered":"2024-08-17T17:35:49.310298+0000","last_clean":"2024-08-17T17:35:49.310298+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:35:49.310298+0000","last_undegraded":"2024-08-17T17:35:49.310298+0000","last_fullsized":"2024-08-17T17:35:49.310298+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":307,"reported_epoch":352,"state":"active+clean","last_fresh":"2024-08-17T17:35:39.365625+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:35:39.365625+0000","last_peered":"2024-08-17T17:35:39.365625+0000","last_clean":"2024-08-17T17:35:39.365625+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:35:39.365625+0000","last_undegraded":"2024-08-17T17:35:39.365625+0000","last_fullsized":"2024-08-17T17:35:39.365625+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":244,"reported_epoch":353,"state":"active+clean","last_fresh":"2024-08-17T17:35:49.325136+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:35:49.325136+0000","last_peered":"2024-08-17T17:35:49.325136+0000","last_clean":"2024-08-17T17:35:49.325136+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:35:49.325136+0000","last_undegraded":"2024-08-17T17:35:49.325136+0000","last_fullsized":"2024-08-17T17:35:49.325136+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6568411160,"num_objects":100227,"num_object_clones":0,"num_object_copies":300681,"num_objects_missing_on_primary":6878,"num_objects_missing":6878,"num_objects_degraded":57242,"num_objects_misplaced":40667,"num_objects_unfound":0,"num_objects_dirty":100227,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136163,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21605,"num_bytes_recovered":1322254336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21267951616,"data_stored":21267939400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104682,"ondisk_log_size":104682,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170036,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463405,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3701544,"kb_used_data":2807840,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90043608,"statfs":{"total":95995035648,"available":92204654592,"internally_reserved":0,"allocated":2875228160,"data_stored":2873376926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492080,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191756,"kb_used_data":804740,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92553396,"statfs":{"total":95995035648,"available":94774677504,"internally_reserved":0,"allocated":824053760,"data_stored":822224826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986672,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2483548,"kb_used_data":1590932,"kb_used_omap":1,"kb_used_meta":892606,"kb_avail":91261604,"statfs":{"total":95995035648,"available":93451882496,"internally_reserved":0,"allocated":1629114368,"data_stored":1627274074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914028888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225499,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4105036,"kb_used_data":3199044,"kb_used_omap":4,"kb_used_meta":905979,"kb_avail":89640116,"statfs":{"total":95995035648,"available":91791478784,"internally_reserved":0,"allocated":3275821056,"data_stored":3273992420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":927722944},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075824,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533716,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3693584,"kb_used_data":2690440,"kb_used_omap":5,"kb_used_meta":1003130,"kb_avail":90051568,"statfs":{"total":95995035648,"available":92212805632,"internally_reserved":0,"allocated":2755010560,"data_stored":2753176602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1027205360},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089693,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3408872,"kb_used_data":2421856,"kb_used_omap":5,"kb_used_meta":987002,"kb_avail":90336280,"statfs":{"total":95995035648,"available":92504350720,"internally_reserved":0,"allocated":2479980544,"data_stored":2478129369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1010690107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2473590784,"data_stored":2473590784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2755530752,"data_stored":2755526680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4513665024,"data_stored":4513660952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3269984256,"data_stored":3269984256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1624637440,"data_stored":1624637440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":818610176,"data_stored":818610176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2870018048,"data_stored":2870018048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2941915136,"data_stored":2941911064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:36:08.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:36:08.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:08.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:08.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:08.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:08.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:08.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:08 smithi142 ceph-mon[106729]: from='client.37086 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:08.488 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:08 smithi142 ceph-mon[103614]: from='client.37086 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:08.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:08.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:08.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:08.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:08.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:08.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:08 smithi152 ceph-mon[93456]: from='client.37086 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:08.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:08.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:08.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:08.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:09.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:09.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:09.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:09.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:09.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:09.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:09.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:09.399 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:36:09.426 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:09.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:09.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:09.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:09.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:09.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:09.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:09.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:09.734 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:09 smithi142 ceph-mon[106729]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 57219/299724 objects degraded (19.091%); 40581/299724 objects misplaced (13.539%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T17:36:09.734 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2698884108' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:09.734 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:09 smithi142 ceph-mon[103614]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 57219/299724 objects degraded (19.091%); 40581/299724 objects misplaced (13.539%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T17:36:09.734 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2698884108' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:09.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:09.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:09 smithi152 ceph-mon[93456]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 57219/299724 objects degraded (19.091%); 40581/299724 objects misplaced (13.539%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T17:36:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2698884108' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:09.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:09.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:09.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:10.039 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:10.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:10.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:10.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:10.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:10.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:10.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:10.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:10.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:10.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:10.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:10.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:10.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:36:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2698884108' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: osdmap e354: 8 total, 8 up, 6 in 2024-08-17T17:36:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:10.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:10.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:10 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:10.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:10.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:10.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:10.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:10.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:10.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:10.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2698884108' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: osdmap e354: 8 total, 8 up, 6 in 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2698884108' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: osdmap e354: 8 total, 8 up, 6 in 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:10.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:10.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:10 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:10.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:10.954 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:10.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:11.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:11.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:11.171 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:11.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:11.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:11.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:11.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:11.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:11 smithi152 ceph-mon[93456]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 57219/299724 objects degraded (19.091%); 40581/299724 objects misplaced (13.539%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:36:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57219/299724 objects degraded (19.091%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:11.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:11.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:11 smithi142 ceph-mon[106729]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 57219/299724 objects degraded (19.091%); 40581/299724 objects misplaced (13.539%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:36:11.604 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57219/299724 objects degraded (19.091%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:11.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:11 smithi142 ceph-mon[103614]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 57219/299724 objects degraded (19.091%); 40581/299724 objects misplaced (13.539%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:36:11.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57219/299724 objects degraded (19.091%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:11.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:11.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:11.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:11.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.838 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:11.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:11.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:36:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:36:11] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:36:11.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:11.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:12.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:12.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:12.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:12.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:12.334 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:12.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:12.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:12.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:12.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:12.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:12.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.731 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:12.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:12.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:12.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:12.747 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:12.747 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:12.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:12.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:12.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:12.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:12.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:13.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:36:13.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:13.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:13.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:13.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:13.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:13.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:13.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:13.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:13.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:13.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:13.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:13.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:13.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:13.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:13.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:13.797 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:13 smithi142 ceph-mon[103614]: pgmap v1263: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 57165/298614 objects degraded (19.143%); 39652/298614 objects misplaced (13.279%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:36:13.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:13 smithi142 ceph-mon[106729]: pgmap v1263: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 57165/298614 objects degraded (19.143%); 39652/298614 objects misplaced (13.279%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:36:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:13 smithi152 ceph-mon[93456]: pgmap v1263: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 57165/298614 objects degraded (19.143%); 39652/298614 objects misplaced (13.279%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:36:13.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:13.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:13.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:13.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:13.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:14.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:14.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:14.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:14.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:14.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:14.342 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:14.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:14.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:14.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:14.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:14.571 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:14.572 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:36:14.573 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:14.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:14.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:14.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:14.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:14.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1476849750' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:14 smithi152 ceph-mon[93456]: pgmap v1264: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 57165/298389 objects degraded (19.158%); 39652/298389 objects misplaced (13.289%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:36:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:14.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1476849750' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:14 smithi142 ceph-mon[103614]: pgmap v1264: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 57165/298389 objects degraded (19.158%); 39652/298389 objects misplaced (13.289%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:36:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1476849750' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:14.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:14 smithi142 ceph-mon[106729]: pgmap v1264: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 57165/298389 objects degraded (19.158%); 39652/298389 objects misplaced (13.289%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:36:14.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:14.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.966 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:14.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:15.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:15.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:15.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:15.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:15.213 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:15.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:15.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:15.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:15.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:15.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:15.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:15.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:15.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:15.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:15.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:15.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:15.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:15.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:15.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:15.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:15.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:16.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:16.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:16.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:16.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:16.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:16.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:16.331 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:16.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:16.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:16.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:16.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:16.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:16.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:16.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:16.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:16 smithi152 ceph-mon[93456]: osdmap e355: 8 total, 8 up, 6 in 2024-08-17T17:36:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 57165/298389 objects degraded (19.158%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:16 smithi152 ceph-mon[93456]: pgmap v1266: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 56811/297972 objects degraded (19.066%); 38859/297972 objects misplaced (13.041%); 13 MiB/s, 258 objects/s recovering 2024-08-17T17:36:16.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:16 smithi142 ceph-mon[103614]: osdmap e355: 8 total, 8 up, 6 in 2024-08-17T17:36:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 57165/298389 objects degraded (19.158%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:16 smithi142 ceph-mon[103614]: pgmap v1266: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 56811/297972 objects degraded (19.066%); 38859/297972 objects misplaced (13.041%); 13 MiB/s, 258 objects/s recovering 2024-08-17T17:36:16.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:16 smithi142 ceph-mon[106729]: osdmap e355: 8 total, 8 up, 6 in 2024-08-17T17:36:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 57165/298389 objects degraded (19.158%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:16 smithi142 ceph-mon[106729]: pgmap v1266: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 56811/297972 objects degraded (19.066%); 38859/297972 objects misplaced (13.041%); 13 MiB/s, 258 objects/s recovering 2024-08-17T17:36:16.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:16.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:16.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:17.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:17.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:17.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:17.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:17.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:17.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:17.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:17.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:17.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:17.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:17.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:17.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:17.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:17.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:17.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:17.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:17.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:17.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:17 smithi152 ceph-mon[93456]: osdmap e356: 8 total, 8 up, 6 in 2024-08-17T17:36:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:17 smithi142 ceph-mon[106729]: osdmap e356: 8 total, 8 up, 6 in 2024-08-17T17:36:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:17 smithi142 ceph-mon[103614]: osdmap e356: 8 total, 8 up, 6 in 2024-08-17T17:36:17.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:17.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:17.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:18.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:18.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:18.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:18.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:18.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:18.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:18.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:18.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:18 smithi152 ceph-mon[93456]: pgmap v1268: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 56795/297525 objects degraded (19.089%); 38819/297525 objects misplaced (13.047%); 13 MiB/s, 260 objects/s recovering 2024-08-17T17:36:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:18 smithi142 ceph-mon[106729]: pgmap v1268: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 56795/297525 objects degraded (19.089%); 38819/297525 objects misplaced (13.047%); 13 MiB/s, 260 objects/s recovering 2024-08-17T17:36:18.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:18 smithi142 ceph-mon[103614]: pgmap v1268: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 56795/297525 objects degraded (19.089%); 38819/297525 objects misplaced (13.047%); 13 MiB/s, 260 objects/s recovering 2024-08-17T17:36:18.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:18.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:19.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:19.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:19.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:19.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:19.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:19.431 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:19.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:19.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:19.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:19.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:19.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:19.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:19.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:19.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:20.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:20.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:20.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:20.300 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:20.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:20.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:20.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:20.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:20.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:20.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:36:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:36:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:20.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:20.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:20.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:20.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:20.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:20.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:21.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:21.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:21.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:21.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:21.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:21.290 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:21.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:21.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:21.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:21.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:21.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:21 smithi152 ceph-mon[93456]: pgmap v1269: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 56795/297525 objects degraded (19.089%); 38819/297525 objects misplaced (13.047%); 6.2 MiB/s, 154 objects/s recovering 2024-08-17T17:36:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56795/297525 objects degraded (19.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:21.611 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:21 smithi142 ceph-mon[106729]: pgmap v1269: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 56795/297525 objects degraded (19.089%); 38819/297525 objects misplaced (13.047%); 6.2 MiB/s, 154 objects/s recovering 2024-08-17T17:36:21.611 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56795/297525 objects degraded (19.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:21.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:21 smithi142 ceph-mon[103614]: pgmap v1269: 89 pgs: 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 6.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 56795/297525 objects degraded (19.089%); 38819/297525 objects misplaced (13.047%); 6.2 MiB/s, 154 objects/s recovering 2024-08-17T17:36:21.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56795/297525 objects degraded (19.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:21.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:21.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:21.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:21.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:36:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:36:21] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:36:21.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:21.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:21.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:22.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:22.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:22.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:22.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:22.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:22.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:22.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:22.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:22.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:22.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:22.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:22.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:22.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:22.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:22.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:22.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:22.883 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:22.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:23.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:23.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:23.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:23.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:23.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:23.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:23.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:23.445 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:23 smithi142 ceph-mon[106729]: pgmap v1270: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 56744/296724 objects degraded (19.123%); 38686/296724 objects misplaced (13.038%); 6.7 MiB/s, 162 objects/s recovering 2024-08-17T17:36:23.445 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:23 smithi142 ceph-mon[103614]: pgmap v1270: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 56744/296724 objects degraded (19.123%); 38686/296724 objects misplaced (13.038%); 6.7 MiB/s, 162 objects/s recovering 2024-08-17T17:36:23.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:23.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:23.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:23 smithi152 ceph-mon[93456]: pgmap v1270: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 56744/296724 objects degraded (19.123%); 38686/296724 objects misplaced (13.038%); 6.7 MiB/s, 162 objects/s recovering 2024-08-17T17:36:23.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:23.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:23.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:23.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:23.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:23.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:23.959 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:36:23.959 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 56744/296724 objects degraded (19.123%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1864,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":356,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":98908,"data_bytes":6480861157,"bytes_used":19011006464,"bytes_avail":556959207424,"bytes_total":575970213888,"degraded_objects":56744,"degraded_total":296724,"degraded_ratio":0.19123495234628812,"misplaced_objects":38686,"misplaced_total":296724,"misplaced_ratio":0.13037705072727518,"recovering_objects_per_sec":162,"recovering_bytes_per_sec":7034988,"recovering_keys_per_sec":0,"num_objects_recovered":1300,"num_bytes_recovered":56295424,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":69},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":31,"modified":"2024-08-17T17:36:14.254703+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:36:24.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:24.025 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242765 10.f e273/222 10:f5d57385:::benchmark_data_smithi142_192453_object117017:head v 273'15354, mlcod=273'15348)", 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 582.98394163199998, 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28726092600000003, 2024-08-17T17:36:24.055 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069106+0000", 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069107+0000", 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8599999999999997e-07 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069115+0000", 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7100000000000007e-06 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069126+0000", 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1321000000000001e-05 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069133+0000", 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4790000000000001e-06 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.070023+0000", 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00089035999999999998 2024-08-17T17:36:24.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.070042+0000", 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8760999999999999e-05 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.356328+0000", 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28628575499999998 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.356367+0000", 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9753999999999999e-05 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242762 10.7 e273/257 10:e5d91e03:::benchmark_data_smithi142_192453_object117014:head v 273'13155, mlcod=273'13153)", 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 582.98376962299994, 2024-08-17T17:36:24.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28567881899999997, 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069278+0000", 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069279+0000", 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5199999999999997e-07 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069286+0000", 2024-08-17T17:36:24.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8059999999999997e-06 2024-08-17T17:36:24.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069295+0000", 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9800000000000004e-06 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.069300+0000", 2024-08-17T17:36:24.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4949999999999999e-06 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.071296+0000", 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0019955060000000002 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.071313+0000", 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7452000000000002e-05 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.354873+0000", 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28356008999999999 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.354957+0000", 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3937999999999995e-05 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.068 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242769 10.7 e273/257 10:e215c346:::benchmark_data_smithi142_192453_object117021:head v 273'13156, mlcod=273'13153)", 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 582.97506706199999, 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.281981127, 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077981+0000", 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077982+0000", 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1100000000000005e-07 2024-08-17T17:36:24.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.077991+0000", 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1940000000000004e-06 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078003+0000", 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2264e-05 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078010+0000", 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0110000000000003e-06 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078277+0000", 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000267135 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.078299+0000", 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1202000000000001e-05 2024-08-17T17:36:24.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359932+0000", 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28163336 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359962+0000", 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0150000000000001e-05 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.074 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36309.0:242772 10.14 e273/257 10:2c787a87:::benchmark_data_smithi142_192453_object117024:head v 273'13212, mlcod=273'13210)", 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 582.97189724199995, 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.278431659, 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:36:24.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081151+0000", 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081152+0000", 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0559999999999999e-06 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.081159+0000", 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3180000000000001e-06 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352055+0000", 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27089613600000001 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352070+0000", 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4620999999999999e-05 2024-08-17T17:36:24.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352219+0000", 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149436 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.352240+0000", 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0767e-05 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359528+0000", 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072875090000000002 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:26:41.359582+0000", 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4815999999999998e-05 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.079 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104285 11.9 11:956711b1:::benchmark_data_smithi142_227468_object104284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 481.79872882699999, 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27987187299999999, 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104285 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254319+0000", 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254320+0000", 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.226e-06 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254340+0000", 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9253999999999999e-05 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254348+0000", 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4440000000000008e-06 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.254353+0000", 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3040000000000004e-06 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.257745+0000", 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003391336 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.257867+0000", 2024-08-17T17:36:24.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012196099999999999 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.258003+0000", 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013589099999999999 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.262664+0000", 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046612679999999997 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.267561+0000", 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048971850000000001 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534082+0000", 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26652041300000001 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534142+0000", 2024-08-17T17:36:24.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0158999999999998e-05 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.534191+0000", 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9431999999999997e-05 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104299 11.b 11:dd1aa66c:::benchmark_data_smithi142_227468_object104298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:36:24.086 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 481.78992205100002, 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27979115300000001, 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104299 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263126+0000", 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263127+0000", 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7000000000000004e-07 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263147+0000", 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0576000000000001e-05 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263158+0000", 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0173999999999999e-05 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.263164+0000", 2024-08-17T17:36:24.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7499999999999997e-06 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266586+0000", 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034218400000000002 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266722+0000", 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013612100000000001 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.266854+0000", 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132001 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.274109+0000", 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072550760000000001 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.275555+0000", 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014456219999999999 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542810+0000", 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26725485300000001 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542864+0000", 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3872999999999997e-05 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.542917+0000", 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3497e-05 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104303 11.9 11:982685b7:::benchmark_data_smithi142_227468_object104302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 481.784823351, 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29623361999999998, 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.093 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104303 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:36:24.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268225+0000", 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4900000000000005e-07 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268246+0000", 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0990000000000001e-05 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268255+0000", 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7369999999999995e-06 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268261+0000", 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6749999999999999e-06 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268310+0000", 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9447999999999998e-05 2024-08-17T17:36:24.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268420+0000", 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010979 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.268546+0000", 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012620700000000001 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272696+0000", 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041496550000000004 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.277035+0000", 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043387249999999999 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564360+0000", 2024-08-17T17:36:24.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28732556599999998 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564416+0000", 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5747e-05 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564458+0000", 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2330999999999998e-05 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.100 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104305 11.9 11:9307d714:::benchmark_data_smithi142_227468_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 481.78034848599998, 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29189848299999999, 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.101 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104305 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272700+0000", 2024-08-17T17:36:24.102 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272701+0000", 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1990000000000001e-06 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272724+0000", 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3428999999999999e-05 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272736+0000", 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2177e-05 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.272743+0000", 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8449999999999997e-06 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273130+0000", 2024-08-17T17:36:24.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00038687699999999999 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273244+0000", 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011396900000000001 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.273376+0000", 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000131812 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.281412+0000", 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080358289999999995 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.282800+0000", 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00138784 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564526+0000", 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.281726537 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564568+0000", 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1570999999999999e-05 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.564598+0000", 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0397999999999999e-05 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.107 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:104325 11.9 11:9715431e:::benchmark_data_smithi142_227468_object104324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 481.76202121099999, 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27710011899999998, 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.108 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 104325 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291027+0000", 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291028+0000", 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2929999999999999e-06 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291051+0000", 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2566999999999999e-05 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291064+0000", 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3695e-05 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291071+0000", 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0219999999999999e-06 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291129+0000", 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7970000000000002e-05 2024-08-17T17:36:24.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291263+0000", 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.291405+0000", 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014159700000000001 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.295160+0000", 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037547869999999999 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.295314+0000", 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015397800000000001 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.567971+0000", 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.27265716699999998 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.568083+0000", 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011186799999999999 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:28:22.568127+0000", 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4175000000000003e-05 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162100 11.0 11:02a7f4f2:::benchmark_data_smithi142_227468_object38681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 414.565053611, 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.30841822800000002, 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162100 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.115 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487994+0000", 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.487996+0000", 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2100000000000001e-06 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.488000+0000", 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6040000000000003e-06 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499381+0000", 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011380988999999999 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499402+0000", 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0930000000000001e-05 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499544+0000", 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014133400000000001 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499675+0000", 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013168000000000001 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499788+0000", 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112625 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511105+0000", 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011317159 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511158+0000", 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3236000000000003e-05 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796105+0000", 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28494646400000001 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796225+0000", 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120295 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796413+0000", 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018770200000000001 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.121 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162107 11.0 11:0142b711:::benchmark_data_smithi142_227468_object38688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 414.553591552, 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29756906700000002, 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.122 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162107 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499457+0000", 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499458+0000", 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8780000000000001e-06 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499465+0000", 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8619999999999997e-06 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499477+0000", 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2011000000000001e-05 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.499483+0000", 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4009999999999996e-06 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.504931+0000", 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054487809999999998 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.505217+0000", 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00028554699999999998 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.505550+0000", 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00033330399999999998 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.511178+0000", 2024-08-17T17:36:24.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0056279959999999997 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.512133+0000", 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00095433599999999996 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796542+0000", 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.28440920800000002 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.796893+0000", 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035122400000000002 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.797026+0000", 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132519 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162133 11.b 11:dac48ceb:::benchmark_data_smithi142_227468_object38714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:36:24.128 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 414.52347802700001, 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29427250700000002, 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162133 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.129 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529570+0000", 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529571+0000", 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.308e-06 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.130 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529576+0000", 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0980000000000001e-06 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529588+0000", 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1579999999999999e-05 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529595+0000", 2024-08-17T17:36:24.131 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.613e-06 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529736+0000", 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014141699999999999 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529840+0000", 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103724 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.132 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.529960+0000", 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000120653 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537643+0000", 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0076821010000000002 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537677+0000", 2024-08-17T17:36:24.133 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4820999999999997e-05 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823705+0000", 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.286027592 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823799+0000", 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3799000000000001e-05 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.134 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.823843+0000", 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3801000000000003e-05 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.135 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.36603.0:162136 11.b 11:d77466cb:::benchmark_data_smithi142_227468_object38717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 414.52155868400001, 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29716111499999998, 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.137 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.36603", 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 162136 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531489+0000", 2024-08-17T17:36:24.138 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531491+0000", 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2160000000000001e-06 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531562+0000", 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1204000000000005e-05 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.139 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531574+0000", 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1950000000000001e-05 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.531581+0000", 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8909999999999998e-06 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534124+0000", 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025436669999999999 2024-08-17T17:36:24.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534233+0000", 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010820899999999999 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.534413+0000", 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018054799999999999 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537671+0000", 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032577740000000002 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.537731+0000", 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0050000000000003e-05 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828485+0000", 2024-08-17T17:36:24.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.29075420899999999 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828589+0000", 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103916 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:29:29.828651+0000", 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1481000000000003e-05 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.143 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:122681 12.a 12:56545667:::benchmark_data_smithi142_261410_object4484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:26.570381+0000", 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 177.482666854, 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.30745330999999998, 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.144 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 122681 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570381+0000", 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570381+0000", 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.145 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570383+0000", 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570389+0000", 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1299999999999998e-06 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570403+0000", 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3514e-05 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570410+0000", 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3989999999999998e-06 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.574337+0000", 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039269960000000003 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.574511+0000", 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017386800000000001 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.655496+0000", 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.080985594999999994 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.655606+0000", 2024-08-17T17:36:24.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010913600000000001 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.655706+0000", 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000100478 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.674413+0000", 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018706901000000001 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.149 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.674422+0000", 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7150000000000004e-06 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.871241+0000", 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.196819094 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.871281+0000", 2024-08-17T17:36:24.150 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0666e-05 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.877835+0000", 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0065531230000000001 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:122686 12.a 12:52048b4e:::benchmark_data_smithi142_261410_object4489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:36:24.151 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 177.47772649800001, 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.277351352, 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 122686 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.152 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.09e-07 2024-08-17T17:36:24.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575326+0000", 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9639999999999997e-06 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575337+0000", 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1129e-05 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.154 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575344+0000", 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.81e-06 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.578351+0000", 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030065280000000001 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.578523+0000", 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017180599999999999 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.155 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.578690+0000", 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016701800000000001 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.582865+0000", 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041752320000000001 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.582901+0000", 2024-08-17T17:36:24.156 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5935000000000001e-05 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.847981+0000", 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26508042399999998 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.848017+0000", 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5571999999999998e-05 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.852673+0000", 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046561249999999997 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124366 12.a 12:540a96b7:::benchmark_data_smithi142_261410_object6169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.839486+0000", 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 174.21356244200001, 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.376478849, 2024-08-17T17:36:24.158 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124366 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839486+0000", 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839486+0000", 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839487+0000", 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0419999999999999e-06 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839491+0000", 2024-08-17T17:36:24.160 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2169999999999996e-06 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839502+0000", 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0606e-05 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839507+0000", 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2660000000000001e-06 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839565+0000", 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8396999999999997e-05 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839679+0000", 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011374099999999999 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192319+0000", 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35264041299999999 2024-08-17T17:36:24.162 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192415+0000", 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5812999999999999e-05 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192566+0000", 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000151161 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.210989+0000", 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018422581 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.211004+0000", 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4922e-05 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.211038+0000", 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4196999999999998e-05 2024-08-17T17:36:24.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.211230+0000", 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019243299999999999 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.215964+0000", 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00473406 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131382 12.b 12:d72dad81:::benchmark_data_smithi142_261410_object13185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 84.570618980000006, 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.96633504400000003, 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131382 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.166 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482430+0000", 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5700000000000002e-07 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482433+0000", 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.692e-06 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482445+0000", 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1564e-05 2024-08-17T17:36:24.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482450+0000", 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5159999999999996e-06 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482507+0000", 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7800000000000002e-05 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:36:24.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482519+0000", 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1567e-05 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.436998+0000", 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95447953500000005 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.437123+0000", 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012464299999999999 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.437248+0000", 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012493899999999999 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448394+0000", 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011145649000000001 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.171 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448408+0000", 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4669e-05 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448665+0000", 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000256531 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448725+0000", 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0028999999999997e-05 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448764+0000", 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9252999999999998e-05 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131388 12.b 12:d3e07cb9:::benchmark_data_smithi142_261410_object13191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 84.501793461000005, 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.89768371400000002, 2024-08-17T17:36:24.173 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131388 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.174 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551256+0000", 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5230000000000001e-06 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551262+0000", 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7860000000000002e-06 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551283+0000", 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.084e-05 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551292+0000", 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4839999999999993e-06 2024-08-17T17:36:24.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551356+0000", 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4068999999999999e-05 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551365+0000", 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8510000000000005e-06 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441263+0000", 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88989788199999997 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441385+0000", 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121975 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441494+0000", 2024-08-17T17:36:24.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108819 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448506+0000", 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0070124380000000002 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448539+0000", 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3089000000000001e-05 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448827+0000", 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00028744399999999998 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448896+0000", 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8782000000000007e-05 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448938+0000", 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2731999999999997e-05 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.180 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131408 12.b 12:df15f4a1:::benchmark_data_smithi142_261410_object13211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 84.059591819999994, 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.45568646400000001, 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.181 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131408 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:36:24.182 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993458+0000", 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0509999999999999e-06 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993466+0000", 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5270000000000001e-06 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993479+0000", 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3621000000000001e-05 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993488+0000", 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2369999999999992e-06 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993547+0000", 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9639000000000003e-05 2024-08-17T17:36:24.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993556+0000", 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9940000000000006e-06 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444651+0000", 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.45109483099999997 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444777+0000", 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012631799999999999 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444902+0000", 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000124721 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448979+0000", 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040765480000000002 2024-08-17T17:36:24.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449036+0000", 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7623e-05 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449058+0000", 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1386000000000002e-05 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449104+0000", 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6388999999999999e-05 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449143+0000", 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8578999999999998e-05 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:137199 12.a 12:52e19bf6:::benchmark_data_smithi142_261410_object19002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-08-17T17:36:24.188 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 8.5743394540000004, 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.056001591, 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 137199 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.189 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478710+0000", 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4600000000000003e-07 2024-08-17T17:36:24.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478714+0000", 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7330000000000003e-06 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478724+0000", 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:36:24.191 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478731+0000", 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1169999999999998e-06 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478795+0000", 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4641999999999995e-05 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478804+0000", 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531212+0000", 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.052407855 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531320+0000", 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000107692 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531454+0000", 2024-08-17T17:36:24.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134055 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534306+0000", 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028519069999999999 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534367+0000", 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1266000000000006e-05 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534598+0000", 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000231373 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534651+0000", 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2369e-05 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:36:24.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534710+0000", 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9517e-05 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:36:24.196 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:36:24.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.198 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:24.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:24.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:24.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:24.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:24.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:24.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:24.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:24.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:24.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3547924073' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:36:24.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3547924073' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:36:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3547924073' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:36:24.656 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:24.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:24.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:24.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:24.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:24.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:24.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:24.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:25.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:25.072 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:25.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:25.198 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:25.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:25.323 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:25.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:25.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:25.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:25.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:25.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:36:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:25.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:25 smithi152 ceph-mon[93456]: pgmap v1271: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 56744/296511 objects degraded (19.137%); 38686/296511 objects misplaced (13.047%); 6.1 MiB/s, 147 objects/s recovering 2024-08-17T17:36:25.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:25 smithi142 ceph-mon[103614]: pgmap v1271: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 56744/296511 objects degraded (19.137%); 38686/296511 objects misplaced (13.047%); 6.1 MiB/s, 147 objects/s recovering 2024-08-17T17:36:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:25 smithi142 ceph-mon[106729]: pgmap v1271: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 56744/296511 objects degraded (19.137%); 38686/296511 objects misplaced (13.047%); 6.1 MiB/s, 147 objects/s recovering 2024-08-17T17:36:25.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:25.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:25.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:25.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:25.863 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:25.920 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:25.942 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:26.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:26.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:26.103 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:36:26.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:36:26.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:26.175 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:26.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:26.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:26.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:26.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:26.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56744/296511 objects degraded (19.137%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:26.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56744/296511 objects degraded (19.137%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:26.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:26.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:26.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:26.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56744/296511 objects degraded (19.137%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:26.577 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:26.679 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:26.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:26.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:26.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:26.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:26.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:26.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:26.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:27.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:27.075 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:27.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:27.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:27.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:27.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:27.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:27.446 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:27.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:27.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:27 smithi152 ceph-mon[93456]: pgmap v1272: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 56719/295965 objects degraded (19.164%); 38604/295965 objects misplaced (13.043%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:36:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:27 smithi142 ceph-mon[106729]: pgmap v1272: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 56719/295965 objects degraded (19.164%); 38604/295965 objects misplaced (13.043%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:36:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:27 smithi142 ceph-mon[103614]: pgmap v1272: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 56719/295965 objects degraded (19.164%); 38604/295965 objects misplaced (13.043%); 857 KiB/s, 13 objects/s recovering 2024-08-17T17:36:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:27.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:27.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:27.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:27.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:27.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:27.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:27.941 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:138141 12.f e356/332 12:fc6b0073:::benchmark_data_smithi142_261410_object19944:head v 356'8640, mlcod=322'102)", 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.764808+0000", 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.19830004200000001, 2024-08-17T17:36:27.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.198323261, 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.764808+0000", 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.764808+0000", 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:27.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.764807+0000", 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.764815+0000", 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4610000000000008e-06 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.764829+0000", 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3906e-05 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.764836+0000", 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5119999999999997e-06 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.895379+0000", 2024-08-17T17:36:27.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13054348700000001 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.895401+0000", 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1761000000000001e-05 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:137993 12.7 12:e56b969a:::benchmark_data_smithi142_261410_object19796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:25.163591+0000", 2024-08-17T17:36:27.997 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.799517206, 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7995823479999999, 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137993 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.163591+0000", 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.163591+0000", 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:27.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.163589+0000", 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.163598+0000", 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.268e-06 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.163614+0000", 2024-08-17T17:36:28.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5699999999999999e-05 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.163623+0000", 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8440000000000004e-06 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.343926+0000", 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18030327500000001 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.354286+0000", 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010360016999999999 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.502177+0000", 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14789069199999999 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.511720+0000", 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0095432480000000007 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138091 12.7 12:e0a3a6e1:::benchmark_data_smithi142_261410_object19894:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:26.854166+0000", 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.108942407, 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.109052666, 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138091 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.854166+0000", 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.854166+0000", 2024-08-17T17:36:28.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.854164+0000", 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.854172+0000", 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5199999999999997e-06 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.854188+0000", 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5532999999999998e-05 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.854212+0000", 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3890999999999999e-05 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.901400+0000", 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.047187804 2024-08-17T17:36:28.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.910184+0000", 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087839600000000004 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.991535+0000", 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.081351079000000007 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.006505+0000", 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014970598999999999 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:138146 12.f e356/332 12:fa0630b7:::benchmark_data_smithi142_261410_object19949:head v 356'8641, mlcod=322'102)", 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.943313+0000", 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.019794881, 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019945608, 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.008 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.943313+0000", 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.943313+0000", 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.943312+0000", 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.943322+0000", 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9019999999999994e-06 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.943337+0000", 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4951e-05 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.943344+0000", 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8889999999999996e-06 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.952110+0000", 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087656520000000005 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.952138+0000", 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8792000000000001e-05 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.012 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138056 12.7 12:e87770b5:::benchmark_data_smithi142_261410_object19859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:26.290913+0000", 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.6721951509999999, 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6723783760000002, 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138056 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.290913+0000", 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.290913+0000", 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.290911+0000", 2024-08-17T17:36:28.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.290919+0000", 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6009999999999997e-06 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.290932+0000", 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2517999999999999e-05 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.290937+0000", 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1989999999999997e-06 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.488640+0000", 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.19770285000000001 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.497639+0000", 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089993350000000007 2024-08-17T17:36:28.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.552268+0000", 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.054628881999999997 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.562787+0000", 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010519139 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138077 12.7 12:e6ecc346:::benchmark_data_smithi142_261410_object19880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:26.432885+0000", 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.530223415, 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.53044739, 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138077 2024-08-17T17:36:28.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.432885+0000", 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.432885+0000", 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.432883+0000", 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.432891+0000", 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1300000000000001e-06 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.432906+0000", 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5207e-05 2024-08-17T17:36:28.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.432914+0000", 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2309999999999996e-06 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.499108+0000", 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.066193313000000004 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.508190+0000", 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090824359999999993 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.651221+0000", 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.143031413 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.661275+0000", 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010053324000000001 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.022 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138121 12.7 12:e482113e:::benchmark_data_smithi142_261410_object19924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.526939+0000", 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.43616920799999997, 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.43643311800000001, 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138121 2024-08-17T17:36:28.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.526939+0000", 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.526939+0000", 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.526938+0000", 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.526944+0000", 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8340000000000004e-06 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.526955+0000", 2024-08-17T17:36:28.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1697e-05 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.526960+0000", 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3440000000000003e-06 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.572932+0000", 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.045972651000000003 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.584655+0000", 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011723185000000001 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.679761+0000", 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.095105766999999994 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.689590+0000", 2024-08-17T17:36:28.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098285620000000008 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138106 12.7 12:e727ec4f:::benchmark_data_smithi142_261410_object19909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.018256+0000", 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.944852056, 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94515831100000003, 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.028 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138106 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.018256+0000", 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.018256+0000", 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.018255+0000", 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.018262+0000", 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8979999999999996e-06 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.018276+0000", 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3774999999999999e-05 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.018283+0000", 2024-08-17T17:36:28.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4860000000000002e-06 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.079060+0000", 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.060777580999999997 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.088358+0000", 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092972369999999999 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.184533+0000", 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.096175705 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.201716+0000", 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017183028999999999 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138034 12.7 12:e89c0215:::benchmark_data_smithi142_261410_object19837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:26.025919+0000", 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.937188637, 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9375350980000001, 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138034 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.034 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.025919+0000", 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.025919+0000", 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.025918+0000", 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.025926+0000", 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9529999999999999e-06 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.025940+0000", 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4268e-05 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.025945+0000", 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3149999999999999e-06 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.100213+0000", 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.074267438000000005 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.109100+0000", 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088869009999999991 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.208100+0000", 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.098999728999999995 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.219754+0000", 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011654687 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138080 12.7 12:e379345b:::benchmark_data_smithi142_261410_object19883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:26.449479+0000", 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.513629484, 2024-08-17T17:36:28.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5140169270000001, 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138080 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.449479+0000", 2024-08-17T17:36:28.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.449479+0000", 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.449476+0000", 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.449487+0000", 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0872e-05 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.449505+0000", 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7963000000000001e-05 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.449515+0000", 2024-08-17T17:36:28.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6590000000000002e-06 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.509278+0000", 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.059762675000000001 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.518473+0000", 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091951010000000007 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.601442+0000", 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082969417000000004 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.611641+0000", 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010198481000000001 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138128 12.7 12:edafeae8:::benchmark_data_smithi142_261410_object19931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.582088+0000", 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.38102053800000002, 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.38145395100000001, 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138128 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.582088+0000", 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.582088+0000", 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.582085+0000", 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.582096+0000", 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0234999999999999e-05 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.582110+0000", 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4005999999999999e-05 2024-08-17T17:36:28.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.582116+0000", 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9769999999999997e-06 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.700351+0000", 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11823553000000001 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.712727+0000", 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012375993 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.873375+0000", 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16064784700000001 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.884808+0000", 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011433319000000001 2024-08-17T17:36:28.049 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138009 12.7 12:e00e4385:::benchmark_data_smithi142_261410_object19812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:25.385335+0000", 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.5777727449999999, 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5782476989999998, 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.050 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138009 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.385335+0000", 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.385335+0000", 2024-08-17T17:36:28.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.385334+0000", 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.385343+0000", 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2709999999999998e-06 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.385358+0000", 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4766999999999999e-05 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.385367+0000", 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9500000000000007e-06 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.514417+0000", 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.12905060800000001 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.523005+0000", 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085875919999999998 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.631906+0000", 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10890156500000001 2024-08-17T17:36:28.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.642104+0000", 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010197962 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138083 12.7 12:e0f6337c:::benchmark_data_smithi142_261410_object19886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:26.476377+0000", 2024-08-17T17:36:28.055 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.486730978, 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4872503269999999, 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138083 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.476377+0000", 2024-08-17T17:36:28.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.476377+0000", 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.476375+0000", 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.476383+0000", 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.422e-06 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.476394+0000", 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0463e-05 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.476399+0000", 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9420000000000003e-06 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.664918+0000", 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18851921999999999 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.674666+0000", 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0097474280000000007 2024-08-17T17:36:28.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.775964+0000", 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.101298049 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:26.788459+0000", 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012495195000000001 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138135 12.7 12:e6fc68e2:::benchmark_data_smithi142_261410_object19938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.727319+0000", 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.23578938599999999, 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23634819200000001, 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138135 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.061 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.727319+0000", 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.727319+0000", 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.727317+0000", 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.727326+0000", 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9879999999999993e-06 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.727346+0000", 2024-08-17T17:36:28.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0465000000000001e-05 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.727353+0000", 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2169999999999997e-06 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.953520+0000", 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22616630300000001 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.064 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138111 12.7 12:e5309c94:::benchmark_data_smithi142_261410_object19914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:27.148900+0000", 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.81420835800000002, 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81479839099999996, 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138111 2024-08-17T17:36:28.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.148900+0000", 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.148900+0000", 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.148898+0000", 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.148907+0000", 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4930000000000004e-06 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.148924+0000", 2024-08-17T17:36:28.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6521e-05 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.148933+0000", 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5699999999999999e-06 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.202848+0000", 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.053914796000000001 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.212052+0000", 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092034279999999996 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.309534+0000", 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.097482750000000007 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:27.319185+0000", 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096508979999999998 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:138012 12.7 12:eb269776:::benchmark_data_smithi142_261410_object19815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:25.673895+0000", 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.2892126739999998, 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2898423669999999, 2024-08-17T17:36:28.070 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 138012 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.673895+0000", 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.673895+0000", 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.673893+0000", 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:28.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.673904+0000", 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0197000000000001e-05 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.673922+0000", 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7963999999999999e-05 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.673929+0000", 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3769999999999998e-06 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.765509+0000", 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.091580208999999996 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.780943+0000", 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015434 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.895342+0000", 2024-08-17T17:36:28.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11439883100000001 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:25.906346+0000", 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011003989 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 16 2024-08-17T17:36:28.075 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:36:28.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:28.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:28.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:28.215 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:28.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:28.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:28.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:28.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:28.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:28.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:28.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:28.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:28.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.717 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:28.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:28.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:28.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:28.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:36:28.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:28.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:29.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:29.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:29.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:29.196 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:29.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:29.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:29.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:29.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:29.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:29.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:29.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:29 smithi152 ceph-mon[93456]: pgmap v1273: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56692/295071 objects degraded (19.213%); 38520/295071 objects misplaced (13.054%); 873 KiB/s, 13 objects/s recovering 2024-08-17T17:36:29.586 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:29 smithi142 ceph-mon[106729]: pgmap v1273: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56692/295071 objects degraded (19.213%); 38520/295071 objects misplaced (13.054%); 873 KiB/s, 13 objects/s recovering 2024-08-17T17:36:29.586 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:29 smithi142 ceph-mon[103614]: pgmap v1273: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56692/295071 objects degraded (19.213%); 38520/295071 objects misplaced (13.054%); 873 KiB/s, 13 objects/s recovering 2024-08-17T17:36:29.590 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:29.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:29.692 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:29.714 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:36:29.714 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:29.715 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:29.716 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:29.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:29.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:29.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:29.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:29.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:29.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:29.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:30.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:30.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:30.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:30.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:30.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:30.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:30.335 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:30.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:30.337 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:36:30.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:30.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:30.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:30.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/766872752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:30.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1388583484' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:30.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:36:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:36:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:30.583 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:30.626 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/766872752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:30.626 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1388583484' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:30.626 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:30.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/766872752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:30.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1388583484' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:30.627 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:30.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:30.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:30.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:30.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:30.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:30.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:30.959 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:30.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:31.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:31.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:31.177 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:31.178 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:31.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:31.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:31.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:31.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:31.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:31.411 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[106729]: pgmap v1274: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 56692/295071 objects degraded (19.213%); 38520/295071 objects misplaced (13.054%); 773 KiB/s, 12 objects/s recovering 2024-08-17T17:36:31.411 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:31.411 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:31.411 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[106729]: osdmap e357: 8 total, 8 up, 6 in 2024-08-17T17:36:31.412 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56692/295071 objects degraded (19.213%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:31.412 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[103614]: pgmap v1274: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 56692/295071 objects degraded (19.213%); 38520/295071 objects misplaced (13.054%); 773 KiB/s, 12 objects/s recovering 2024-08-17T17:36:31.412 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:31.412 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:31.412 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[103614]: osdmap e357: 8 total, 8 up, 6 in 2024-08-17T17:36:31.413 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56692/295071 objects degraded (19.213%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:31.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:31.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:31.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:31 smithi152 ceph-mon[93456]: pgmap v1274: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 56692/295071 objects degraded (19.213%); 38520/295071 objects misplaced (13.054%); 773 KiB/s, 12 objects/s recovering 2024-08-17T17:36:31.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:31 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:31.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:31.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:31 smithi152 ceph-mon[93456]: osdmap e357: 8 total, 8 up, 6 in 2024-08-17T17:36:31.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56692/295071 objects degraded (19.213%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:31.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:31.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:31.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:31.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:31.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:31.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:31.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:36:31.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:31.959 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:32.025 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:36:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:36:31] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:36:32.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:32.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:32.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:32.198 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:32.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:32.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:32.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:32.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:32.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:32.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:32.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:32.583 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:32.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:32.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:32.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:32.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:32.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:32.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:33.049 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:33.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:33.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:33.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:33.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:36:33.184 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:36:33.185 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:36:33.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:33.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:33.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:33.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:33.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:33.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:33.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:33.557 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:33 smithi152 ceph-mon[93456]: pgmap v1276: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56641/294282 objects degraded (19.247%); 38390/294282 objects misplaced (13.045%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:33.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:33.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:33 smithi142 ceph-mon[106729]: pgmap v1276: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56641/294282 objects degraded (19.247%); 38390/294282 objects misplaced (13.045%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:33 smithi142 ceph-mon[103614]: pgmap v1276: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56641/294282 objects degraded (19.247%); 38390/294282 objects misplaced (13.045%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:33.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:33.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:33.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:33.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:33.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:33.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:33.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:34.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:34.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:34.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:34.130 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1276,"stamp":"2024-08-17T17:36:32.260024+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6427514853,"num_objects":98094,"num_object_clones":0,"num_object_copies":294282,"num_objects_missing_on_primary":6743,"num_objects_missing":6743,"num_objects_degraded":56641,"num_objects_misplaced":38390,"num_objects_unfound":0,"num_objects_dirty":98094,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":138600,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23962,"num_bytes_recovered":1447502478,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104748,"ondisk_log_size":104748,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18457512,"kb_used_data":13357492,"kb_used_omap":27,"kb_used_meta":5099940,"kb_avail":544013400,"statfs":{"total":575970213888,"available":557069721600,"internally_reserved":0,"allocated":13678071808,"data_stored":13666874320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5222339304},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,1,1,1,3,5,4],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-53346304,"num_objects":-814,"num_object_clones":0,"num_object_copies":-2442,"num_objects_missing_on_primary":-52,"num_objects_missing":-52,"num_objects_degraded":-103,"num_objects_misplaced":-296,"num_objects_unfound":0,"num_objects_dirty":-814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":814,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003199"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":555,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334189+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:36:30.334189+0000","last_peered":"2024-08-17T17:36:30.334189+0000","last_clean":"2024-08-17T17:36:30.334189+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:36:30.334189+0000","last_undegraded":"2024-08-17T17:36:30.334189+0000","last_fullsized":"2024-08-17T17:36:30.334189+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":66,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511190+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:36:16.511190+0000","last_peered":"2024-08-17T17:36:16.511190+0000","last_clean":"2024-08-17T17:36:16.511190+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:36:16.511190+0000","last_undegraded":"2024-08-17T17:36:16.511190+0000","last_fullsized":"2024-08-17T17:36:16.511190+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":259,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609458+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:36:15.609458+0000","last_peered":"2024-08-17T17:36:15.609458+0000","last_clean":"2024-08-17T17:36:15.609458+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:36:15.609458+0000","last_undegraded":"2024-08-17T17:36:15.609458+0000","last_fullsized":"2024-08-17T17:36:15.609458+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334405+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:36:30.334405+0000","last_peered":"2024-08-17T17:36:30.334405+0000","last_clean":"2024-08-17T17:36:30.334405+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:36:30.334405+0000","last_undegraded":"2024-08-17T17:36:30.334405+0000","last_fullsized":"2024-08-17T17:36:30.334405+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":430,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335044+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:36:30.335044+0000","last_peered":"2024-08-17T17:36:30.335044+0000","last_clean":"2024-08-17T17:36:30.335044+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:36:30.335044+0000","last_undegraded":"2024-08-17T17:36:30.335044+0000","last_fullsized":"2024-08-17T17:36:30.335044+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510722+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:36:16.510722+0000","last_peered":"2024-08-17T17:36:16.510722+0000","last_clean":"2024-08-17T17:36:16.510722+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:36:16.510722+0000","last_undegraded":"2024-08-17T17:36:16.510722+0000","last_fullsized":"2024-08-17T17:36:16.510722+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":523,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334572+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:36:30.334572+0000","last_peered":"2024-08-17T17:36:30.334572+0000","last_clean":"2024-08-17T17:36:30.334572+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:36:30.334572+0000","last_undegraded":"2024-08-17T17:36:30.334572+0000","last_fullsized":"2024-08-17T17:36:30.334572+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":377,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.512165+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:36:16.512165+0000","last_peered":"2024-08-17T17:36:16.512165+0000","last_clean":"2024-08-17T17:36:16.512165+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:36:16.512165+0000","last_undegraded":"2024-08-17T17:36:16.512165+0000","last_fullsized":"2024-08-17T17:36:16.512165+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"357'8598","reported_seq":9701,"reported_epoch":357,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:36:30.913041+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:36:30.913041+0000","last_peered":"2024-08-17T17:36:30.913041+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:36:30.913041+0000","last_undegraded":"2024-08-17T17:36:30.913041+0000","last_fullsized":"2024-08-17T17:36:30.913041+0000","mapping_epoch":337,"log_start":"324'1763","ondisk_log_start":"324'1763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6106,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8598,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":517,"num_bytes_recovered":33882112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":6106}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":62,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.589611+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:36:16.589611+0000","last_peered":"2024-08-17T17:36:16.589611+0000","last_clean":"2024-08-17T17:36:16.589611+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:36:16.589611+0000","last_undegraded":"2024-08-17T17:36:16.589611+0000","last_fullsized":"2024-08-17T17:36:16.589611+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":242,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609137+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:36:15.609137+0000","last_peered":"2024-08-17T17:36:15.609137+0000","last_clean":"2024-08-17T17:36:15.609137+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:36:15.609137+0000","last_undegraded":"2024-08-17T17:36:15.609137+0000","last_fullsized":"2024-08-17T17:36:15.609137+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"357'8853","reported_seq":8908,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:31.763811+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:36:31.763811+0000","last_peered":"2024-08-17T17:36:31.763811+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:36:31.763811+0000","last_undegraded":"2024-08-17T17:36:31.763811+0000","last_fullsized":"2024-08-17T17:36:31.763811+0000","mapping_epoch":337,"log_start":"324'2282","ondisk_log_start":"324'2282","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407699456,"num_objects":6221,"num_object_clones":0,"num_object_copies":18663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6221,"num_objects_unfound":0,"num_objects_dirty":6221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8853,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":6221}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":64,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511126+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:36:16.511126+0000","last_peered":"2024-08-17T17:36:16.511126+0000","last_clean":"2024-08-17T17:36:16.511126+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:36:16.511126+0000","last_undegraded":"2024-08-17T17:36:16.511126+0000","last_fullsized":"2024-08-17T17:36:16.511126+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":328,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510645+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:36:16.510645+0000","last_peered":"2024-08-17T17:36:16.510645+0000","last_clean":"2024-08-17T17:36:16.510645+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:36:16.510645+0000","last_undegraded":"2024-08-17T17:36:16.510645+0000","last_fullsized":"2024-08-17T17:36:16.510645+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"357'8587","reported_seq":8645,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:30.817362+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:36:30.817362+0000","last_peered":"2024-08-17T17:36:30.817362+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:36:30.817362+0000","last_undegraded":"2024-08-17T17:36:30.817362+0000","last_fullsized":"2024-08-17T17:36:30.817362+0000","mapping_epoch":337,"log_start":"324'1753","ondisk_log_start":"324'1753","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402849792,"num_objects":6147,"num_object_clones":0,"num_object_copies":18441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6147,"num_objects_unfound":0,"num_objects_dirty":6147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8587,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":6147}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609443+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:36:15.609443+0000","last_peered":"2024-08-17T17:36:15.609443+0000","last_clean":"2024-08-17T17:36:15.609443+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:36:15.609443+0000","last_undegraded":"2024-08-17T17:36:15.609443+0000","last_fullsized":"2024-08-17T17:36:15.609443+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":338,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509457+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:36:16.509457+0000","last_peered":"2024-08-17T17:36:16.509457+0000","last_clean":"2024-08-17T17:36:16.509457+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:36:16.509457+0000","last_undegraded":"2024-08-17T17:36:16.509457+0000","last_fullsized":"2024-08-17T17:36:16.509457+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"356'8742","reported_seq":8832,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:27.942548+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:36:27.942548+0000","last_peered":"2024-08-17T17:36:27.942548+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:36:27.942548+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'1713","ondisk_log_start":"324'1713","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407371776,"num_objects":6216,"num_object_clones":0,"num_object_copies":18648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8742,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-553},{"shards":"0,3,7","objects":6216}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":467,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334361+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:36:30.334361+0000","last_peered":"2024-08-17T17:36:30.334361+0000","last_clean":"2024-08-17T17:36:30.334361+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:36:30.334361+0000","last_undegraded":"2024-08-17T17:36:30.334361+0000","last_fullsized":"2024-08-17T17:36:30.334361+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":231,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.516715+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:36:16.516715+0000","last_peered":"2024-08-17T17:36:16.516715+0000","last_clean":"2024-08-17T17:36:16.516715+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:36:16.516715+0000","last_undegraded":"2024-08-17T17:36:16.516715+0000","last_fullsized":"2024-08-17T17:36:16.516715+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"357'8528","reported_seq":8574,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:31.736742+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:36:31.736742+0000","last_peered":"2024-08-17T17:36:31.736742+0000","last_clean":"2024-08-17T17:36:31.736742+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:36:31.736742+0000","last_undegraded":"2024-08-17T17:36:31.736742+0000","last_fullsized":"2024-08-17T17:36:31.736742+0000","mapping_epoch":319,"log_start":"324'1974","ondisk_log_start":"324'1974","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397541376,"num_objects":6066,"num_object_clones":0,"num_object_copies":18198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8528,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":556,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334164+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:36:30.334164+0000","last_peered":"2024-08-17T17:36:30.334164+0000","last_clean":"2024-08-17T17:36:30.334164+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:36:30.334164+0000","last_undegraded":"2024-08-17T17:36:30.334164+0000","last_fullsized":"2024-08-17T17:36:30.334164+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510457+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:36:16.510457+0000","last_peered":"2024-08-17T17:36:16.510457+0000","last_clean":"2024-08-17T17:36:16.510457+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:36:16.510457+0000","last_undegraded":"2024-08-17T17:36:16.510457+0000","last_fullsized":"2024-08-17T17:36:16.510457+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"357'8697","reported_seq":8753,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:31.752596+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:36:31.752596+0000","last_peered":"2024-08-17T17:36:31.752596+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:36:31.752596+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'2135","ondisk_log_start":"324'2135","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408616960,"num_objects":6235,"num_object_clones":0,"num_object_copies":18705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8697,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":6235},{"shards":"1,6","objects":-1131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":312,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509883+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:36:16.509883+0000","last_peered":"2024-08-17T17:36:16.509883+0000","last_clean":"2024-08-17T17:36:16.509883+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:36:16.509883+0000","last_undegraded":"2024-08-17T17:36:16.509883+0000","last_fullsized":"2024-08-17T17:36:16.509883+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":338,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510278+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:36:16.510278+0000","last_peered":"2024-08-17T17:36:16.510278+0000","last_clean":"2024-08-17T17:36:16.510278+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:36:16.510278+0000","last_undegraded":"2024-08-17T17:36:16.510278+0000","last_fullsized":"2024-08-17T17:36:16.510278+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"357'8756","reported_seq":8849,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:31.450187+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:36:31.450187+0000","last_peered":"2024-08-17T17:36:31.450187+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:36:31.450187+0000","last_undegraded":"2024-08-17T17:36:31.450187+0000","last_fullsized":"2024-08-17T17:36:31.450187+0000","mapping_epoch":337,"log_start":"325'3161","ondisk_log_start":"325'3161","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409862144,"num_objects":6254,"num_object_clones":0,"num_object_copies":18762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6254,"num_objects_unfound":0,"num_objects_dirty":6254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8756,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":6254}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":349,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510850+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:36:16.510850+0000","last_peered":"2024-08-17T17:36:16.510850+0000","last_clean":"2024-08-17T17:36:16.510850+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:36:16.510850+0000","last_undegraded":"2024-08-17T17:36:16.510850+0000","last_fullsized":"2024-08-17T17:36:16.510850+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511203+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:36:16.511203+0000","last_peered":"2024-08-17T17:36:16.511203+0000","last_clean":"2024-08-17T17:36:16.511203+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:36:16.511203+0000","last_undegraded":"2024-08-17T17:36:16.511203+0000","last_fullsized":"2024-08-17T17:36:16.511203+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"357'8654","reported_seq":9921,"reported_epoch":357,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:30.869354+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:36:30.869354+0000","last_peered":"2024-08-17T17:36:30.869354+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:36:30.869354+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:36:30.869354+0000","mapping_epoch":337,"log_start":"324'1741","ondisk_log_start":"324'1741","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396492800,"num_objects":6050,"num_object_clones":0,"num_object_copies":18150,"num_objects_missing_on_primary":6743,"num_objects_missing":6743,"num_objects_degraded":6050,"num_objects_misplaced":6743,"num_objects_unfound":0,"num_objects_dirty":6050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8654,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":596,"num_bytes_recovered":38993920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":6037}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":526,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334144+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:36:30.334144+0000","last_peered":"2024-08-17T17:36:30.334144+0000","last_clean":"2024-08-17T17:36:30.334144+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:36:30.334144+0000","last_undegraded":"2024-08-17T17:36:30.334144+0000","last_fullsized":"2024-08-17T17:36:30.334144+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":463,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334652+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:36:30.334652+0000","last_peered":"2024-08-17T17:36:30.334652+0000","last_clean":"2024-08-17T17:36:30.334652+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:36:30.334652+0000","last_undegraded":"2024-08-17T17:36:30.334652+0000","last_fullsized":"2024-08-17T17:36:30.334652+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"357'8727","reported_seq":9867,"reported_epoch":357,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:31.535109+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:36:31.535109+0000","last_peered":"2024-08-17T17:36:31.535109+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:36:31.535109+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:36:31.535109+0000","mapping_epoch":337,"log_start":"325'3144","ondisk_log_start":"325'3144","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404226072,"num_objects":6169,"num_object_clones":0,"num_object_copies":18507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6169,"num_objects_misplaced":6919,"num_objects_unfound":0,"num_objects_dirty":6169,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8727,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":529,"num_bytes_recovered":34603008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":6169}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":654,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:16.457900+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:36:16.457900+0000","last_peered":"2024-08-17T17:36:16.457900+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:36:16.457900+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":285,"reported_epoch":355,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:36:15.609353+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:36:15.609353+0000","last_peered":"2024-08-17T17:36:15.609353+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:36:15.609353+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:36:15.609353+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":505,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334368+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:36:30.334368+0000","last_peered":"2024-08-17T17:36:30.334368+0000","last_clean":"2024-08-17T17:36:30.334368+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:36:30.334368+0000","last_undegraded":"2024-08-17T17:36:30.334368+0000","last_fullsized":"2024-08-17T17:36:30.334368+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":321,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510470+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:36:16.510470+0000","last_peered":"2024-08-17T17:36:16.510470+0000","last_clean":"2024-08-17T17:36:16.510470+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:36:16.510470+0000","last_undegraded":"2024-08-17T17:36:16.510470+0000","last_fullsized":"2024-08-17T17:36:16.510470+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"356'8648","reported_seq":8741,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:26.850152+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:36:26.850152+0000","last_peered":"2024-08-17T17:36:26.850152+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:36:26.850152+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1621","ondisk_log_start":"324'1621","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7027,"log_dups_size":0,"ondisk_log_size":7027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399507456,"num_objects":6096,"num_object_clones":0,"num_object_copies":18288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-551},{"shards":"0,4,7","objects":6096}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":321,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334311+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:36:30.334311+0000","last_peered":"2024-08-17T17:36:30.334311+0000","last_clean":"2024-08-17T17:36:30.334311+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:36:30.334311+0000","last_undegraded":"2024-08-17T17:36:30.334311+0000","last_fullsized":"2024-08-17T17:36:30.334311+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":293,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509547+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:36:16.509547+0000","last_peered":"2024-08-17T17:36:16.509547+0000","last_clean":"2024-08-17T17:36:16.509547+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:36:16.509547+0000","last_undegraded":"2024-08-17T17:36:16.509547+0000","last_fullsized":"2024-08-17T17:36:16.509547+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":245,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.518386+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:36:16.518386+0000","last_peered":"2024-08-17T17:36:16.518386+0000","last_clean":"2024-08-17T17:36:16.518386+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:36:16.518386+0000","last_undegraded":"2024-08-17T17:36:16.518386+0000","last_fullsized":"2024-08-17T17:36:16.518386+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"357'8660","reported_seq":8716,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:31.728657+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:36:31.728657+0000","last_peered":"2024-08-17T17:36:31.728657+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:36:31.728657+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'2117","ondisk_log_start":"324'2117","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6543,"log_dups_size":0,"ondisk_log_size":6543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402128896,"num_objects":6136,"num_object_clones":0,"num_object_copies":18408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8660,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":6136},{"shards":"5,6","objects":-1160}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":413,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334127+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:36:30.334127+0000","last_peered":"2024-08-17T17:36:30.334127+0000","last_clean":"2024-08-17T17:36:30.334127+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:36:30.334127+0000","last_undegraded":"2024-08-17T17:36:30.334127+0000","last_fullsized":"2024-08-17T17:36:30.334127+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":93,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.512070+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:36:16.512070+0000","last_peered":"2024-08-17T17:36:16.512070+0000","last_clean":"2024-08-17T17:36:16.512070+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:36:16.512070+0000","last_undegraded":"2024-08-17T17:36:16.512070+0000","last_fullsized":"2024-08-17T17:36:16.512070+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":598,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335146+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:36:30.335146+0000","last_peered":"2024-08-17T17:36:30.335146+0000","last_clean":"2024-08-17T17:36:30.335146+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:36:30.335146+0000","last_undegraded":"2024-08-17T17:36:30.335146+0000","last_fullsized":"2024-08-17T17:36:30.335146+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"356'8534","reported_seq":9351,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:26.955123+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:36:26.955123+0000","last_peered":"2024-08-17T17:36:26.955123+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:36:26.955123+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1516","ondisk_log_start":"324'1516","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400556032,"num_objects":6112,"num_object_clones":0,"num_object_copies":18336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8534,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":6112},{"shards":"0,5","objects":-542}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":389,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334928+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:36:30.334928+0000","last_peered":"2024-08-17T17:36:30.334928+0000","last_clean":"2024-08-17T17:36:30.334928+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:36:30.334928+0000","last_undegraded":"2024-08-17T17:36:30.334928+0000","last_fullsized":"2024-08-17T17:36:30.334928+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":332,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511510+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:36:16.511510+0000","last_peered":"2024-08-17T17:36:16.511510+0000","last_clean":"2024-08-17T17:36:16.511510+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:36:16.511510+0000","last_undegraded":"2024-08-17T17:36:16.511510+0000","last_fullsized":"2024-08-17T17:36:16.511510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334102+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:36:30.334102+0000","last_peered":"2024-08-17T17:36:30.334102+0000","last_clean":"2024-08-17T17:36:30.334102+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:36:30.334102+0000","last_undegraded":"2024-08-17T17:36:30.334102+0000","last_fullsized":"2024-08-17T17:36:30.334102+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"357'8688","reported_seq":23626,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334320+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:36:30.334320+0000","last_peered":"2024-08-17T17:36:30.334320+0000","last_clean":"2024-08-17T17:36:30.334320+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:36:30.334320+0000","last_undegraded":"2024-08-17T17:36:30.334320+0000","last_fullsized":"2024-08-17T17:36:30.334320+0000","mapping_epoch":355,"log_start":"324'2430","ondisk_log_start":"324'2430","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8688,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":270,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609383+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:36:15.609383+0000","last_peered":"2024-08-17T17:36:15.609383+0000","last_clean":"2024-08-17T17:36:15.609383+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:36:15.609383+0000","last_undegraded":"2024-08-17T17:36:15.609383+0000","last_fullsized":"2024-08-17T17:36:15.609383+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":53,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334007+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:36:30.334007+0000","last_peered":"2024-08-17T17:36:30.334007+0000","last_clean":"2024-08-17T17:36:30.334007+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:36:30.334007+0000","last_undegraded":"2024-08-17T17:36:30.334007+0000","last_fullsized":"2024-08-17T17:36:30.334007+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":548,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335221+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:36:30.335221+0000","last_peered":"2024-08-17T17:36:30.335221+0000","last_clean":"2024-08-17T17:36:30.335221+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:36:30.335221+0000","last_undegraded":"2024-08-17T17:36:30.335221+0000","last_fullsized":"2024-08-17T17:36:30.335221+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"357'8411","reported_seq":22917,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334057+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:36:30.334057+0000","last_peered":"2024-08-17T17:36:30.334057+0000","last_clean":"2024-08-17T17:36:30.334057+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:36:30.334057+0000","last_undegraded":"2024-08-17T17:36:30.334057+0000","last_fullsized":"2024-08-17T17:36:30.334057+0000","mapping_epoch":345,"log_start":"324'2072","ondisk_log_start":"324'2072","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392757248,"num_objects":5993,"num_object_clones":0,"num_object_copies":17979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8411,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":266,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609531+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:36:15.609531+0000","last_peered":"2024-08-17T17:36:15.609531+0000","last_clean":"2024-08-17T17:36:15.609531+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:36:15.609531+0000","last_undegraded":"2024-08-17T17:36:15.609531+0000","last_fullsized":"2024-08-17T17:36:15.609531+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":302,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.458150+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:36:16.458150+0000","last_peered":"2024-08-17T17:36:16.458150+0000","last_clean":"2024-08-17T17:36:16.458150+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:36:16.458150+0000","last_undegraded":"2024-08-17T17:36:16.458150+0000","last_fullsized":"2024-08-17T17:36:16.458150+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":408,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609296+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:36:15.609296+0000","last_peered":"2024-08-17T17:36:15.609296+0000","last_clean":"2024-08-17T17:36:15.609296+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:36:15.609296+0000","last_undegraded":"2024-08-17T17:36:15.609296+0000","last_fullsized":"2024-08-17T17:36:15.609296+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"357'8689","reported_seq":8803,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:30.343114+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:36:30.343114+0000","last_peered":"2024-08-17T17:36:30.343114+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:36:30.343114+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2387","ondisk_log_start":"324'2387","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6302,"log_dups_size":0,"ondisk_log_size":6302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399441920,"num_objects":6095,"num_object_clones":0,"num_object_copies":18285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8689,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":6095},{"shards":"0,3","objects":-1153}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":579,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334781+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:36:30.334781+0000","last_peered":"2024-08-17T17:36:30.334781+0000","last_clean":"2024-08-17T17:36:30.334781+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:36:30.334781+0000","last_undegraded":"2024-08-17T17:36:30.334781+0000","last_fullsized":"2024-08-17T17:36:30.334781+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":547,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335587+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:36:30.335587+0000","last_peered":"2024-08-17T17:36:30.335587+0000","last_clean":"2024-08-17T17:36:30.335587+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:36:30.335587+0000","last_undegraded":"2024-08-17T17:36:30.335587+0000","last_fullsized":"2024-08-17T17:36:30.335587+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":564,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333892+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:36:30.333892+0000","last_peered":"2024-08-17T17:36:30.333892+0000","last_clean":"2024-08-17T17:36:30.333892+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:36:30.333892+0000","last_undegraded":"2024-08-17T17:36:30.333892+0000","last_fullsized":"2024-08-17T17:36:30.333892+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"357'8550","reported_seq":23204,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:31.278271+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:36:31.278271+0000","last_peered":"2024-08-17T17:36:31.278271+0000","last_clean":"2024-08-17T17:36:31.278271+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:36:31.278271+0000","last_undegraded":"2024-08-17T17:36:31.278271+0000","last_fullsized":"2024-08-17T17:36:31.278271+0000","mapping_epoch":347,"log_start":"324'1972","ondisk_log_start":"324'1972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394788864,"num_objects":6024,"num_object_clones":0,"num_object_copies":18072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8550,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":235,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.458292+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:36:16.458292+0000","last_peered":"2024-08-17T17:36:16.458292+0000","last_clean":"2024-08-17T17:36:16.458292+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:36:16.458292+0000","last_undegraded":"2024-08-17T17:36:16.458292+0000","last_fullsized":"2024-08-17T17:36:16.458292+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":361,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511967+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:36:16.511967+0000","last_peered":"2024-08-17T17:36:16.511967+0000","last_clean":"2024-08-17T17:36:16.511967+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:36:16.511967+0000","last_undegraded":"2024-08-17T17:36:16.511967+0000","last_fullsized":"2024-08-17T17:36:16.511967+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510784+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:36:16.510784+0000","last_peered":"2024-08-17T17:36:16.510784+0000","last_clean":"2024-08-17T17:36:16.510784+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:36:16.510784+0000","last_undegraded":"2024-08-17T17:36:16.510784+0000","last_fullsized":"2024-08-17T17:36:16.510784+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511410+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:36:16.511410+0000","last_peered":"2024-08-17T17:36:16.511410+0000","last_clean":"2024-08-17T17:36:16.511410+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:36:16.511410+0000","last_undegraded":"2024-08-17T17:36:16.511410+0000","last_fullsized":"2024-08-17T17:36:16.511410+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":354,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511026+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:36:16.511026+0000","last_peered":"2024-08-17T17:36:16.511026+0000","last_clean":"2024-08-17T17:36:16.511026+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:36:16.511026+0000","last_undegraded":"2024-08-17T17:36:16.511026+0000","last_fullsized":"2024-08-17T17:36:16.511026+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":554,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335020+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:36:30.335020+0000","last_peered":"2024-08-17T17:36:30.335020+0000","last_clean":"2024-08-17T17:36:30.335020+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:36:30.335020+0000","last_undegraded":"2024-08-17T17:36:30.335020+0000","last_fullsized":"2024-08-17T17:36:30.335020+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":94,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334847+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:36:30.334847+0000","last_peered":"2024-08-17T17:36:30.334847+0000","last_clean":"2024-08-17T17:36:30.334847+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:36:30.334847+0000","last_undegraded":"2024-08-17T17:36:30.334847+0000","last_fullsized":"2024-08-17T17:36:30.334847+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":260,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.609621+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:36:15.609621+0000","last_peered":"2024-08-17T17:36:15.609621+0000","last_clean":"2024-08-17T17:36:15.609621+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:36:15.609621+0000","last_undegraded":"2024-08-17T17:36:15.609621+0000","last_fullsized":"2024-08-17T17:36:15.609621+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":512,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333787+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:36:30.333787+0000","last_peered":"2024-08-17T17:36:30.333787+0000","last_clean":"2024-08-17T17:36:30.333787+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:36:30.333787+0000","last_undegraded":"2024-08-17T17:36:30.333787+0000","last_fullsized":"2024-08-17T17:36:30.333787+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":404,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-17T17:36:15.610256+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:36:15.610256+0000","last_peered":"2024-08-17T17:36:15.610256+0000","last_clean":"2024-08-17T17:36:15.610256+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:36:15.610256+0000","last_undegraded":"2024-08-17T17:36:15.610256+0000","last_fullsized":"2024-08-17T17:36:15.610256+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":506,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333733+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:36:30.333733+0000","last_peered":"2024-08-17T17:36:30.333733+0000","last_clean":"2024-08-17T17:36:30.333733+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:36:30.333733+0000","last_undegraded":"2024-08-17T17:36:30.333733+0000","last_fullsized":"2024-08-17T17:36:30.333733+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":328,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511838+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:36:16.511838+0000","last_peered":"2024-08-17T17:36:16.511838+0000","last_clean":"2024-08-17T17:36:16.511838+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:36:16.511838+0000","last_undegraded":"2024-08-17T17:36:16.511838+0000","last_fullsized":"2024-08-17T17:36:16.511838+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":537,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333990+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:36:30.333990+0000","last_peered":"2024-08-17T17:36:30.333990+0000","last_clean":"2024-08-17T17:36:30.333990+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:36:30.333990+0000","last_undegraded":"2024-08-17T17:36:30.333990+0000","last_fullsized":"2024-08-17T17:36:30.333990+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335022+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:36:30.335022+0000","last_peered":"2024-08-17T17:36:30.335022+0000","last_clean":"2024-08-17T17:36:30.335022+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:36:30.335022+0000","last_undegraded":"2024-08-17T17:36:30.335022+0000","last_fullsized":"2024-08-17T17:36:30.335022+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":545,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334695+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:36:30.334695+0000","last_peered":"2024-08-17T17:36:30.334695+0000","last_clean":"2024-08-17T17:36:30.334695+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:36:30.334695+0000","last_undegraded":"2024-08-17T17:36:30.334695+0000","last_fullsized":"2024-08-17T17:36:30.334695+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":505,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334656+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:36:30.334656+0000","last_peered":"2024-08-17T17:36:30.334656+0000","last_clean":"2024-08-17T17:36:30.334656+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:36:30.334656+0000","last_undegraded":"2024-08-17T17:36:30.334656+0000","last_fullsized":"2024-08-17T17:36:30.334656+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":340,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510993+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:36:16.510993+0000","last_peered":"2024-08-17T17:36:16.510993+0000","last_clean":"2024-08-17T17:36:16.510993+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:36:16.510993+0000","last_undegraded":"2024-08-17T17:36:16.510993+0000","last_fullsized":"2024-08-17T17:36:16.510993+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":194,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334274+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:36:30.334274+0000","last_peered":"2024-08-17T17:36:30.334274+0000","last_clean":"2024-08-17T17:36:30.334274+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:36:30.334274+0000","last_undegraded":"2024-08-17T17:36:30.334274+0000","last_fullsized":"2024-08-17T17:36:30.334274+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":248,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.458269+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:36:16.458269+0000","last_peered":"2024-08-17T17:36:16.458269+0000","last_clean":"2024-08-17T17:36:16.458269+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:36:16.458269+0000","last_undegraded":"2024-08-17T17:36:16.458269+0000","last_fullsized":"2024-08-17T17:36:16.458269+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":348,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511343+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:36:16.511343+0000","last_peered":"2024-08-17T17:36:16.511343+0000","last_clean":"2024-08-17T17:36:16.511343+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:36:16.511343+0000","last_undegraded":"2024-08-17T17:36:16.511343+0000","last_fullsized":"2024-08-17T17:36:16.511343+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":211,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510718+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:36:16.510718+0000","last_peered":"2024-08-17T17:36:16.510718+0000","last_clean":"2024-08-17T17:36:16.510718+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:36:16.510718+0000","last_undegraded":"2024-08-17T17:36:16.510718+0000","last_fullsized":"2024-08-17T17:36:16.510718+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":348,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510846+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:36:16.510846+0000","last_peered":"2024-08-17T17:36:16.510846+0000","last_clean":"2024-08-17T17:36:16.510846+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:36:16.510846+0000","last_undegraded":"2024-08-17T17:36:16.510846+0000","last_fullsized":"2024-08-17T17:36:16.510846+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":211,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510167+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:36:16.510167+0000","last_peered":"2024-08-17T17:36:16.510167+0000","last_clean":"2024-08-17T17:36:16.510167+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:36:16.510167+0000","last_undegraded":"2024-08-17T17:36:16.510167+0000","last_fullsized":"2024-08-17T17:36:16.510167+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":557,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334221+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:36:30.334221+0000","last_peered":"2024-08-17T17:36:30.334221+0000","last_clean":"2024-08-17T17:36:30.334221+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:36:30.334221+0000","last_undegraded":"2024-08-17T17:36:30.334221+0000","last_fullsized":"2024-08-17T17:36:30.334221+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":530,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335103+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:36:30.335103+0000","last_peered":"2024-08-17T17:36:30.335103+0000","last_clean":"2024-08-17T17:36:30.335103+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:36:30.335103+0000","last_undegraded":"2024-08-17T17:36:30.335103+0000","last_fullsized":"2024-08-17T17:36:30.335103+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509482+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:36:16.509482+0000","last_peered":"2024-08-17T17:36:16.509482+0000","last_clean":"2024-08-17T17:36:16.509482+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:36:16.509482+0000","last_undegraded":"2024-08-17T17:36:16.509482+0000","last_fullsized":"2024-08-17T17:36:16.509482+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":250,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.458285+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:36:16.458285+0000","last_peered":"2024-08-17T17:36:16.458285+0000","last_clean":"2024-08-17T17:36:16.458285+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:36:16.458285+0000","last_undegraded":"2024-08-17T17:36:16.458285+0000","last_fullsized":"2024-08-17T17:36:16.458285+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6426918936,"num_objects":98068,"num_object_clones":0,"num_object_copies":294204,"num_objects_missing_on_primary":6743,"num_objects_missing":6743,"num_objects_degraded":56638,"num_objects_misplaced":38390,"num_objects_unfound":0,"num_objects_dirty":98068,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138322,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23921,"num_bytes_recovered":1445134336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20936208384,"data_stored":20936196168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104541,"ondisk_log_size":104541,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170041,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463411,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3646760,"kb_used_data":2744736,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":90098392,"statfs":{"total":95995035648,"available":92260753408,"internally_reserved":0,"allocated":2810609664,"data_stored":2808727324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492085,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179836,"kb_used_data":788660,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92565316,"statfs":{"total":95995035648,"available":94786883584,"internally_reserved":0,"allocated":807587840,"data_stored":805735593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986677,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2463904,"kb_used_data":1562964,"kb_used_omap":1,"kb_used_meta":900926,"kb_avail":91281248,"statfs":{"total":95995035648,"available":93471997952,"internally_reserved":0,"allocated":1600475136,"data_stored":1598603736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":922548568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,1,1,1,3,5,4],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225504,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4030780,"kb_used_data":3137268,"kb_used_omap":4,"kb_used_meta":893499,"kb_avail":89714372,"statfs":{"total":95995035648,"available":91867516928,"internally_reserved":0,"allocated":3212562432,"data_stored":3210710483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":914943424},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075829,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533722,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3773648,"kb_used_data":2752456,"kb_used_omap":5,"kb_used_meta":1021178,"kb_avail":89971504,"statfs":{"total":95995035648,"available":92130820096,"internally_reserved":0,"allocated":2818514944,"data_stored":2816649880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1045686512},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089698,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3362584,"kb_used_data":2371408,"kb_used_omap":5,"kb_used_meta":991162,"kb_avail":90382568,"statfs":{"total":95995035648,"available":92551749632,"internally_reserved":0,"allocated":2428321792,"data_stored":2426447304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1014949947},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2422210560,"data_stored":2422210560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2813726720,"data_stored":2813722648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4389867520,"data_stored":4389863448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3206348800,"data_stored":3206348800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1595539456,"data_stored":1595539456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":803012608,"data_stored":803012608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2802581504,"data_stored":2802581504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2902921216,"data_stored":2902917144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:36:34.136 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:34.137 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:34.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:34.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:34.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:34.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:34.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:34 smithi142 ceph-mon[103614]: from='client.37122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:34.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:34 smithi142 ceph-mon[106729]: from='client.37122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:34.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:34.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:34.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:34 smithi152 ceph-mon[93456]: from='client.37122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:34.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:34.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:34.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:34.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:34.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:34.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:34.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:36:34.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:34.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:34.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:35.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:35.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:35.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:35.103 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:35.109 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:35.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:35.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:35.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:35.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:35.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:35.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:35.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:35.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:35 smithi152 ceph-mon[93456]: pgmap v1277: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 78 op/s; 56641/294144 objects degraded (19.256%); 38390/294144 objects misplaced (13.051%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:35.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:35.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:35.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:35 smithi142 ceph-mon[106729]: pgmap v1277: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 78 op/s; 56641/294144 objects degraded (19.256%); 38390/294144 objects misplaced (13.051%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:35 smithi142 ceph-mon[103614]: pgmap v1277: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 78 op/s; 56641/294144 objects degraded (19.256%); 38390/294144 objects misplaced (13.051%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:35.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:35.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:35.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:35.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:36.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:36.034 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:36.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:36.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:36.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:36.282 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:36.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:36.405 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:36.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:36.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:36.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56641/294144 objects degraded (19.256%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:36.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56641/294144 objects degraded (19.256%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:36.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:36.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:36.680 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56641/294144 objects degraded (19.256%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:36.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:36.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:36.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:36.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:36.903 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:36.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:36.933 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:36.933 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:37.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:37.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:37.069 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:36:37.069 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:37.071 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:36:37.072 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:37.087 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:36:37.088 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:37.104 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:36:37.105 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:37.120 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:36:37.120 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:37.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:37.135 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:36:37.135 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:37.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:36:37.151 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:36:37.151 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:37.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:37.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:37.169 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:36:37.169 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:37.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:37.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:37 smithi142 ceph-mon[106729]: pgmap v1278: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 56615/293775 objects degraded (19.272%); 38321/293775 objects misplaced (13.044%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:37.358 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1848178315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:37.358 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:37 smithi142 ceph-mon[103614]: pgmap v1278: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 56615/293775 objects degraded (19.272%); 38321/293775 objects misplaced (13.044%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:37.359 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1848178315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:37.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:37.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:37.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:37.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:37 smithi152 ceph-mon[93456]: pgmap v1278: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 72 op/s; 56615/293775 objects degraded (19.272%); 38321/293775 objects misplaced (13.044%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:36:37.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1848178315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:37.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:37.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:36:37.649 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:37.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:37.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:37.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:37.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:37.943 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:38.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:38.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:38.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:38.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:38.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:38.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:38.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:38.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:38.388 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:36:38.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:38.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:38.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:38.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:38.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/998469087' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:38.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:38 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/998469087' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:38.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:38 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:38.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:38.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:38.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/998469087' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:38.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:38 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:38.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:38.771 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:38.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:38.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:38.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:38.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:39.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:39.099 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:39.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:39.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:39.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:39.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:39.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:39.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:39.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:39.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:39.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:39.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:39.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[106729]: pgmap v1279: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 63 op/s; 56594/293154 objects degraded (19.305%); 38254/293154 objects misplaced (13.049%); 877 KiB/s, 13 objects/s recovering 2024-08-17T17:36:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:39.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[106729]: osdmap e358: 8 total, 8 up, 6 in 2024-08-17T17:36:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[103614]: pgmap v1279: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 63 op/s; 56594/293154 objects degraded (19.305%); 38254/293154 objects misplaced (13.049%); 877 KiB/s, 13 objects/s recovering 2024-08-17T17:36:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:39 smithi142 ceph-mon[103614]: osdmap e358: 8 total, 8 up, 6 in 2024-08-17T17:36:39.644 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:39.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:39 smithi152 ceph-mon[93456]: pgmap v1279: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 63 op/s; 56594/293154 objects degraded (19.305%); 38254/293154 objects misplaced (13.049%); 877 KiB/s, 13 objects/s recovering 2024-08-17T17:36:39.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:39 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:39.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:39 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:39.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:39 smithi152 ceph-mon[93456]: osdmap e358: 8 total, 8 up, 6 in 2024-08-17T17:36:39.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:39.768 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:39.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:39.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:39.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:39.961 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:36:39.961 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:36:39.961 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:36:39.961 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:36:39.961 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:36:39.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:40.018 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:40.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:40.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:40.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:40.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:40.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:40.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:40.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:40.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:40.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:40.497 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:40.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:40.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:36:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:36:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:40.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:40.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:36:40.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:40.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:40.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:40.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:40.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:40.939 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:40.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:40.983 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:41.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:41.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:41.079 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:36:41.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:41.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:41.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:41.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:41.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:41.357 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:41 smithi152 ceph-mon[93456]: pgmap v1281: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 56594/293154 objects degraded (19.305%); 38254/293154 objects misplaced (13.049%); 883 KiB/s, 13 objects/s recovering 2024-08-17T17:36:41.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:41.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:41.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:41.461 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:41 smithi142 ceph-mon[106729]: pgmap v1281: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 56594/293154 objects degraded (19.305%); 38254/293154 objects misplaced (13.049%); 883 KiB/s, 13 objects/s recovering 2024-08-17T17:36:41.461 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56594/293154 objects degraded (19.305%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:41.462 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:41 smithi142 ceph-mon[103614]: pgmap v1281: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 56594/293154 objects degraded (19.305%); 38254/293154 objects misplaced (13.049%); 883 KiB/s, 13 objects/s recovering 2024-08-17T17:36:41.462 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56594/293154 objects degraded (19.305%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:41.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.566 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:36:41.567 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:36:41.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:41.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:41.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:41.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:41.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:41.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56594/293154 objects degraded (19.305%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:41.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:41.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:42.028 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:42.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.068 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:42.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:36:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:36:41] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:36:42.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:42.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:42.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:42.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:42.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:42.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:42.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:42.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:42.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:42.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:42.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:42.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:42.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:42.717 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:42.717 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:42.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:42.812 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:42.812 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:42.812 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:36:42.813 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:42.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36603.0:162106 11.3 11:cd377b44:::benchmark_data_smithi142_227468_object38687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:36:42.835 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 433.32458960899999, 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32571883000000001, 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36603", 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/3607185491", 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 162106 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:36:42.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493699+0000", 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493700+0000", 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7239999999999999e-06 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493705+0000", 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2899999999999996e-06 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.493707+0000", 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1909999999999999e-06 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.498299+0000", 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045918419999999996 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.498402+0000", 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010297399999999999 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.498504+0000", 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102322 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.502521+0000", 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040166159999999998 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.502874+0000", 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000353467 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.819325+0000", 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31645051899999999 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.819379+0000", 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4036999999999998e-05 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:29:29.819418+0000", 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9140000000000001e-05 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:56985 12.4 12:257f5782:::benchmark_data_smithi142_261410_object56984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.008917+0000", 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 266.809371236, 2024-08-17T17:36:42.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33832952500000002, 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 56985 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.008917+0000", 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.008917+0000", 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.008916+0000", 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.008975+0000", 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9092999999999998e-05 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.008986+0000", 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0471e-05 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.008990+0000", 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9740000000000004e-06 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.009033+0000", 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3525000000000001e-05 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.009174+0000", 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014048199999999999 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.009314+0000", 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013989699999999999 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.011878+0000", 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025644090000000001 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.014816+0000", 2024-08-17T17:36:42.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029378490000000002 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347141+0000", 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.332324758 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347201+0000", 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0535999999999998e-05 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347247+0000", 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5547000000000003e-05 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:57002 12.5 12:a933516b:::benchmark_data_smithi142_261410_object57001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.022179+0000", 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 266.796109849, 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32824058, 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.849 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 57002 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022179+0000", 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022179+0000", 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022177+0000", 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022232+0000", 2024-08-17T17:36:42.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4956e-05 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022246+0000", 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4003e-05 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022252+0000", 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2709999999999998e-06 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022289+0000", 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6823000000000003e-05 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022370+0000", 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0533999999999998e-05 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.022476+0000", 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010618100000000001 2024-08-17T17:36:42.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.024646+0000", 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021702610000000002 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.026089+0000", 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001443223 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.350368+0000", 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32427863499999998 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.350398+0000", 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0275000000000001e-05 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.350419+0000", 2024-08-17T17:36:42.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1022e-05 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112415 12.4 12:226a051b:::benchmark_data_smithi142_261410_object112414:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.971157+0000", 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 212.84713173, 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344866377, 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.856 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112415 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971157+0000", 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971157+0000", 2024-08-17T17:36:42.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971156+0000", 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971225+0000", 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9197999999999997e-05 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971238+0000", 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2839e-05 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971255+0000", 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7621e-05 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971289+0000", 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3884999999999998e-05 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971422+0000", 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013329300000000001 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.971557+0000", 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013436700000000001 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.979456+0000", 2024-08-17T17:36:42.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078994789999999992 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.979496+0000", 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0170000000000003e-05 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.315962+0000", 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33646543800000001 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.315999+0000", 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7512000000000003e-05 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.316023+0000", 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3842999999999999e-05 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.862 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112419 12.4 12:213f767a:::benchmark_data_smithi142_261410_object112418:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.974639+0000", 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 212.84364942400001, 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34580895299999997, 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112419 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974639+0000", 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974639+0000", 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974638+0000", 2024-08-17T17:36:42.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974703+0000", 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5258000000000004e-05 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974718+0000", 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5023e-05 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974849+0000", 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013064700000000001 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974895+0000", 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5748000000000003e-05 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.975003+0000", 2024-08-17T17:36:42.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108089 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.975079+0000", 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6202e-05 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.979483+0000", 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044043870000000001 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.979502+0000", 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8598999999999999e-05 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320366+0000", 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34086429600000001 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320414+0000", 2024-08-17T17:36:42.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7788999999999999e-05 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320448+0000", 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4255000000000003e-05 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112434 12.4 12:23cb9568:::benchmark_data_smithi142_261410_object112433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.989074+0000", 2024-08-17T17:36:42.869 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 212.829214342, 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33403828099999999, 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112434 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989074+0000", 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989074+0000", 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989074+0000", 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989129+0000", 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5467999999999998e-05 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989136+0000", 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989140+0000", 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0740000000000003e-06 2024-08-17T17:36:42.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989180+0000", 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0441000000000001e-05 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989297+0000", 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116543 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.989452+0000", 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015485799999999999 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993423+0000", 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039712699999999998 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993518+0000", 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5169999999999999e-05 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.323046+0000", 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32952796699999998 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.323080+0000", 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4311999999999999e-05 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.323113+0000", 2024-08-17T17:36:42.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2255000000000002e-05 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124377 12.5 12:a163769e:::benchmark_data_smithi142_261410_object6180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.868147+0000", 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 192.95014195100001, 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.338594018, 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.876 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124377 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868147+0000", 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868147+0000", 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868145+0000", 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868152+0000", 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5429999999999999e-06 2024-08-17T17:36:42.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868165+0000", 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2697999999999999e-05 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868170+0000", 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2270000000000001e-06 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868210+0000", 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9857999999999997e-05 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868324+0000", 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011449499999999999 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.868443+0000", 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119073 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.871561+0000", 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031179409999999999 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.875490+0000", 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039291430000000004 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.199632+0000", 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32414186099999998 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.199702+0000", 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9788000000000003e-05 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.206741+0000", 2024-08-17T17:36:42.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070387310000000003 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124378 12.9 12:91fb9325:::benchmark_data_smithi142_261410_object6181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.869866+0000", 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 192.94842223699999, 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33737797200000003, 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.883 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124378 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869866+0000", 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869866+0000", 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869866+0000", 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869870+0000", 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8080000000000004e-06 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869882+0000", 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1090999999999999e-05 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869886+0000", 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1969999999999998e-06 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.869936+0000", 2024-08-17T17:36:42.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0441e-05 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.870094+0000", 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000157906 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.870258+0000", 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016345199999999999 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.873458+0000", 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032002189999999998 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.874567+0000", 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011090099999999999 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.202894+0000", 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32832749900000002 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.202951+0000", 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6397000000000002e-05 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.207244+0000", 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042936809999999997 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.889 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124384 12.9 12:98f3a7b5:::benchmark_data_smithi142_261410_object6187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.893663+0000", 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 192.92462593100001, 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.346019729, 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.890 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124384 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893663+0000", 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893663+0000", 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893661+0000", 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893672+0000", 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0596e-05 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893687+0000", 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5231999999999999e-05 2024-08-17T17:36:42.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893695+0000", 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7570000000000004e-06 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893747+0000", 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2324000000000003e-05 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.893942+0000", 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019500799999999999 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.894129+0000", 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000187214 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.900130+0000", 2024-08-17T17:36:42.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060004940000000003 2024-08-17T17:36:42.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.901144+0000", 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010142689999999999 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.234379+0000", 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33323500499999997 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.234440+0000", 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1121e-05 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.239682+0000", 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052423180000000002 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.897 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124385 12.9 12:9a4c7f9d:::benchmark_data_smithi142_261410_object6188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.895222+0000", 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 192.92306643399999, 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34914930900000002, 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.898 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124385 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895222+0000", 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895222+0000", 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895221+0000", 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895226+0000", 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2820000000000004e-06 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895237+0000", 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0229e-05 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895242+0000", 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4219999999999999e-06 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.897492+0000", 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022502989999999999 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.897643+0000", 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015033399999999999 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.897739+0000", 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6673000000000001e-05 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.900260+0000", 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025210559999999998 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.901269+0000", 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001008985 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.239730+0000", 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33846039500000002 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.239766+0000", 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6640000000000002e-05 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.244372+0000", 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046050800000000001 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.904 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129851 12.f 12:fa60de86:::benchmark_data_smithi142_261410_object11654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 138.17089575200001, 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34876531300000002, 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129851 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647391+0000", 2024-08-17T17:36:42.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647400+0000", 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1290000000000002e-06 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647417+0000", 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7139e-05 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647425+0000", 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3310000000000004e-06 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647482+0000", 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6628000000000001e-05 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647642+0000", 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000160402 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647812+0000", 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016993499999999999 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.650735+0000", 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002922371 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.651441+0000", 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070664600000000001 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.992978+0000", 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34153630699999998 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993043+0000", 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4863999999999994e-05 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.996158+0000", 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031156859999999999 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129928 12.f 12:fb7bc46a:::benchmark_data_smithi142_261410_object11731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:26.987883+0000", 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 135.830405824, 2024-08-17T17:36:42.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34221478399999999, 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129928 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987883+0000", 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987883+0000", 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987882+0000", 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987887+0000", 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1830000000000002e-06 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987896+0000", 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7610000000000002e-06 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987900+0000", 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6519999999999999e-06 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.987942+0000", 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1511999999999998e-05 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.988099+0000", 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015699900000000001 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.988255+0000", 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015687899999999999 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.991284+0000", 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003028443 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.991947+0000", 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066338599999999997 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.327276+0000", 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33532906899999998 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.327314+0000", 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7143e-05 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.330098+0000", 2024-08-17T17:36:42.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002784066 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131200 12.f 12:fe694d48:::benchmark_data_smithi142_261410_object13003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:54.647650+0000", 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 108.170638711, 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34767719600000002, 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.918 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131200 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647650+0000", 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647650+0000", 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647649+0000", 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647654+0000", 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8719999999999999e-06 2024-08-17T17:36:42.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647665+0000", 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0196e-05 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647670+0000", 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.772e-06 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647707+0000", 2024-08-17T17:36:42.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6968999999999998e-05 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647816+0000", 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010883900000000001 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647924+0000", 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107377 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.651120+0000", 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031966210000000002 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.651133+0000", 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2846e-05 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.991970+0000", 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34083642600000003 2024-08-17T17:36:42.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.992029+0000", 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9824000000000002e-05 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.995327+0000", 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032977639999999999 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131638 12.f 12:fb3cd103:::benchmark_data_smithi142_261410_object13441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 99.556147577000004, 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35214242000000001, 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131638 2024-08-17T17:36:42.925 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262140+0000", 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262145+0000", 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8250000000000004e-06 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262153+0000", 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2870000000000004e-06 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262157+0000", 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6629999999999999e-06 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262194+0000", 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7626000000000002e-05 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262367+0000", 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000172366 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262545+0000", 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017861400000000001 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.265416+0000", 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028709809999999999 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.266086+0000", 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066993100000000002 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.611065+0000", 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34497888500000001 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.611104+0000", 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8411999999999997e-05 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.614283+0000", 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003179856 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.931 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131763 12.f 12:f549739a:::benchmark_data_smithi142_261410_object13566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 98.266013088999998, 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37794038699999999, 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.932 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131763 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552275+0000", 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552279+0000", 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3080000000000001e-06 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552290+0000", 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0900000000000001e-05 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552295+0000", 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8879999999999996e-06 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552328+0000", 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3683999999999998e-05 2024-08-17T17:36:42.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552433+0000", 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104164 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552533+0000", 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100392 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.555327+0000", 2024-08-17T17:36:42.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002794019 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.555992+0000", 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066522 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.927119+0000", 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.371126601 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.927153+0000", 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3800000000000002e-05 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.930216+0000", 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030633309999999999 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.938 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131820 12.9 12:9a926c4c:::benchmark_data_smithi142_261410_object13623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 97.167076206000004, 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81736743099999998, 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131820 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651211+0000", 2024-08-17T17:36:42.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651218+0000", 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6810000000000001e-06 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651229+0000", 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1258e-05 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651234+0000", 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5379999999999996e-06 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651287+0000", 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3192000000000002e-05 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651298+0000", 2024-08-17T17:36:42.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.15e-05 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461578+0000", 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81027978499999997 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461702+0000", 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012391299999999999 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461829+0000", 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012717300000000001 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.465382+0000", 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035532430000000002 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.466095+0000", 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00071217299999999997 2024-08-17T17:36:42.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468482+0000", 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023874 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468535+0000", 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3319000000000002e-05 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468580+0000", 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4569e-05 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:134422 12.f 12:fa29169b:::benchmark_data_smithi142_261410_object16225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 62.368026020000002, 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.45255063099999998, 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.946 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 134422 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450261+0000", 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450270+0000", 2024-08-17T17:36:42.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5539999999999998e-06 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450286+0000", 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6568000000000001e-05 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450292+0000", 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1550000000000004e-06 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450333+0000", 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0472000000000001e-05 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450457+0000", 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012431599999999999 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450565+0000", 2024-08-17T17:36:42.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108013 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.453491+0000", 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029262239999999998 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.453504+0000", 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2989999999999999e-05 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.898771+0000", 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.44526617200000002 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.898830+0000", 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9506999999999998e-05 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.902813+0000", 2024-08-17T17:36:42.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039831399999999996 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:134490 12.f 12:f6a35853:::benchmark_data_smithi142_261410_object16293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 61.474363633000003, 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36548067000000001, 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.953 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 134490 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:36:42.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343924+0000", 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343929+0000", 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3769999999999998e-06 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343937+0000", 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1969999999999996e-06 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343940+0000", 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.641e-06 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343972+0000", 2024-08-17T17:36:42.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1508999999999999e-05 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.344083+0000", 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011066200000000001 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.344183+0000", 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9959999999999998e-05 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.347096+0000", 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029132110000000002 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.347261+0000", 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016498399999999999 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.706519+0000", 2024-08-17T17:36:42.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35925865099999998 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.706556+0000", 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6850999999999997e-05 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.709406+0000", 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028494760000000001 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.959 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:135448 12.f 12:fe02232f:::benchmark_data_smithi142_261410_object17251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 48.384939598999999, 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.46790546399999999, 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 135448 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433347+0000", 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433357+0000", 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0322999999999999e-05 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433376+0000", 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8244999999999999e-05 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433382+0000", 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433433+0000", 2024-08-17T17:36:42.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0903999999999999e-05 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433554+0000", 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012107099999999999 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433661+0000", 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010699 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.436511+0000", 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028494309999999999 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.436665+0000", 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015454999999999999 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.897905+0000", 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.46124029100000002 2024-08-17T17:36:42.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.897963+0000", 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7336000000000001e-05 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.901255+0000", 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00329168 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.966 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:135735 12.f 12:ff30985f:::benchmark_data_smithi142_261410_object17538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 44.385525014999999, 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35154203099999998, 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 135735 2024-08-17T17:36:42.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:36:42.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432762+0000", 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432767+0000", 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.262e-06 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432776+0000", 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7870000000000007e-06 2024-08-17T17:36:42.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432779+0000", 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8859999999999997e-06 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436659+0000", 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003879263 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:36:42.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436779+0000", 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119862 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436904+0000", 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012594099999999999 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.439927+0000", 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030223350000000001 2024-08-17T17:36:42.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.440109+0000", 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018190799999999999 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.781461+0000", 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34135219100000003 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:36:42.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.781503+0000", 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1789000000000003e-05 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.784306+0000", 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028029550000000002 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:36:42.973 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:36:42.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:43.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:43.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:43.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:43.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:43.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:43.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:43.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:43.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:43 smithi152 ceph-mon[93456]: from='client.37140 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:43 smithi152 ceph-mon[93456]: pgmap v1282: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 56536/291948 objects degraded (19.365%); 38076/291948 objects misplaced (13.042%); 1004 KiB/s, 15 objects/s recovering 2024-08-17T17:36:43.577 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:36:43.577 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:36:43.577 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:36:43.577 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:36:43.577 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:36:43.578 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:43.578 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:43.578 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:43.578 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:43.578 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:43.578 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:36:43.579 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:43.594 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:36:43.594 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:43.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:43.609 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:36:43.609 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:43.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:43.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:43.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:43.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:36:43.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:43.871 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:43 smithi142 ceph-mon[106729]: from='client.37140 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:43 smithi142 ceph-mon[106729]: pgmap v1282: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 56536/291948 objects degraded (19.365%); 38076/291948 objects misplaced (13.042%); 1004 KiB/s, 15 objects/s recovering 2024-08-17T17:36:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:43 smithi142 ceph-mon[103614]: from='client.37140 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:43 smithi142 ceph-mon[103614]: pgmap v1282: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 56536/291948 objects degraded (19.365%); 38076/291948 objects misplaced (13.042%); 1004 KiB/s, 15 objects/s recovering 2024-08-17T17:36:43.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:43.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:44.006 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1281,"stamp":"2024-08-17T17:36:40.262566+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6402873317,"num_objects":97718,"num_object_clones":0,"num_object_copies":293154,"num_objects_missing_on_primary":6715,"num_objects_missing":6715,"num_objects_degraded":56594,"num_objects_misplaced":38254,"num_objects_unfound":0,"num_objects_dirty":97718,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":138976,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24036,"num_bytes_recovered":1452352142,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104924,"ondisk_log_size":104924,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18402408,"kb_used_data":13302388,"kb_used_omap":27,"kb_used_meta":5099940,"kb_avail":544068504,"statfs":{"total":575970213888,"available":557126148096,"internally_reserved":0,"allocated":13621645312,"data_stored":13610416652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5222339304},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-41877504,"num_objects":-639,"num_object_clones":0,"num_object_copies":-1917,"num_objects_missing_on_primary":-53,"num_objects_missing":-53,"num_objects_degraded":-98,"num_objects_misplaced":-266,"num_objects_unfound":0,"num_objects_dirty":-639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":639,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.929964"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":555,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334189+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:36:30.334189+0000","last_peered":"2024-08-17T17:36:30.334189+0000","last_clean":"2024-08-17T17:36:30.334189+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:36:30.334189+0000","last_undegraded":"2024-08-17T17:36:30.334189+0000","last_fullsized":"2024-08-17T17:36:30.334189+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":66,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511190+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:36:16.511190+0000","last_peered":"2024-08-17T17:36:16.511190+0000","last_clean":"2024-08-17T17:36:16.511190+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:36:16.511190+0000","last_undegraded":"2024-08-17T17:36:16.511190+0000","last_fullsized":"2024-08-17T17:36:16.511190+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":261,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416923+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:36:30.416923+0000","last_peered":"2024-08-17T17:36:30.416923+0000","last_clean":"2024-08-17T17:36:30.416923+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:36:30.416923+0000","last_undegraded":"2024-08-17T17:36:30.416923+0000","last_fullsized":"2024-08-17T17:36:30.416923+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334405+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:36:30.334405+0000","last_peered":"2024-08-17T17:36:30.334405+0000","last_clean":"2024-08-17T17:36:30.334405+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:36:30.334405+0000","last_undegraded":"2024-08-17T17:36:30.334405+0000","last_fullsized":"2024-08-17T17:36:30.334405+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":430,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335044+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:36:30.335044+0000","last_peered":"2024-08-17T17:36:30.335044+0000","last_clean":"2024-08-17T17:36:30.335044+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:36:30.335044+0000","last_undegraded":"2024-08-17T17:36:30.335044+0000","last_fullsized":"2024-08-17T17:36:30.335044+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510722+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:36:16.510722+0000","last_peered":"2024-08-17T17:36:16.510722+0000","last_clean":"2024-08-17T17:36:16.510722+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:36:16.510722+0000","last_undegraded":"2024-08-17T17:36:16.510722+0000","last_fullsized":"2024-08-17T17:36:16.510722+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":523,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334572+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:36:30.334572+0000","last_peered":"2024-08-17T17:36:30.334572+0000","last_clean":"2024-08-17T17:36:30.334572+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:36:30.334572+0000","last_undegraded":"2024-08-17T17:36:30.334572+0000","last_fullsized":"2024-08-17T17:36:30.334572+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":377,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.512165+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:36:16.512165+0000","last_peered":"2024-08-17T17:36:16.512165+0000","last_clean":"2024-08-17T17:36:16.512165+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:36:16.512165+0000","last_undegraded":"2024-08-17T17:36:16.512165+0000","last_fullsized":"2024-08-17T17:36:16.512165+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"357'8624","reported_seq":9779,"reported_epoch":357,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:36:35.720491+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:36:35.720491+0000","last_peered":"2024-08-17T17:36:35.720491+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:36:35.720491+0000","last_undegraded":"2024-08-17T17:36:35.720491+0000","last_fullsized":"2024-08-17T17:36:35.720491+0000","mapping_epoch":337,"log_start":"324'1763","ondisk_log_start":"324'1763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6861,"log_dups_size":0,"ondisk_log_size":6861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398458880,"num_objects":6080,"num_object_clones":0,"num_object_copies":18240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6080,"num_objects_unfound":0,"num_objects_dirty":6080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8624,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35586048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":6080}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":62,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.589611+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:36:16.589611+0000","last_peered":"2024-08-17T17:36:16.589611+0000","last_clean":"2024-08-17T17:36:16.589611+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:36:16.589611+0000","last_undegraded":"2024-08-17T17:36:16.589611+0000","last_fullsized":"2024-08-17T17:36:16.589611+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":244,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416328+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:36:30.416328+0000","last_peered":"2024-08-17T17:36:30.416328+0000","last_clean":"2024-08-17T17:36:30.416328+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:36:30.416328+0000","last_undegraded":"2024-08-17T17:36:30.416328+0000","last_fullsized":"2024-08-17T17:36:30.416328+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"357'8879","reported_seq":8934,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:36.840066+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:36:36.840066+0000","last_peered":"2024-08-17T17:36:36.840066+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:36:36.840066+0000","last_undegraded":"2024-08-17T17:36:36.840066+0000","last_fullsized":"2024-08-17T17:36:36.840066+0000","mapping_epoch":337,"log_start":"324'2282","ondisk_log_start":"324'2282","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405995520,"num_objects":6195,"num_object_clones":0,"num_object_copies":18585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6195,"num_objects_unfound":0,"num_objects_dirty":6195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8879,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":6195}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":64,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511126+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:36:16.511126+0000","last_peered":"2024-08-17T17:36:16.511126+0000","last_clean":"2024-08-17T17:36:16.511126+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:36:16.511126+0000","last_undegraded":"2024-08-17T17:36:16.511126+0000","last_fullsized":"2024-08-17T17:36:16.511126+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":328,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510645+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:36:16.510645+0000","last_peered":"2024-08-17T17:36:16.510645+0000","last_clean":"2024-08-17T17:36:16.510645+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:36:16.510645+0000","last_undegraded":"2024-08-17T17:36:16.510645+0000","last_fullsized":"2024-08-17T17:36:16.510645+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"357'8602","reported_seq":8660,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:35.669367+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:36:35.669367+0000","last_peered":"2024-08-17T17:36:35.669367+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:36:35.669367+0000","last_undegraded":"2024-08-17T17:36:35.669367+0000","last_fullsized":"2024-08-17T17:36:35.669367+0000","mapping_epoch":337,"log_start":"324'1753","ondisk_log_start":"324'1753","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6849,"log_dups_size":0,"ondisk_log_size":6849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401866752,"num_objects":6132,"num_object_clones":0,"num_object_copies":18396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6132,"num_objects_unfound":0,"num_objects_dirty":6132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8602,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":6132}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":61,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416875+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:36:30.416875+0000","last_peered":"2024-08-17T17:36:30.416875+0000","last_clean":"2024-08-17T17:36:30.416875+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:36:30.416875+0000","last_undegraded":"2024-08-17T17:36:30.416875+0000","last_fullsized":"2024-08-17T17:36:30.416875+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":338,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509457+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:36:16.509457+0000","last_peered":"2024-08-17T17:36:16.509457+0000","last_clean":"2024-08-17T17:36:16.509457+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:36:16.509457+0000","last_undegraded":"2024-08-17T17:36:16.509457+0000","last_fullsized":"2024-08-17T17:36:16.509457+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"357'8782","reported_seq":8874,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:38.037897+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:36:38.037897+0000","last_peered":"2024-08-17T17:36:38.037897+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:36:38.037897+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'1713","ondisk_log_start":"324'1713","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404750336,"num_objects":6176,"num_object_clones":0,"num_object_copies":18528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8782,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-593},{"shards":"0,3,7","objects":6176}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":467,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334361+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:36:30.334361+0000","last_peered":"2024-08-17T17:36:30.334361+0000","last_clean":"2024-08-17T17:36:30.334361+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:36:30.334361+0000","last_undegraded":"2024-08-17T17:36:30.334361+0000","last_fullsized":"2024-08-17T17:36:30.334361+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":233,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.339373+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:36:30.339373+0000","last_peered":"2024-08-17T17:36:30.339373+0000","last_clean":"2024-08-17T17:36:30.339373+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:36:30.339373+0000","last_undegraded":"2024-08-17T17:36:30.339373+0000","last_fullsized":"2024-08-17T17:36:30.339373+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"357'8547","reported_seq":8593,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:36.502979+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:36:36.502979+0000","last_peered":"2024-08-17T17:36:36.502979+0000","last_clean":"2024-08-17T17:36:36.502979+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:36:36.502979+0000","last_undegraded":"2024-08-17T17:36:36.502979+0000","last_fullsized":"2024-08-17T17:36:36.502979+0000","mapping_epoch":319,"log_start":"324'1974","ondisk_log_start":"324'1974","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396296192,"num_objects":6047,"num_object_clones":0,"num_object_copies":18141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8547,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":556,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334164+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:36:30.334164+0000","last_peered":"2024-08-17T17:36:30.334164+0000","last_clean":"2024-08-17T17:36:30.334164+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:36:30.334164+0000","last_undegraded":"2024-08-17T17:36:30.334164+0000","last_fullsized":"2024-08-17T17:36:30.334164+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510457+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:36:16.510457+0000","last_peered":"2024-08-17T17:36:16.510457+0000","last_clean":"2024-08-17T17:36:16.510457+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:36:16.510457+0000","last_undegraded":"2024-08-17T17:36:16.510457+0000","last_fullsized":"2024-08-17T17:36:16.510457+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"357'8709","reported_seq":8765,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:35.738546+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:36:35.738546+0000","last_peered":"2024-08-17T17:36:35.738546+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:36:35.738546+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'2135","ondisk_log_start":"324'2135","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407830528,"num_objects":6223,"num_object_clones":0,"num_object_copies":18669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8709,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":6223},{"shards":"1,6","objects":-1143}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":312,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509883+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:36:16.509883+0000","last_peered":"2024-08-17T17:36:16.509883+0000","last_clean":"2024-08-17T17:36:16.509883+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:36:16.509883+0000","last_undegraded":"2024-08-17T17:36:16.509883+0000","last_fullsized":"2024-08-17T17:36:16.509883+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":338,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510278+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:36:16.510278+0000","last_peered":"2024-08-17T17:36:16.510278+0000","last_clean":"2024-08-17T17:36:16.510278+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:36:16.510278+0000","last_undegraded":"2024-08-17T17:36:16.510278+0000","last_fullsized":"2024-08-17T17:36:16.510278+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"357'8776","reported_seq":8869,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:36.832040+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:36:36.832040+0000","last_peered":"2024-08-17T17:36:36.832040+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:36:36.832040+0000","last_undegraded":"2024-08-17T17:36:36.832040+0000","last_fullsized":"2024-08-17T17:36:36.832040+0000","mapping_epoch":337,"log_start":"325'3161","ondisk_log_start":"325'3161","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408551424,"num_objects":6234,"num_object_clones":0,"num_object_copies":18702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6234,"num_objects_unfound":0,"num_objects_dirty":6234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8776,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":6234}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":349,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510850+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:36:16.510850+0000","last_peered":"2024-08-17T17:36:16.510850+0000","last_clean":"2024-08-17T17:36:16.510850+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:36:16.510850+0000","last_undegraded":"2024-08-17T17:36:16.510850+0000","last_fullsized":"2024-08-17T17:36:16.510850+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511203+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:36:16.511203+0000","last_peered":"2024-08-17T17:36:16.511203+0000","last_clean":"2024-08-17T17:36:16.511203+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:36:16.511203+0000","last_undegraded":"2024-08-17T17:36:16.511203+0000","last_fullsized":"2024-08-17T17:36:16.511203+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"357'8680","reported_seq":10003,"reported_epoch":357,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:36.047297+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:36:36.047297+0000","last_peered":"2024-08-17T17:36:36.047297+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:36:36.047297+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:36:36.047297+0000","mapping_epoch":337,"log_start":"324'1841","ondisk_log_start":"324'1841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394788864,"num_objects":6024,"num_object_clones":0,"num_object_copies":18072,"num_objects_missing_on_primary":6715,"num_objects_missing":6715,"num_objects_degraded":6024,"num_objects_misplaced":6715,"num_objects_unfound":0,"num_objects_dirty":6024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8680,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":623,"num_bytes_recovered":40763392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":14},{"shards":"1,2,7","objects":6009},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":526,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334144+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:36:30.334144+0000","last_peered":"2024-08-17T17:36:30.334144+0000","last_clean":"2024-08-17T17:36:30.334144+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:36:30.334144+0000","last_undegraded":"2024-08-17T17:36:30.334144+0000","last_fullsized":"2024-08-17T17:36:30.334144+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":463,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334652+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:36:30.334652+0000","last_peered":"2024-08-17T17:36:30.334652+0000","last_clean":"2024-08-17T17:36:30.334652+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:36:30.334652+0000","last_undegraded":"2024-08-17T17:36:30.334652+0000","last_fullsized":"2024-08-17T17:36:30.334652+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"357'8748","reported_seq":9930,"reported_epoch":357,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:36.705707+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:36:36.705707+0000","last_peered":"2024-08-17T17:36:36.705707+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:36:36.705707+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:36:36.705707+0000","mapping_epoch":337,"log_start":"325'3144","ondisk_log_start":"325'3144","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402849816,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6148,"num_objects_misplaced":6898,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8748,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":550,"num_bytes_recovered":35979264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":6148}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":656,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:30.338636+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:36:30.338636+0000","last_peered":"2024-08-17T17:36:30.338636+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:36:30.338636+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":287,"reported_epoch":357,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:36:30.416586+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:36:30.416586+0000","last_peered":"2024-08-17T17:36:30.416586+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:36:30.416586+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:36:30.416586+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":505,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334368+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:36:30.334368+0000","last_peered":"2024-08-17T17:36:30.334368+0000","last_clean":"2024-08-17T17:36:30.334368+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:36:30.334368+0000","last_undegraded":"2024-08-17T17:36:30.334368+0000","last_fullsized":"2024-08-17T17:36:30.334368+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":321,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510470+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:36:16.510470+0000","last_peered":"2024-08-17T17:36:16.510470+0000","last_clean":"2024-08-17T17:36:16.510470+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:36:16.510470+0000","last_undegraded":"2024-08-17T17:36:16.510470+0000","last_fullsized":"2024-08-17T17:36:16.510470+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"357'8679","reported_seq":8774,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:37.820473+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:36:37.820473+0000","last_peered":"2024-08-17T17:36:37.820473+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:36:37.820473+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1621","ondisk_log_start":"324'1621","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397475840,"num_objects":6065,"num_object_clones":0,"num_object_copies":18195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8679,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-582},{"shards":"0,4,7","objects":6065}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":321,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334311+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:36:30.334311+0000","last_peered":"2024-08-17T17:36:30.334311+0000","last_clean":"2024-08-17T17:36:30.334311+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:36:30.334311+0000","last_undegraded":"2024-08-17T17:36:30.334311+0000","last_fullsized":"2024-08-17T17:36:30.334311+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":293,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509547+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:36:16.509547+0000","last_peered":"2024-08-17T17:36:16.509547+0000","last_clean":"2024-08-17T17:36:16.509547+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:36:16.509547+0000","last_undegraded":"2024-08-17T17:36:16.509547+0000","last_fullsized":"2024-08-17T17:36:16.509547+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":247,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.339500+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:36:30.339500+0000","last_peered":"2024-08-17T17:36:30.339500+0000","last_clean":"2024-08-17T17:36:30.339500+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:36:30.339500+0000","last_undegraded":"2024-08-17T17:36:30.339500+0000","last_fullsized":"2024-08-17T17:36:30.339500+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"357'8687","reported_seq":8743,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:36.973073+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:36:36.973073+0000","last_peered":"2024-08-17T17:36:36.973073+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:36:36.973073+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'2117","ondisk_log_start":"324'2117","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400359424,"num_objects":6109,"num_object_clones":0,"num_object_copies":18327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8687,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":6109},{"shards":"5,6","objects":-1187}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":413,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334127+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:36:30.334127+0000","last_peered":"2024-08-17T17:36:30.334127+0000","last_clean":"2024-08-17T17:36:30.334127+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:36:30.334127+0000","last_undegraded":"2024-08-17T17:36:30.334127+0000","last_fullsized":"2024-08-17T17:36:30.334127+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":93,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.512070+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:36:16.512070+0000","last_peered":"2024-08-17T17:36:16.512070+0000","last_clean":"2024-08-17T17:36:16.512070+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:36:16.512070+0000","last_undegraded":"2024-08-17T17:36:16.512070+0000","last_fullsized":"2024-08-17T17:36:16.512070+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":598,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335146+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:36:30.335146+0000","last_peered":"2024-08-17T17:36:30.335146+0000","last_clean":"2024-08-17T17:36:30.335146+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:36:30.335146+0000","last_undegraded":"2024-08-17T17:36:30.335146+0000","last_fullsized":"2024-08-17T17:36:30.335146+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"357'8569","reported_seq":9388,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:38.082613+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:36:38.082613+0000","last_peered":"2024-08-17T17:36:38.082613+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:36:38.082613+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1516","ondisk_log_start":"324'1516","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7053,"log_dups_size":0,"ondisk_log_size":7053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398262272,"num_objects":6077,"num_object_clones":0,"num_object_copies":18231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8569,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":6077},{"shards":"0,5","objects":-577}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":389,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334928+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:36:30.334928+0000","last_peered":"2024-08-17T17:36:30.334928+0000","last_clean":"2024-08-17T17:36:30.334928+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:36:30.334928+0000","last_undegraded":"2024-08-17T17:36:30.334928+0000","last_fullsized":"2024-08-17T17:36:30.334928+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":332,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511510+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:36:16.511510+0000","last_peered":"2024-08-17T17:36:16.511510+0000","last_clean":"2024-08-17T17:36:16.511510+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:36:16.511510+0000","last_undegraded":"2024-08-17T17:36:16.511510+0000","last_fullsized":"2024-08-17T17:36:16.511510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334102+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:36:30.334102+0000","last_peered":"2024-08-17T17:36:30.334102+0000","last_clean":"2024-08-17T17:36:30.334102+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:36:30.334102+0000","last_undegraded":"2024-08-17T17:36:30.334102+0000","last_fullsized":"2024-08-17T17:36:30.334102+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"357'8709","reported_seq":23647,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:35.115077+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:36:35.115077+0000","last_peered":"2024-08-17T17:36:35.115077+0000","last_clean":"2024-08-17T17:36:35.115077+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:36:35.115077+0000","last_undegraded":"2024-08-17T17:36:35.115077+0000","last_fullsized":"2024-08-17T17:36:35.115077+0000","mapping_epoch":355,"log_start":"324'2430","ondisk_log_start":"324'2430","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401539072,"num_objects":6127,"num_object_clones":0,"num_object_copies":18381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8709,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":272,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416798+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:36:30.416798+0000","last_peered":"2024-08-17T17:36:30.416798+0000","last_clean":"2024-08-17T17:36:30.416798+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:36:30.416798+0000","last_undegraded":"2024-08-17T17:36:30.416798+0000","last_fullsized":"2024-08-17T17:36:30.416798+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":53,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334007+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:36:30.334007+0000","last_peered":"2024-08-17T17:36:30.334007+0000","last_clean":"2024-08-17T17:36:30.334007+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:36:30.334007+0000","last_undegraded":"2024-08-17T17:36:30.334007+0000","last_fullsized":"2024-08-17T17:36:30.334007+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":548,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335221+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:36:30.335221+0000","last_peered":"2024-08-17T17:36:30.335221+0000","last_clean":"2024-08-17T17:36:30.335221+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:36:30.335221+0000","last_undegraded":"2024-08-17T17:36:30.335221+0000","last_fullsized":"2024-08-17T17:36:30.335221+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"357'8431","reported_seq":22937,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:34.910551+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:36:34.910551+0000","last_peered":"2024-08-17T17:36:34.910551+0000","last_clean":"2024-08-17T17:36:34.910551+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:36:34.910551+0000","last_undegraded":"2024-08-17T17:36:34.910551+0000","last_fullsized":"2024-08-17T17:36:34.910551+0000","mapping_epoch":345,"log_start":"324'2172","ondisk_log_start":"324'2172","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391446528,"num_objects":5973,"num_object_clones":0,"num_object_copies":17919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8431,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":268,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416821+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:36:30.416821+0000","last_peered":"2024-08-17T17:36:30.416821+0000","last_clean":"2024-08-17T17:36:30.416821+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:36:30.416821+0000","last_undegraded":"2024-08-17T17:36:30.416821+0000","last_fullsized":"2024-08-17T17:36:30.416821+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":304,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.338913+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:36:30.338913+0000","last_peered":"2024-08-17T17:36:30.338913+0000","last_clean":"2024-08-17T17:36:30.338913+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:36:30.338913+0000","last_undegraded":"2024-08-17T17:36:30.338913+0000","last_fullsized":"2024-08-17T17:36:30.338913+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":410,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416507+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:36:30.416507+0000","last_peered":"2024-08-17T17:36:30.416507+0000","last_clean":"2024-08-17T17:36:30.416507+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:36:30.416507+0000","last_undegraded":"2024-08-17T17:36:30.416507+0000","last_fullsized":"2024-08-17T17:36:30.416507+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"357'8704","reported_seq":8818,"reported_epoch":357,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:34.842112+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:36:34.842112+0000","last_peered":"2024-08-17T17:36:34.842112+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:36:34.842112+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2387","ondisk_log_start":"324'2387","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6317,"log_dups_size":0,"ondisk_log_size":6317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398458880,"num_objects":6080,"num_object_clones":0,"num_object_copies":18240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8704,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":6080},{"shards":"0,3","objects":-1168}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":579,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334781+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:36:30.334781+0000","last_peered":"2024-08-17T17:36:30.334781+0000","last_clean":"2024-08-17T17:36:30.334781+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:36:30.334781+0000","last_undegraded":"2024-08-17T17:36:30.334781+0000","last_fullsized":"2024-08-17T17:36:30.334781+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":547,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335587+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:36:30.335587+0000","last_peered":"2024-08-17T17:36:30.335587+0000","last_clean":"2024-08-17T17:36:30.335587+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:36:30.335587+0000","last_undegraded":"2024-08-17T17:36:30.335587+0000","last_fullsized":"2024-08-17T17:36:30.335587+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":564,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333892+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:36:30.333892+0000","last_peered":"2024-08-17T17:36:30.333892+0000","last_clean":"2024-08-17T17:36:30.333892+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:36:30.333892+0000","last_undegraded":"2024-08-17T17:36:30.333892+0000","last_fullsized":"2024-08-17T17:36:30.333892+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"357'8572","reported_seq":23226,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:36.506287+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:36:36.506287+0000","last_peered":"2024-08-17T17:36:36.506287+0000","last_clean":"2024-08-17T17:36:36.506287+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:36:36.506287+0000","last_undegraded":"2024-08-17T17:36:36.506287+0000","last_fullsized":"2024-08-17T17:36:36.506287+0000","mapping_epoch":347,"log_start":"324'1972","ondisk_log_start":"324'1972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6600,"log_dups_size":0,"ondisk_log_size":6600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393347072,"num_objects":6002,"num_object_clones":0,"num_object_copies":18006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8572,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":237,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.339179+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:36:30.339179+0000","last_peered":"2024-08-17T17:36:30.339179+0000","last_clean":"2024-08-17T17:36:30.339179+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:36:30.339179+0000","last_undegraded":"2024-08-17T17:36:30.339179+0000","last_fullsized":"2024-08-17T17:36:30.339179+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":361,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511967+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:36:16.511967+0000","last_peered":"2024-08-17T17:36:16.511967+0000","last_clean":"2024-08-17T17:36:16.511967+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:36:16.511967+0000","last_undegraded":"2024-08-17T17:36:16.511967+0000","last_fullsized":"2024-08-17T17:36:16.511967+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510784+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:36:16.510784+0000","last_peered":"2024-08-17T17:36:16.510784+0000","last_clean":"2024-08-17T17:36:16.510784+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:36:16.510784+0000","last_undegraded":"2024-08-17T17:36:16.510784+0000","last_fullsized":"2024-08-17T17:36:16.510784+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511410+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:36:16.511410+0000","last_peered":"2024-08-17T17:36:16.511410+0000","last_clean":"2024-08-17T17:36:16.511410+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:36:16.511410+0000","last_undegraded":"2024-08-17T17:36:16.511410+0000","last_fullsized":"2024-08-17T17:36:16.511410+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":354,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511026+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:36:16.511026+0000","last_peered":"2024-08-17T17:36:16.511026+0000","last_clean":"2024-08-17T17:36:16.511026+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:36:16.511026+0000","last_undegraded":"2024-08-17T17:36:16.511026+0000","last_fullsized":"2024-08-17T17:36:16.511026+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":554,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335020+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:36:30.335020+0000","last_peered":"2024-08-17T17:36:30.335020+0000","last_clean":"2024-08-17T17:36:30.335020+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:36:30.335020+0000","last_undegraded":"2024-08-17T17:36:30.335020+0000","last_fullsized":"2024-08-17T17:36:30.335020+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":94,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334847+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:36:30.334847+0000","last_peered":"2024-08-17T17:36:30.334847+0000","last_clean":"2024-08-17T17:36:30.334847+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:36:30.334847+0000","last_undegraded":"2024-08-17T17:36:30.334847+0000","last_fullsized":"2024-08-17T17:36:30.334847+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":262,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.416919+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:36:30.416919+0000","last_peered":"2024-08-17T17:36:30.416919+0000","last_clean":"2024-08-17T17:36:30.416919+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:36:30.416919+0000","last_undegraded":"2024-08-17T17:36:30.416919+0000","last_fullsized":"2024-08-17T17:36:30.416919+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":512,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333787+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:36:30.333787+0000","last_peered":"2024-08-17T17:36:30.333787+0000","last_clean":"2024-08-17T17:36:30.333787+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:36:30.333787+0000","last_undegraded":"2024-08-17T17:36:30.333787+0000","last_fullsized":"2024-08-17T17:36:30.333787+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":406,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.417170+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:36:30.417170+0000","last_peered":"2024-08-17T17:36:30.417170+0000","last_clean":"2024-08-17T17:36:30.417170+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:36:30.417170+0000","last_undegraded":"2024-08-17T17:36:30.417170+0000","last_fullsized":"2024-08-17T17:36:30.417170+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":506,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333733+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:36:30.333733+0000","last_peered":"2024-08-17T17:36:30.333733+0000","last_clean":"2024-08-17T17:36:30.333733+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:36:30.333733+0000","last_undegraded":"2024-08-17T17:36:30.333733+0000","last_fullsized":"2024-08-17T17:36:30.333733+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":328,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511838+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:36:16.511838+0000","last_peered":"2024-08-17T17:36:16.511838+0000","last_clean":"2024-08-17T17:36:16.511838+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:36:16.511838+0000","last_undegraded":"2024-08-17T17:36:16.511838+0000","last_fullsized":"2024-08-17T17:36:16.511838+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":537,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.333990+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:36:30.333990+0000","last_peered":"2024-08-17T17:36:30.333990+0000","last_clean":"2024-08-17T17:36:30.333990+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:36:30.333990+0000","last_undegraded":"2024-08-17T17:36:30.333990+0000","last_fullsized":"2024-08-17T17:36:30.333990+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335022+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:36:30.335022+0000","last_peered":"2024-08-17T17:36:30.335022+0000","last_clean":"2024-08-17T17:36:30.335022+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:36:30.335022+0000","last_undegraded":"2024-08-17T17:36:30.335022+0000","last_fullsized":"2024-08-17T17:36:30.335022+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":545,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334695+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:36:30.334695+0000","last_peered":"2024-08-17T17:36:30.334695+0000","last_clean":"2024-08-17T17:36:30.334695+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:36:30.334695+0000","last_undegraded":"2024-08-17T17:36:30.334695+0000","last_fullsized":"2024-08-17T17:36:30.334695+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":505,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334656+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:36:30.334656+0000","last_peered":"2024-08-17T17:36:30.334656+0000","last_clean":"2024-08-17T17:36:30.334656+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:36:30.334656+0000","last_undegraded":"2024-08-17T17:36:30.334656+0000","last_fullsized":"2024-08-17T17:36:30.334656+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":340,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510993+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:36:16.510993+0000","last_peered":"2024-08-17T17:36:16.510993+0000","last_clean":"2024-08-17T17:36:16.510993+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:36:16.510993+0000","last_undegraded":"2024-08-17T17:36:16.510993+0000","last_fullsized":"2024-08-17T17:36:16.510993+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":194,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334274+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:36:30.334274+0000","last_peered":"2024-08-17T17:36:30.334274+0000","last_clean":"2024-08-17T17:36:30.334274+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:36:30.334274+0000","last_undegraded":"2024-08-17T17:36:30.334274+0000","last_fullsized":"2024-08-17T17:36:30.334274+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":250,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.339183+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:36:30.339183+0000","last_peered":"2024-08-17T17:36:30.339183+0000","last_clean":"2024-08-17T17:36:30.339183+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:36:30.339183+0000","last_undegraded":"2024-08-17T17:36:30.339183+0000","last_fullsized":"2024-08-17T17:36:30.339183+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":348,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.511343+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:36:16.511343+0000","last_peered":"2024-08-17T17:36:16.511343+0000","last_clean":"2024-08-17T17:36:16.511343+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:36:16.511343+0000","last_undegraded":"2024-08-17T17:36:16.511343+0000","last_fullsized":"2024-08-17T17:36:16.511343+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":211,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510718+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:36:16.510718+0000","last_peered":"2024-08-17T17:36:16.510718+0000","last_clean":"2024-08-17T17:36:16.510718+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:36:16.510718+0000","last_undegraded":"2024-08-17T17:36:16.510718+0000","last_fullsized":"2024-08-17T17:36:16.510718+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":348,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510846+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:36:16.510846+0000","last_peered":"2024-08-17T17:36:16.510846+0000","last_clean":"2024-08-17T17:36:16.510846+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:36:16.510846+0000","last_undegraded":"2024-08-17T17:36:16.510846+0000","last_fullsized":"2024-08-17T17:36:16.510846+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":211,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.510167+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:36:16.510167+0000","last_peered":"2024-08-17T17:36:16.510167+0000","last_clean":"2024-08-17T17:36:16.510167+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:36:16.510167+0000","last_undegraded":"2024-08-17T17:36:16.510167+0000","last_fullsized":"2024-08-17T17:36:16.510167+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":557,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.334221+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:36:30.334221+0000","last_peered":"2024-08-17T17:36:30.334221+0000","last_clean":"2024-08-17T17:36:30.334221+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:36:30.334221+0000","last_undegraded":"2024-08-17T17:36:30.334221+0000","last_fullsized":"2024-08-17T17:36:30.334221+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":530,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.335103+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:36:30.335103+0000","last_peered":"2024-08-17T17:36:30.335103+0000","last_clean":"2024-08-17T17:36:30.335103+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:36:30.335103+0000","last_undegraded":"2024-08-17T17:36:30.335103+0000","last_fullsized":"2024-08-17T17:36:30.335103+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":311,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-17T17:36:16.509482+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:36:16.509482+0000","last_peered":"2024-08-17T17:36:16.509482+0000","last_clean":"2024-08-17T17:36:16.509482+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:36:16.509482+0000","last_undegraded":"2024-08-17T17:36:16.509482+0000","last_fullsized":"2024-08-17T17:36:16.509482+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":252,"reported_epoch":357,"state":"active+clean","last_fresh":"2024-08-17T17:36:30.339207+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:36:30.339207+0000","last_peered":"2024-08-17T17:36:30.339207+0000","last_clean":"2024-08-17T17:36:30.339207+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:36:30.339207+0000","last_undegraded":"2024-08-17T17:36:30.339207+0000","last_fullsized":"2024-08-17T17:36:30.339207+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6402277400,"num_objects":97692,"num_object_clones":0,"num_object_copies":293076,"num_objects_missing_on_primary":6715,"num_objects_missing":6715,"num_objects_degraded":56591,"num_objects_misplaced":38254,"num_objects_unfound":0,"num_objects_dirty":97692,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138698,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23995,"num_bytes_recovered":1449984000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20814508032,"data_stored":20814495816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104717,"ondisk_log_size":104717,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170042,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463412,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3634536,"kb_used_data":2732512,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":90110616,"statfs":{"total":95995035648,"available":92273270784,"internally_reserved":0,"allocated":2798092288,"data_stored":2796209948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492087,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174048,"kb_used_data":782868,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92571104,"statfs":{"total":95995035648,"available":94792810496,"internally_reserved":0,"allocated":801656832,"data_stored":799788999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986678,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2458780,"kb_used_data":1557844,"kb_used_omap":1,"kb_used_meta":900926,"kb_avail":91286372,"statfs":{"total":95995035648,"available":93477244928,"internally_reserved":0,"allocated":1595232256,"data_stored":1593360856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":922548568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225505,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4020684,"kb_used_data":3127172,"kb_used_omap":4,"kb_used_meta":893499,"kb_avail":89724468,"statfs":{"total":95995035648,"available":91877855232,"internally_reserved":0,"allocated":3202224128,"data_stored":3200364386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":914943424},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075831,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533723,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3765008,"kb_used_data":2743816,"kb_used_omap":5,"kb_used_meta":1021178,"kb_avail":89980144,"statfs":{"total":95995035648,"available":92139667456,"internally_reserved":0,"allocated":2809667584,"data_stored":2807802520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1045686512},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089700,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3349352,"kb_used_data":2358176,"kb_used_omap":5,"kb_used_meta":991162,"kb_avail":90395800,"statfs":{"total":95995035648,"available":92565299200,"internally_reserved":0,"allocated":2414772224,"data_stored":2412889943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1014949947},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2407202816,"data_stored":2407202816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2804617216,"data_stored":2804613144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4330885120,"data_stored":4330881048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3195797504,"data_stored":3195797504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1590689792,"data_stored":1590689792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":796917760,"data_stored":796917760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2792882176,"data_stored":2792882176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2895515648,"data_stored":2895511576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:36:44.010 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:36:44.092 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:44.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:44.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:44.226 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:44.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:44.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:44.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:44.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:44.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:44.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:44.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:44.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3905460387' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:44.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:44.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:44.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:44.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:44.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:44.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3905460387' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3905460387' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:44.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:44.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:45.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:45.120 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:45.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:45.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:45.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:45.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:45.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:45.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:45.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:45.618 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:45 smithi142 ceph-mon[106729]: pgmap v1283: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 56536/291675 objects degraded (19.383%); 38076/291675 objects misplaced (13.054%); 1005 KiB/s, 15 objects/s recovering 2024-08-17T17:36:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:45 smithi142 ceph-mon[103614]: pgmap v1283: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 56536/291675 objects degraded (19.383%); 38076/291675 objects misplaced (13.054%); 1005 KiB/s, 15 objects/s recovering 2024-08-17T17:36:45.663 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:45 smithi152 ceph-mon[93456]: pgmap v1283: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 56536/291675 objects degraded (19.383%); 38076/291675 objects misplaced (13.054%); 1005 KiB/s, 15 objects/s recovering 2024-08-17T17:36:45.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:45.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:45.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:45.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:45.917 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:45.917 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:45.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:45.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:45.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:46.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:46.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:46.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:46.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:46.229 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:46.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:46.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:46.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:46.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:46.460 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:36:46.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:46.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:46.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56536/291675 objects degraded (19.383%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:46 smithi152 ceph-mon[93456]: pgmap v1284: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 56511/291147 objects degraded (19.410%); 37994/291147 objects misplaced (13.050%); 992 KiB/s, 15 objects/s recovering 2024-08-17T17:36:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/54007645' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:46 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:46.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56536/291675 objects degraded (19.383%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[106729]: pgmap v1284: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 56511/291147 objects degraded (19.410%); 37994/291147 objects misplaced (13.050%); 992 KiB/s, 15 objects/s recovering 2024-08-17T17:36:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/54007645' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56536/291675 objects degraded (19.383%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[103614]: pgmap v1284: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 56511/291147 objects degraded (19.410%); 37994/291147 objects misplaced (13.050%); 992 KiB/s, 15 objects/s recovering 2024-08-17T17:36:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/54007645' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:46 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:36:46.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:46.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:46.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:46.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:46.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:46.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:46.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:46.971 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:46.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:47.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:47.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:47.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:47.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:47.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:47.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:47.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:47.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:47.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.496 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:47.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:47.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:47.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:47.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:47 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:47 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:47 smithi142 ceph-mon[103614]: osdmap e359: 8 total, 8 up, 6 in 2024-08-17T17:36:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:47 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:47 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:47.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:47 smithi142 ceph-mon[106729]: osdmap e359: 8 total, 8 up, 6 in 2024-08-17T17:36:47.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:47.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:47.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:47 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:47 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:47 smithi152 ceph-mon[93456]: osdmap e359: 8 total, 8 up, 6 in 2024-08-17T17:36:47.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:47.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:47.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:47.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:47.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:48.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:48.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:48.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:48.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:48.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:48.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:48.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:48.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:48.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:48 smithi142 ceph-mon[103614]: pgmap v1286: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 56492/290388 objects degraded (19.454%); 37937/290388 objects misplaced (13.064%); 988 KiB/s, 15 objects/s recovering 2024-08-17T17:36:48.607 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:48 smithi142 ceph-mon[106729]: pgmap v1286: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 56492/290388 objects degraded (19.454%); 37937/290388 objects misplaced (13.064%); 988 KiB/s, 15 objects/s recovering 2024-08-17T17:36:48.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:48.671 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:36:48.672 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 56492/290388 objects degraded (19.454%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1889,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":359,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":96796,"data_bytes":6342449125,"bytes_used":18742800384,"bytes_avail":557227413504,"bytes_total":575970213888,"degraded_objects":56492,"degraded_total":290388,"degraded_ratio":0.19453971927214619,"misplaced_objects":37937,"misplaced_total":290388,"misplaced_ratio":0.13064245078997755,"recovering_objects_per_sec":15,"recovering_bytes_per_sec":1011692,"recovering_keys_per_sec":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":93},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":31,"modified":"2024-08-17T17:36:14.254703+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:36:48.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.732 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:48 smithi152 ceph-mon[93456]: pgmap v1286: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 56492/290388 objects degraded (19.454%); 37937/290388 objects misplaced (13.064%); 988 KiB/s, 15 objects/s recovering 2024-08-17T17:36:48.734 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:48.734 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:48.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:48.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:48.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:48.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:48.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:49.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:49.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:49.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:49.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:49.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:49.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:49.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:49.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:36:49.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:49.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:49.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:49.609 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:49.699 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:49.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:49.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:49.737 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/876380800' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:36:49.738 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/876380800' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:36:49.761 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:36:49.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:36:49.796 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/876380800' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:36:49.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:49.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:49.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:49.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:50.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:50.107 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:50.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:50.209 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:50.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:50.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:50.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:50.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:50.465 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:36:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:36:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:50.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:50.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:50.599 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:50.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:50.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:50 smithi142 ceph-mon[106729]: pgmap v1287: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 56492/290388 objects degraded (19.454%); 37937/290388 objects misplaced (13.064%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:36:50.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:50 smithi142 ceph-mon[103614]: pgmap v1287: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 56492/290388 objects degraded (19.454%); 37937/290388 objects misplaced (13.064%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:36:50.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:50.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:50 smithi152 ceph-mon[93456]: pgmap v1287: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 56492/290388 objects degraded (19.454%); 37937/290388 objects misplaced (13.064%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:36:50.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.851 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:50.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:50.953 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:50.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:51.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:36:51.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:51.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:51.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:51.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:51.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:51.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:51.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:51.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:51.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:51.525 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56492/290388 objects degraded (19.454%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:51.528 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:51.528 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:51.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:51.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:51.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:51.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:51.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56492/290388 objects degraded (19.454%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:51.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56492/290388 objects degraded (19.454%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:51.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:51.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:51.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:51.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:51.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:52.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:52.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:36:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:36:51] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:36:52.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:52.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:36:52.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:52.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:52.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:52.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:52.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:52.466 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:52.483 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:52 smithi142 ceph-mon[106729]: pgmap v1288: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 56440/289278 objects degraded (19.511%); 37759/289278 objects misplaced (13.053%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:36:52.484 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:52 smithi142 ceph-mon[103614]: pgmap v1288: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 56440/289278 objects degraded (19.511%); 37759/289278 objects misplaced (13.053%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:36:52.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:52.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:52.527 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:52.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:52.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:52.691 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:36:52.691 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:52.710 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:36:52.710 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:52.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:52.727 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:36:52.727 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:52.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:52.743 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:36:52.744 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:52.760 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:36:52.761 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:52.776 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:36:52.777 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:52.794 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:36:52.794 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:52.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:52 smithi152 ceph-mon[93456]: pgmap v1288: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 56440/289278 objects degraded (19.511%); 37759/289278 objects misplaced (13.053%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:36:52.812 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:36:52.812 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:52.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:52.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:52.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:52.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:52.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:52.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:53.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:53.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:53.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:53.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:53.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:53.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:53.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:53.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:53.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:53.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:53.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:53.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:53.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:53.777 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2423170867' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:53.825 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2423170867' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:53.825 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2423170867' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:53.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:53.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:53.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:53.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:54.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:54.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:54.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.243 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:54.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:54.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:54.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:54.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:54.383 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:36:54.383 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:36:54.383 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:36:54.383 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:36:54.457 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:54.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:54.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:54.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:54.519 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:36:54.520 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:36:54.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:54.616 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:54.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:54.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:54.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:54.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:54.745 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:54 smithi152 ceph-mon[93456]: pgmap v1289: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 56440/289095 objects degraded (19.523%); 37759/289095 objects misplaced (13.061%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:36:54.745 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:54 smithi142 ceph-mon[106729]: pgmap v1289: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 56440/289095 objects degraded (19.523%); 37759/289095 objects misplaced (13.061%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:36:54.746 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:54 smithi142 ceph-mon[103614]: pgmap v1289: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 85 op/s; 56440/289095 objects degraded (19.523%); 37759/289095 objects misplaced (13.061%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:36:54.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:54.864 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:54.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:54.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:55.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:36:55.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:55.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:55.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:55.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:55.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:55.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:55.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:55.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:55.370 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:55.403 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1289,"stamp":"2024-08-17T17:36:54.266511+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6314203109,"num_objects":96365,"num_object_clones":0,"num_object_copies":289095,"num_objects_missing_on_primary":6641,"num_objects_missing":6641,"num_objects_degraded":56440,"num_objects_misplaced":37759,"num_objects_unfound":0,"num_objects_dirty":96365,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":140329,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24275,"num_bytes_recovered":1468015246,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104777,"ondisk_log_size":104777,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18236504,"kb_used_data":13119844,"kb_used_omap":27,"kb_used_meta":5116580,"kb_avail":544234408,"statfs":{"total":575970213888,"available":557296033792,"internally_reserved":0,"allocated":13434720256,"data_stored":13423405873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5239378664},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-56360960,"num_objects":-860,"num_object_clones":0,"num_object_copies":-2580,"num_objects_missing_on_primary":-50,"num_objects_missing":-50,"num_objects_degraded":-96,"num_objects_misplaced":-317,"num_objects_unfound":0,"num_objects_dirty":-860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":860,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":10223616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002738"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":559,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456661+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:36:46.456661+0000","last_peered":"2024-08-17T17:36:46.456661+0000","last_clean":"2024-08-17T17:36:46.456661+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:36:46.456661+0000","last_undegraded":"2024-08-17T17:36:46.456661+0000","last_fullsized":"2024-08-17T17:36:46.456661+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":68,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357335+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:36:38.357335+0000","last_peered":"2024-08-17T17:36:38.357335+0000","last_clean":"2024-08-17T17:36:38.357335+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:36:38.357335+0000","last_undegraded":"2024-08-17T17:36:38.357335+0000","last_fullsized":"2024-08-17T17:36:38.357335+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":262,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373386+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:36:38.373386+0000","last_peered":"2024-08-17T17:36:38.373386+0000","last_clean":"2024-08-17T17:36:38.373386+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:36:38.373386+0000","last_undegraded":"2024-08-17T17:36:38.373386+0000","last_fullsized":"2024-08-17T17:36:38.373386+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":257,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458902+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:36:46.458902+0000","last_peered":"2024-08-17T17:36:46.458902+0000","last_clean":"2024-08-17T17:36:46.458902+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:36:46.458902+0000","last_undegraded":"2024-08-17T17:36:46.458902+0000","last_fullsized":"2024-08-17T17:36:46.458902+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":434,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457266+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:36:46.457266+0000","last_peered":"2024-08-17T17:36:46.457266+0000","last_clean":"2024-08-17T17:36:46.457266+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:36:46.457266+0000","last_undegraded":"2024-08-17T17:36:46.457266+0000","last_fullsized":"2024-08-17T17:36:46.457266+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357176+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:36:38.357176+0000","last_peered":"2024-08-17T17:36:38.357176+0000","last_clean":"2024-08-17T17:36:38.357176+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:36:38.357176+0000","last_undegraded":"2024-08-17T17:36:38.357176+0000","last_fullsized":"2024-08-17T17:36:38.357176+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":528,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456382+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:36:46.456382+0000","last_peered":"2024-08-17T17:36:46.456382+0000","last_clean":"2024-08-17T17:36:46.456382+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:36:46.456382+0000","last_undegraded":"2024-08-17T17:36:46.456382+0000","last_fullsized":"2024-08-17T17:36:46.456382+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":379,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.361019+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:36:38.361019+0000","last_peered":"2024-08-17T17:36:38.361019+0000","last_clean":"2024-08-17T17:36:38.361019+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:36:38.361019+0000","last_undegraded":"2024-08-17T17:36:38.361019+0000","last_fullsized":"2024-08-17T17:36:38.361019+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"359'8708","reported_seq":10035,"reported_epoch":359,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:36:51.213973+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:36:51.213973+0000","last_peered":"2024-08-17T17:36:51.213973+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:36:51.213973+0000","last_undegraded":"2024-08-17T17:36:51.213973+0000","last_fullsized":"2024-08-17T17:36:51.213973+0000","mapping_epoch":337,"log_start":"324'1863","ondisk_log_start":"324'1863","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6845,"log_dups_size":0,"ondisk_log_size":6845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392953856,"num_objects":5996,"num_object_clones":0,"num_object_copies":17988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5995,"num_objects_unfound":0,"num_objects_dirty":5996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8708,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":628,"num_bytes_recovered":41156608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5995},{"shards":"4,6,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":64,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.460567+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:36:38.460567+0000","last_peered":"2024-08-17T17:36:38.460567+0000","last_clean":"2024-08-17T17:36:38.460567+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:36:38.460567+0000","last_undegraded":"2024-08-17T17:36:38.460567+0000","last_fullsized":"2024-08-17T17:36:38.460567+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":245,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373154+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:36:38.373154+0000","last_peered":"2024-08-17T17:36:38.373154+0000","last_clean":"2024-08-17T17:36:38.373154+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:36:38.373154+0000","last_undegraded":"2024-08-17T17:36:38.373154+0000","last_fullsized":"2024-08-17T17:36:38.373154+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"359'8957","reported_seq":9014,"reported_epoch":359,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:51.431898+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:36:51.431898+0000","last_peered":"2024-08-17T17:36:51.431898+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:36:51.431898+0000","last_undegraded":"2024-08-17T17:36:51.431898+0000","last_fullsized":"2024-08-17T17:36:51.431898+0000","mapping_epoch":337,"log_start":"324'2382","ondisk_log_start":"324'2382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6575,"log_dups_size":0,"ondisk_log_size":6575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400883712,"num_objects":6117,"num_object_clones":0,"num_object_copies":18351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6117,"num_objects_unfound":0,"num_objects_dirty":6117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8957,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":6117}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":66,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357549+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:36:38.357549+0000","last_peered":"2024-08-17T17:36:38.357549+0000","last_clean":"2024-08-17T17:36:38.357549+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:36:38.357549+0000","last_undegraded":"2024-08-17T17:36:38.357549+0000","last_fullsized":"2024-08-17T17:36:38.357549+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":330,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357145+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:36:38.357145+0000","last_peered":"2024-08-17T17:36:38.357145+0000","last_clean":"2024-08-17T17:36:38.357145+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:36:38.357145+0000","last_undegraded":"2024-08-17T17:36:38.357145+0000","last_fullsized":"2024-08-17T17:36:38.357145+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"359'8696","reported_seq":8756,"reported_epoch":359,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:50.385695+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:36:50.385695+0000","last_peered":"2024-08-17T17:36:50.385695+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:36:50.385695+0000","last_undegraded":"2024-08-17T17:36:50.385695+0000","last_fullsized":"2024-08-17T17:36:50.385695+0000","mapping_epoch":337,"log_start":"324'1853","ondisk_log_start":"324'1853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395706368,"num_objects":6038,"num_object_clones":0,"num_object_copies":18114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6038,"num_objects_unfound":0,"num_objects_dirty":6038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8696,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":6038}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":62,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373321+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:36:38.373321+0000","last_peered":"2024-08-17T17:36:38.373321+0000","last_clean":"2024-08-17T17:36:38.373321+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:36:38.373321+0000","last_undegraded":"2024-08-17T17:36:38.373321+0000","last_fullsized":"2024-08-17T17:36:38.373321+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":340,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.360602+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:36:38.360602+0000","last_peered":"2024-08-17T17:36:38.360602+0000","last_clean":"2024-08-17T17:36:38.360602+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:36:38.360602+0000","last_undegraded":"2024-08-17T17:36:38.360602+0000","last_fullsized":"2024-08-17T17:36:38.360602+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"359'8858","reported_seq":8954,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:51.824043+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:36:51.824043+0000","last_peered":"2024-08-17T17:36:51.824043+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:36:51.824043+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'1813","ondisk_log_start":"324'1813","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7045,"log_dups_size":0,"ondisk_log_size":7045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399769600,"num_objects":6100,"num_object_clones":0,"num_object_copies":18300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8858,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-669},{"shards":"0,3,7","objects":6100}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":471,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456821+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:36:46.456821+0000","last_peered":"2024-08-17T17:36:46.456821+0000","last_clean":"2024-08-17T17:36:46.456821+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:36:46.456821+0000","last_undegraded":"2024-08-17T17:36:46.456821+0000","last_fullsized":"2024-08-17T17:36:46.456821+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":237,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.565039+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:36:46.565039+0000","last_peered":"2024-08-17T17:36:46.565039+0000","last_clean":"2024-08-17T17:36:46.565039+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:36:46.565039+0000","last_undegraded":"2024-08-17T17:36:46.565039+0000","last_fullsized":"2024-08-17T17:36:46.565039+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"359'8633","reported_seq":8681,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:51.612432+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:36:51.612432+0000","last_peered":"2024-08-17T17:36:51.612432+0000","last_clean":"2024-08-17T17:36:51.612432+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:36:51.612432+0000","last_undegraded":"2024-08-17T17:36:51.612432+0000","last_fullsized":"2024-08-17T17:36:51.612432+0000","mapping_epoch":319,"log_start":"324'2074","ondisk_log_start":"324'2074","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390660096,"num_objects":5961,"num_object_clones":0,"num_object_copies":17883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8633,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":560,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.459186+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:36:46.459186+0000","last_peered":"2024-08-17T17:36:46.459186+0000","last_clean":"2024-08-17T17:36:46.459186+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:36:46.459186+0000","last_undegraded":"2024-08-17T17:36:46.459186+0000","last_fullsized":"2024-08-17T17:36:46.459186+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":313,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357343+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:36:38.357343+0000","last_peered":"2024-08-17T17:36:38.357343+0000","last_clean":"2024-08-17T17:36:38.357343+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:36:38.357343+0000","last_undegraded":"2024-08-17T17:36:38.357343+0000","last_fullsized":"2024-08-17T17:36:38.357343+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"359'8801","reported_seq":8859,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:51.815773+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:36:51.815773+0000","last_peered":"2024-08-17T17:36:51.815773+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:36:51.815773+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'2235","ondisk_log_start":"324'2235","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6566,"log_dups_size":0,"ondisk_log_size":6566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401801216,"num_objects":6131,"num_object_clones":0,"num_object_copies":18393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8801,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":6131},{"shards":"1,6","objects":-1235}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":314,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.360679+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:36:38.360679+0000","last_peered":"2024-08-17T17:36:38.360679+0000","last_clean":"2024-08-17T17:36:38.360679+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:36:38.360679+0000","last_undegraded":"2024-08-17T17:36:38.360679+0000","last_fullsized":"2024-08-17T17:36:38.360679+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":340,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.358035+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:36:38.358035+0000","last_peered":"2024-08-17T17:36:38.358035+0000","last_clean":"2024-08-17T17:36:38.358035+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:36:38.358035+0000","last_undegraded":"2024-08-17T17:36:38.358035+0000","last_fullsized":"2024-08-17T17:36:38.358035+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"359'8862","reported_seq":8959,"reported_epoch":359,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:36:51.720312+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:36:51.720312+0000","last_peered":"2024-08-17T17:36:51.720312+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:36:51.720312+0000","last_undegraded":"2024-08-17T17:36:51.720312+0000","last_fullsized":"2024-08-17T17:36:51.720312+0000","mapping_epoch":337,"log_start":"325'3261","ondisk_log_start":"325'3261","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5601,"log_dups_size":0,"ondisk_log_size":5601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6148,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8862,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":6148}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":351,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357519+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:36:38.357519+0000","last_peered":"2024-08-17T17:36:38.357519+0000","last_clean":"2024-08-17T17:36:38.357519+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:36:38.357519+0000","last_undegraded":"2024-08-17T17:36:38.357519+0000","last_fullsized":"2024-08-17T17:36:38.357519+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":313,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357670+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:36:38.357670+0000","last_peered":"2024-08-17T17:36:38.357670+0000","last_clean":"2024-08-17T17:36:38.357670+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:36:38.357670+0000","last_undegraded":"2024-08-17T17:36:38.357670+0000","last_fullsized":"2024-08-17T17:36:38.357670+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"359'8756","reported_seq":10231,"reported_epoch":359,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:51.133022+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:36:51.133022+0000","last_peered":"2024-08-17T17:36:51.133022+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:36:51.133022+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:36:51.133022+0000","mapping_epoch":337,"log_start":"324'1841","ondisk_log_start":"324'1841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389808128,"num_objects":5948,"num_object_clones":0,"num_object_copies":17844,"num_objects_missing_on_primary":6641,"num_objects_missing":6641,"num_objects_degraded":5948,"num_objects_misplaced":6641,"num_objects_unfound":0,"num_objects_dirty":5948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8756,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":699,"num_bytes_recovered":45744128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":5935},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":530,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458133+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:36:46.458133+0000","last_peered":"2024-08-17T17:36:46.458133+0000","last_clean":"2024-08-17T17:36:46.458133+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:36:46.458133+0000","last_undegraded":"2024-08-17T17:36:46.458133+0000","last_fullsized":"2024-08-17T17:36:46.458133+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":467,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456429+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:36:46.456429+0000","last_peered":"2024-08-17T17:36:46.456429+0000","last_clean":"2024-08-17T17:36:46.456429+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:36:46.456429+0000","last_undegraded":"2024-08-17T17:36:46.456429+0000","last_fullsized":"2024-08-17T17:36:46.456429+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"359'8826","reported_seq":10168,"reported_epoch":359,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:36:51.875551+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:36:51.875551+0000","last_peered":"2024-08-17T17:36:51.875551+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:36:51.875551+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:36:51.875551+0000","mapping_epoch":337,"log_start":"325'3244","ondisk_log_start":"325'3244","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5582,"log_dups_size":0,"ondisk_log_size":5582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397738008,"num_objects":6070,"num_object_clones":0,"num_object_copies":18210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6070,"num_objects_misplaced":6820,"num_objects_unfound":0,"num_objects_dirty":6070,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8826,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":628,"num_bytes_recovered":41091072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":6070}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":660,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:46.562610+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:36:46.562610+0000","last_peered":"2024-08-17T17:36:46.562610+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:36:46.562610+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":288,"reported_epoch":358,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:36:38.373658+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:36:38.373658+0000","last_peered":"2024-08-17T17:36:38.373658+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:36:38.373658+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:36:38.373658+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":509,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458853+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:36:46.458853+0000","last_peered":"2024-08-17T17:36:46.458853+0000","last_clean":"2024-08-17T17:36:46.458853+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:36:46.458853+0000","last_undegraded":"2024-08-17T17:36:46.458853+0000","last_fullsized":"2024-08-17T17:36:46.458853+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":323,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.361520+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:36:38.361520+0000","last_peered":"2024-08-17T17:36:38.361520+0000","last_clean":"2024-08-17T17:36:38.361520+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:36:38.361520+0000","last_undegraded":"2024-08-17T17:36:38.361520+0000","last_fullsized":"2024-08-17T17:36:38.361520+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"359'8758","reported_seq":8857,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:51.830728+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:36:51.830728+0000","last_peered":"2024-08-17T17:36:51.830728+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:36:51.830728+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1721","ondisk_log_start":"324'1721","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8758,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-661},{"shards":"0,4,7","objects":5986}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":325,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456453+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:36:46.456453+0000","last_peered":"2024-08-17T17:36:46.456453+0000","last_clean":"2024-08-17T17:36:46.456453+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:36:46.456453+0000","last_undegraded":"2024-08-17T17:36:46.456453+0000","last_fullsized":"2024-08-17T17:36:46.456453+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":295,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357088+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:36:38.357088+0000","last_peered":"2024-08-17T17:36:38.357088+0000","last_clean":"2024-08-17T17:36:38.357088+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:36:38.357088+0000","last_undegraded":"2024-08-17T17:36:38.357088+0000","last_fullsized":"2024-08-17T17:36:38.357088+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":251,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.563144+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:36:46.563144+0000","last_peered":"2024-08-17T17:36:46.563144+0000","last_clean":"2024-08-17T17:36:46.563144+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:36:46.563144+0000","last_undegraded":"2024-08-17T17:36:46.563144+0000","last_fullsized":"2024-08-17T17:36:46.563144+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"359'8775","reported_seq":8833,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:51.868624+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:36:51.868624+0000","last_peered":"2024-08-17T17:36:51.868624+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:36:51.868624+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'2217","ondisk_log_start":"324'2217","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394592256,"num_objects":6021,"num_object_clones":0,"num_object_copies":18063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8775,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":6021},{"shards":"5,6","objects":-1275}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":417,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458629+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:36:46.458629+0000","last_peered":"2024-08-17T17:36:46.458629+0000","last_clean":"2024-08-17T17:36:46.458629+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:36:46.458629+0000","last_undegraded":"2024-08-17T17:36:46.458629+0000","last_fullsized":"2024-08-17T17:36:46.458629+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.360538+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:36:38.360538+0000","last_peered":"2024-08-17T17:36:38.360538+0000","last_clean":"2024-08-17T17:36:38.360538+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:36:38.360538+0000","last_undegraded":"2024-08-17T17:36:38.360538+0000","last_fullsized":"2024-08-17T17:36:38.360538+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":602,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457800+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:36:46.457800+0000","last_peered":"2024-08-17T17:36:46.457800+0000","last_clean":"2024-08-17T17:36:46.457800+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:36:46.457800+0000","last_undegraded":"2024-08-17T17:36:46.457800+0000","last_fullsized":"2024-08-17T17:36:46.457800+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"359'8660","reported_seq":9483,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:53.003795+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:36:53.003795+0000","last_peered":"2024-08-17T17:36:53.003795+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:36:53.003795+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1616","ondisk_log_start":"324'1616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8660,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5986},{"shards":"0,5","objects":-668}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":393,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456282+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:36:46.456282+0000","last_peered":"2024-08-17T17:36:46.456282+0000","last_clean":"2024-08-17T17:36:46.456282+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:36:46.456282+0000","last_undegraded":"2024-08-17T17:36:46.456282+0000","last_fullsized":"2024-08-17T17:36:46.456282+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":334,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357638+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:36:38.357638+0000","last_peered":"2024-08-17T17:36:38.357638+0000","last_clean":"2024-08-17T17:36:38.357638+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:36:38.357638+0000","last_undegraded":"2024-08-17T17:36:38.357638+0000","last_fullsized":"2024-08-17T17:36:38.357638+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":188,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456576+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:36:46.456576+0000","last_peered":"2024-08-17T17:36:46.456576+0000","last_clean":"2024-08-17T17:36:46.456576+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:36:46.456576+0000","last_undegraded":"2024-08-17T17:36:46.456576+0000","last_fullsized":"2024-08-17T17:36:46.456576+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"359'8783","reported_seq":23725,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:50.054297+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:36:50.054297+0000","last_peered":"2024-08-17T17:36:50.054297+0000","last_clean":"2024-08-17T17:36:50.054297+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:36:50.054297+0000","last_undegraded":"2024-08-17T17:36:50.054297+0000","last_fullsized":"2024-08-17T17:36:50.054297+0000","mapping_epoch":355,"log_start":"324'2530","ondisk_log_start":"324'2530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396689408,"num_objects":6053,"num_object_clones":0,"num_object_copies":18159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8783,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":273,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373246+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:36:38.373246+0000","last_peered":"2024-08-17T17:36:38.373246+0000","last_clean":"2024-08-17T17:36:38.373246+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:36:38.373246+0000","last_undegraded":"2024-08-17T17:36:38.373246+0000","last_fullsized":"2024-08-17T17:36:38.373246+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":57,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456077+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:36:46.456077+0000","last_peered":"2024-08-17T17:36:46.456077+0000","last_clean":"2024-08-17T17:36:46.456077+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:36:46.456077+0000","last_undegraded":"2024-08-17T17:36:46.456077+0000","last_fullsized":"2024-08-17T17:36:46.456077+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":552,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456582+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:36:46.456582+0000","last_peered":"2024-08-17T17:36:46.456582+0000","last_clean":"2024-08-17T17:36:46.456582+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:36:46.456582+0000","last_undegraded":"2024-08-17T17:36:46.456582+0000","last_fullsized":"2024-08-17T17:36:46.456582+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"359'8531","reported_seq":23041,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:50.179916+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:36:50.179916+0000","last_peered":"2024-08-17T17:36:50.179916+0000","last_clean":"2024-08-17T17:36:50.179916+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:36:50.179916+0000","last_undegraded":"2024-08-17T17:36:50.179916+0000","last_fullsized":"2024-08-17T17:36:50.179916+0000","mapping_epoch":345,"log_start":"324'2272","ondisk_log_start":"324'2272","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384892928,"num_objects":5873,"num_object_clones":0,"num_object_copies":17619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8531,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":269,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373999+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:36:38.373999+0000","last_peered":"2024-08-17T17:36:38.373999+0000","last_clean":"2024-08-17T17:36:38.373999+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:36:38.373999+0000","last_undegraded":"2024-08-17T17:36:38.373999+0000","last_fullsized":"2024-08-17T17:36:38.373999+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":308,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.562901+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:36:46.562901+0000","last_peered":"2024-08-17T17:36:46.562901+0000","last_clean":"2024-08-17T17:36:46.562901+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:36:46.562901+0000","last_undegraded":"2024-08-17T17:36:46.562901+0000","last_fullsized":"2024-08-17T17:36:46.562901+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":411,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373569+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:36:38.373569+0000","last_peered":"2024-08-17T17:36:38.373569+0000","last_clean":"2024-08-17T17:36:38.373569+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:36:38.373569+0000","last_undegraded":"2024-08-17T17:36:38.373569+0000","last_fullsized":"2024-08-17T17:36:38.373569+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"359'8790","reported_seq":8908,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:50.309666+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:36:50.309666+0000","last_peered":"2024-08-17T17:36:50.309666+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:36:50.309666+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2487","ondisk_log_start":"324'2487","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392822784,"num_objects":5994,"num_object_clones":0,"num_object_copies":17982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8790,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5994},{"shards":"0,3","objects":-1254}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":583,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457068+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:36:46.457068+0000","last_peered":"2024-08-17T17:36:46.457068+0000","last_clean":"2024-08-17T17:36:46.457068+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:36:46.457068+0000","last_undegraded":"2024-08-17T17:36:46.457068+0000","last_fullsized":"2024-08-17T17:36:46.457068+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":551,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457429+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:36:46.457429+0000","last_peered":"2024-08-17T17:36:46.457429+0000","last_clean":"2024-08-17T17:36:46.457429+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:36:46.457429+0000","last_undegraded":"2024-08-17T17:36:46.457429+0000","last_fullsized":"2024-08-17T17:36:46.457429+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":569,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457945+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:36:46.457945+0000","last_peered":"2024-08-17T17:36:46.457945+0000","last_clean":"2024-08-17T17:36:46.457945+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:36:46.457945+0000","last_undegraded":"2024-08-17T17:36:46.457945+0000","last_fullsized":"2024-08-17T17:36:46.457945+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"359'8657","reported_seq":23313,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:51.438899+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:36:51.438899+0000","last_peered":"2024-08-17T17:36:51.438899+0000","last_clean":"2024-08-17T17:36:51.438899+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:36:51.438899+0000","last_undegraded":"2024-08-17T17:36:51.438899+0000","last_fullsized":"2024-08-17T17:36:51.438899+0000","mapping_epoch":347,"log_start":"324'2072","ondisk_log_start":"324'2072","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387776512,"num_objects":5917,"num_object_clones":0,"num_object_copies":17751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8657,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":241,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.563049+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:36:46.563049+0000","last_peered":"2024-08-17T17:36:46.563049+0000","last_clean":"2024-08-17T17:36:46.563049+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:36:46.563049+0000","last_undegraded":"2024-08-17T17:36:46.563049+0000","last_fullsized":"2024-08-17T17:36:46.563049+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":363,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.361451+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:36:38.361451+0000","last_peered":"2024-08-17T17:36:38.361451+0000","last_clean":"2024-08-17T17:36:38.361451+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:36:38.361451+0000","last_undegraded":"2024-08-17T17:36:38.361451+0000","last_fullsized":"2024-08-17T17:36:38.361451+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":313,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357213+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:36:38.357213+0000","last_peered":"2024-08-17T17:36:38.357213+0000","last_clean":"2024-08-17T17:36:38.357213+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:36:38.357213+0000","last_undegraded":"2024-08-17T17:36:38.357213+0000","last_fullsized":"2024-08-17T17:36:38.357213+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":313,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357510+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:36:38.357510+0000","last_peered":"2024-08-17T17:36:38.357510+0000","last_clean":"2024-08-17T17:36:38.357510+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:36:38.357510+0000","last_undegraded":"2024-08-17T17:36:38.357510+0000","last_fullsized":"2024-08-17T17:36:38.357510+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":356,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.358120+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:36:38.358120+0000","last_peered":"2024-08-17T17:36:38.358120+0000","last_clean":"2024-08-17T17:36:38.358120+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:36:38.358120+0000","last_undegraded":"2024-08-17T17:36:38.358120+0000","last_fullsized":"2024-08-17T17:36:38.358120+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":558,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456384+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:36:46.456384+0000","last_peered":"2024-08-17T17:36:46.456384+0000","last_clean":"2024-08-17T17:36:46.456384+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:36:46.456384+0000","last_undegraded":"2024-08-17T17:36:46.456384+0000","last_fullsized":"2024-08-17T17:36:46.456384+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":98,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457778+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:36:46.457778+0000","last_peered":"2024-08-17T17:36:46.457778+0000","last_clean":"2024-08-17T17:36:46.457778+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:36:46.457778+0000","last_undegraded":"2024-08-17T17:36:46.457778+0000","last_fullsized":"2024-08-17T17:36:46.457778+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":263,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.373562+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:36:38.373562+0000","last_peered":"2024-08-17T17:36:38.373562+0000","last_clean":"2024-08-17T17:36:38.373562+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:36:38.373562+0000","last_undegraded":"2024-08-17T17:36:38.373562+0000","last_fullsized":"2024-08-17T17:36:38.373562+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":516,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458387+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:36:46.458387+0000","last_peered":"2024-08-17T17:36:46.458387+0000","last_clean":"2024-08-17T17:36:46.458387+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:36:46.458387+0000","last_undegraded":"2024-08-17T17:36:46.458387+0000","last_fullsized":"2024-08-17T17:36:46.458387+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":407,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.375381+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:36:38.375381+0000","last_peered":"2024-08-17T17:36:38.375381+0000","last_clean":"2024-08-17T17:36:38.375381+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:36:38.375381+0000","last_undegraded":"2024-08-17T17:36:38.375381+0000","last_fullsized":"2024-08-17T17:36:38.375381+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":510,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456072+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:36:46.456072+0000","last_peered":"2024-08-17T17:36:46.456072+0000","last_clean":"2024-08-17T17:36:46.456072+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:36:46.456072+0000","last_undegraded":"2024-08-17T17:36:46.456072+0000","last_fullsized":"2024-08-17T17:36:46.456072+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":330,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.361389+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:36:38.361389+0000","last_peered":"2024-08-17T17:36:38.361389+0000","last_clean":"2024-08-17T17:36:38.361389+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:36:38.361389+0000","last_undegraded":"2024-08-17T17:36:38.361389+0000","last_fullsized":"2024-08-17T17:36:38.361389+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":542,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458074+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:36:46.458074+0000","last_peered":"2024-08-17T17:36:46.458074+0000","last_clean":"2024-08-17T17:36:46.458074+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:36:46.458074+0000","last_undegraded":"2024-08-17T17:36:46.458074+0000","last_fullsized":"2024-08-17T17:36:46.458074+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":257,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457200+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:36:46.457200+0000","last_peered":"2024-08-17T17:36:46.457200+0000","last_clean":"2024-08-17T17:36:46.457200+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:36:46.457200+0000","last_undegraded":"2024-08-17T17:36:46.457200+0000","last_fullsized":"2024-08-17T17:36:46.457200+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":550,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458288+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:36:46.458288+0000","last_peered":"2024-08-17T17:36:46.458288+0000","last_clean":"2024-08-17T17:36:46.458288+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:36:46.458288+0000","last_undegraded":"2024-08-17T17:36:46.458288+0000","last_fullsized":"2024-08-17T17:36:46.458288+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":509,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457623+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:36:46.457623+0000","last_peered":"2024-08-17T17:36:46.457623+0000","last_clean":"2024-08-17T17:36:46.457623+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:36:46.457623+0000","last_undegraded":"2024-08-17T17:36:46.457623+0000","last_fullsized":"2024-08-17T17:36:46.457623+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":342,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.358243+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:36:38.358243+0000","last_peered":"2024-08-17T17:36:38.358243+0000","last_clean":"2024-08-17T17:36:38.358243+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:36:38.358243+0000","last_undegraded":"2024-08-17T17:36:38.358243+0000","last_fullsized":"2024-08-17T17:36:38.358243+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":198,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.456742+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:36:46.456742+0000","last_peered":"2024-08-17T17:36:46.456742+0000","last_clean":"2024-08-17T17:36:46.456742+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:36:46.456742+0000","last_undegraded":"2024-08-17T17:36:46.456742+0000","last_fullsized":"2024-08-17T17:36:46.456742+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":254,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.562954+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:36:46.562954+0000","last_peered":"2024-08-17T17:36:46.562954+0000","last_clean":"2024-08-17T17:36:46.562954+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:36:46.562954+0000","last_undegraded":"2024-08-17T17:36:46.562954+0000","last_fullsized":"2024-08-17T17:36:46.562954+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":350,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357480+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:36:38.357480+0000","last_peered":"2024-08-17T17:36:38.357480+0000","last_clean":"2024-08-17T17:36:38.357480+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:36:38.357480+0000","last_undegraded":"2024-08-17T17:36:38.357480+0000","last_fullsized":"2024-08-17T17:36:38.357480+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":213,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357426+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:36:38.357426+0000","last_peered":"2024-08-17T17:36:38.357426+0000","last_clean":"2024-08-17T17:36:38.357426+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:36:38.357426+0000","last_undegraded":"2024-08-17T17:36:38.357426+0000","last_fullsized":"2024-08-17T17:36:38.357426+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":350,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.358173+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:36:38.358173+0000","last_peered":"2024-08-17T17:36:38.358173+0000","last_clean":"2024-08-17T17:36:38.358173+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:36:38.358173+0000","last_undegraded":"2024-08-17T17:36:38.358173+0000","last_fullsized":"2024-08-17T17:36:38.358173+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":213,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357919+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:36:38.357919+0000","last_peered":"2024-08-17T17:36:38.357919+0000","last_clean":"2024-08-17T17:36:38.357919+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:36:38.357919+0000","last_undegraded":"2024-08-17T17:36:38.357919+0000","last_fullsized":"2024-08-17T17:36:38.357919+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":562,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.458722+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:36:46.458722+0000","last_peered":"2024-08-17T17:36:46.458722+0000","last_clean":"2024-08-17T17:36:46.458722+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:36:46.458722+0000","last_undegraded":"2024-08-17T17:36:46.458722+0000","last_fullsized":"2024-08-17T17:36:46.458722+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":535,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.457421+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:36:46.457421+0000","last_peered":"2024-08-17T17:36:46.457421+0000","last_clean":"2024-08-17T17:36:46.457421+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:36:46.457421+0000","last_undegraded":"2024-08-17T17:36:46.457421+0000","last_fullsized":"2024-08-17T17:36:46.457421+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":313,"reported_epoch":358,"state":"active+clean","last_fresh":"2024-08-17T17:36:38.357253+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:36:38.357253+0000","last_peered":"2024-08-17T17:36:38.357253+0000","last_clean":"2024-08-17T17:36:38.357253+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:36:38.357253+0000","last_undegraded":"2024-08-17T17:36:38.357253+0000","last_fullsized":"2024-08-17T17:36:38.357253+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":256,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.563258+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:36:46.563258+0000","last_peered":"2024-08-17T17:36:46.563258+0000","last_clean":"2024-08-17T17:36:46.563258+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:36:46.563258+0000","last_undegraded":"2024-08-17T17:36:46.563258+0000","last_fullsized":"2024-08-17T17:36:46.563258+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6313607192,"num_objects":96339,"num_object_clones":0,"num_object_copies":289017,"num_objects_missing_on_primary":6641,"num_objects_missing":6641,"num_objects_degraded":56437,"num_objects_misplaced":37759,"num_objects_unfound":0,"num_objects_dirty":96339,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140051,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24234,"num_bytes_recovered":1465647104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20541878272,"data_stored":20541866056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104570,"ondisk_log_size":104570,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170045,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463415,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3583240,"kb_used_data":2693696,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90161912,"statfs":{"total":95995035648,"available":92325797888,"internally_reserved":0,"allocated":2758344704,"data_stored":2756446778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492089,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1166000,"kb_used_data":774820,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92579152,"statfs":{"total":95995035648,"available":94801051648,"internally_reserved":0,"allocated":793415680,"data_stored":791540054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986681,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2440700,"kb_used_data":1535604,"kb_used_omap":1,"kb_used_meta":905086,"kb_avail":91304452,"statfs":{"total":95995035648,"available":93495758848,"internally_reserved":0,"allocated":1572458496,"data_stored":1570571510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926808408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225508,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3981228,"kb_used_data":3083556,"kb_used_omap":4,"kb_used_meta":897659,"kb_avail":89763924,"statfs":{"total":95995035648,"available":91918258176,"internally_reserved":0,"allocated":3157561344,"data_stored":3155686016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":919203264},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075833,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533726,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3744816,"kb_used_data":2706984,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90000336,"statfs":{"total":95995035648,"available":92160344064,"internally_reserved":0,"allocated":2771951616,"data_stored":2770070966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089703,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3320520,"kb_used_data":2325184,"kb_used_omap":5,"kb_used_meta":995322,"kb_avail":90424632,"statfs":{"total":95995035648,"available":92594823168,"internally_reserved":0,"allocated":2380988416,"data_stored":2379090549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1019209787},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2374959104,"data_stored":2374959104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2766540800,"data_stored":2766536728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4271902720,"data_stored":4271898648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3151036416,"data_stored":3151036416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1567358976,"data_stored":1567358976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":788725760,"data_stored":788725760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2753101824,"data_stored":2753101824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2868252672,"data_stored":2868248600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:36:55.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:55.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:55.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:55.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:55.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:55.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:55.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:55.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:55.801 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:55 smithi152 ceph-mon[93456]: from='client.37170 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:55.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:55.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:55.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:55.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:55 smithi142 ceph-mon[106729]: from='client.37170 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:55 smithi142 ceph-mon[103614]: from='client.37170 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:36:55.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:55.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:56.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:56.105 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:56.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:56.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:56.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:56.355 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:56.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:56.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:56.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56440/289095 objects degraded (19.523%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:56 smithi152 ceph-mon[93456]: pgmap v1290: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 56440/288873 objects degraded (19.538%); 37759/288873 objects misplaced (13.071%); 672 KiB/s, 10 objects/s recovering 2024-08-17T17:36:56.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:56.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:56.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:56.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:56.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.731 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:56.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:56.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:56.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:56.870 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56440/289095 objects degraded (19.523%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:56.870 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:56 smithi142 ceph-mon[106729]: pgmap v1290: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 56440/288873 objects degraded (19.538%); 37759/288873 objects misplaced (13.071%); 672 KiB/s, 10 objects/s recovering 2024-08-17T17:36:56.870 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:56.870 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56440/289095 objects degraded (19.523%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:56.870 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:56 smithi142 ceph-mon[103614]: pgmap v1290: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 56440/288873 objects degraded (19.538%); 37759/288873 objects misplaced (13.071%); 672 KiB/s, 10 objects/s recovering 2024-08-17T17:36:56.870 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:56.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:56.978 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:57.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:57.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:57.132 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:36:57.132 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:36:57.133 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:36:57.133 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:36:57.133 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:36:57.186 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:57.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:57.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:57.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:57.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:57.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:57.482 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:57.584 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:57.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:57.664 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:36:57.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:57.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3267697284' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:57.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:57.836 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3267697284' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:57.836 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3267697284' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:57.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:36:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:57.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:57.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:57.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:58.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:58.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:58.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:58.165 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:58.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:58.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:58.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:58.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:58.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:58.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:36:58.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:58.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:58.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:58.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:58.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:58.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:58 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3267697284' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:58 smithi152 ceph-mon[93456]: osdmap e360: 8 total, 8 up, 6 in 2024-08-17T17:36:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:36:58 smithi152 ceph-mon[93456]: pgmap v1292: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 56394/287859 objects degraded (19.591%); 37617/287859 objects misplaced (13.068%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:36:58.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:58.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:58.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3267697284' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[106729]: osdmap e360: 8 total, 8 up, 6 in 2024-08-17T17:36:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[106729]: pgmap v1292: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 56394/287859 objects degraded (19.591%); 37617/287859 objects misplaced (13.068%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:36:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3267697284' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[103614]: osdmap e360: 8 total, 8 up, 6 in 2024-08-17T17:36:58.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:36:58 smithi142 ceph-mon[103614]: pgmap v1292: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 56394/287859 objects degraded (19.591%); 37617/287859 objects misplaced (13.068%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:36:58.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:58.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:59.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:59.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:59.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:59.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:59.314 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:59.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:59.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:59.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:59.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:59.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:59.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:59.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:59.719 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:59.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:59.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:36:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:59.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.975 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:36:59.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:00.038 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:37:00.038 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:00.050 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:37:00.051 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:00.068 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:37:00.068 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:00.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:00.084 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:37:00.084 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:00.084 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3930369342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:00.085 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3930369342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:00.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:00.100 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:37:00.100 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:00.115 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:37:00.115 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:00.132 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:37:00.132 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:00.150 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:37:00.150 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:00.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:00.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3930369342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:00.216 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:00.216 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:00.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:00.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:00.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:00.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:00.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:00.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:00.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:00.512 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:37:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:37:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:00.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:00.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:00.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:00.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:00.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:00.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:00.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:00.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:37:00.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:00.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:01.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:01.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:01.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:01 smithi142 ceph-mon[103614]: pgmap v1293: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 56394/287859 objects degraded (19.591%); 37617/287859 objects misplaced (13.068%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56394/287859 objects degraded (19.591%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:01 smithi142 ceph-mon[106729]: pgmap v1293: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 56394/287859 objects degraded (19.591%); 37617/287859 objects misplaced (13.068%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:01.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56394/287859 objects degraded (19.591%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:01.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:01.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:01.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:01.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:37:01.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:01 smithi152 ceph-mon[93456]: pgmap v1293: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 56394/287859 objects degraded (19.591%); 37617/287859 objects misplaced (13.068%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:01.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56394/287859 objects degraded (19.591%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:01.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:01.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:01.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:01.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:01.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:01.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:01.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:01.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:01.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:01.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:01.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:01.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:01.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:01.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:01.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:02.035 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:37:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:37:01] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-08-17T17:37:02.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:02.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:02.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:02.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:02.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:02.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:02.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:02.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:02.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:02.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:02.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:02.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:02.717 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:02.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:02.722 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:02.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:02.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.884 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:37:02.885 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:37:02.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:02.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:03.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:03.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:03.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:03.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:03.378 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:03 smithi142 ceph-mon[106729]: pgmap v1294: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56346/286845 objects degraded (19.643%); 37465/286845 objects misplaced (13.061%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:03.379 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:03 smithi142 ceph-mon[103614]: pgmap v1294: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56346/286845 objects degraded (19.643%); 37465/286845 objects misplaced (13.061%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:03.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:03.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:03.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:03.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:03 smithi152 ceph-mon[93456]: pgmap v1294: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 56346/286845 objects degraded (19.643%); 37465/286845 objects misplaced (13.061%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:03.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:03.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:03.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:03.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:03.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:03.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:04.046 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1294,"stamp":"2024-08-17T17:37:02.268573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6265051109,"num_objects":95615,"num_object_clones":0,"num_object_copies":286845,"num_objects_missing_on_primary":6588,"num_objects_missing":6588,"num_objects_degraded":56346,"num_objects_misplaced":37465,"num_objects_unfound":0,"num_objects_dirty":95615,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":141079,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24427,"num_bytes_recovered":1477976718,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105027,"ondisk_log_size":105027,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18144616,"kb_used_data":13018228,"kb_used_omap":27,"kb_used_meta":5126308,"kb_avail":544326296,"statfs":{"total":575970213888,"available":557390127104,"internally_reserved":0,"allocated":13330665472,"data_stored":13319312124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5249340136},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2,1,3,4,5],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-53149696,"num_objects":-811,"num_object_clones":0,"num_object_copies":-2433,"num_objects_missing_on_primary":-53,"num_objects_missing":-53,"num_objects_degraded":-94,"num_objects_misplaced":-294,"num_objects_unfound":0,"num_objects_dirty":-811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":811,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":152,"num_bytes_recovered":9961472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002651"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":561,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681699+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:36:57.681699+0000","last_peered":"2024-08-17T17:36:57.681699+0000","last_clean":"2024-08-17T17:36:57.681699+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:36:57.681699+0000","last_undegraded":"2024-08-17T17:36:57.681699+0000","last_fullsized":"2024-08-17T17:36:57.681699+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":69,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494024+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:36:46.494024+0000","last_peered":"2024-08-17T17:36:46.494024+0000","last_clean":"2024-08-17T17:36:46.494024+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:36:46.494024+0000","last_undegraded":"2024-08-17T17:36:46.494024+0000","last_fullsized":"2024-08-17T17:36:46.494024+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":263,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.495090+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:36:46.495090+0000","last_peered":"2024-08-17T17:36:46.495090+0000","last_clean":"2024-08-17T17:36:46.495090+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:36:46.495090+0000","last_undegraded":"2024-08-17T17:36:46.495090+0000","last_fullsized":"2024-08-17T17:36:46.495090+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":259,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681796+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:36:57.681796+0000","last_peered":"2024-08-17T17:36:57.681796+0000","last_clean":"2024-08-17T17:36:57.681796+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:36:57.681796+0000","last_undegraded":"2024-08-17T17:36:57.681796+0000","last_fullsized":"2024-08-17T17:36:57.681796+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":436,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.682287+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:36:57.682287+0000","last_peered":"2024-08-17T17:36:57.682287+0000","last_clean":"2024-08-17T17:36:57.682287+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:36:57.682287+0000","last_undegraded":"2024-08-17T17:36:57.682287+0000","last_fullsized":"2024-08-17T17:36:57.682287+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":212,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.493748+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:36:46.493748+0000","last_peered":"2024-08-17T17:36:46.493748+0000","last_clean":"2024-08-17T17:36:46.493748+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:36:46.493748+0000","last_undegraded":"2024-08-17T17:36:46.493748+0000","last_fullsized":"2024-08-17T17:36:46.493748+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":530,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681722+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:36:57.681722+0000","last_peered":"2024-08-17T17:36:57.681722+0000","last_clean":"2024-08-17T17:36:57.681722+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:36:57.681722+0000","last_undegraded":"2024-08-17T17:36:57.681722+0000","last_fullsized":"2024-08-17T17:36:57.681722+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":380,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.591653+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:36:46.591653+0000","last_peered":"2024-08-17T17:36:46.591653+0000","last_clean":"2024-08-17T17:36:46.591653+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:36:46.591653+0000","last_undegraded":"2024-08-17T17:36:46.591653+0000","last_fullsized":"2024-08-17T17:36:46.591653+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"360'8768","reported_seq":10214,"reported_epoch":360,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:37:00.998851+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:37:00.998851+0000","last_peered":"2024-08-17T17:37:00.998851+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:37:00.998851+0000","last_undegraded":"2024-08-17T17:37:00.998851+0000","last_fullsized":"2024-08-17T17:37:00.998851+0000","mapping_epoch":337,"log_start":"324'1863","ondisk_log_start":"324'1863","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389021696,"num_objects":5936,"num_object_clones":0,"num_object_copies":17808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5936,"num_objects_unfound":0,"num_objects_dirty":5936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8768,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":687,"num_bytes_recovered":45023232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5936}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":65,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.547190+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:36:46.547190+0000","last_peered":"2024-08-17T17:36:46.547190+0000","last_clean":"2024-08-17T17:36:46.547190+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:36:46.547190+0000","last_undegraded":"2024-08-17T17:36:46.547190+0000","last_fullsized":"2024-08-17T17:36:46.547190+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494208+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:36:46.494208+0000","last_peered":"2024-08-17T17:36:46.494208+0000","last_clean":"2024-08-17T17:36:46.494208+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:36:46.494208+0000","last_undegraded":"2024-08-17T17:36:46.494208+0000","last_fullsized":"2024-08-17T17:36:46.494208+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"360'8994","reported_seq":9052,"reported_epoch":360,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:01.717732+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:37:01.717732+0000","last_peered":"2024-08-17T17:37:01.717732+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:37:01.717732+0000","last_undegraded":"2024-08-17T17:37:01.717732+0000","last_fullsized":"2024-08-17T17:37:01.717732+0000","mapping_epoch":337,"log_start":"324'2382","ondisk_log_start":"324'2382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398458880,"num_objects":6080,"num_object_clones":0,"num_object_copies":18240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6080,"num_objects_unfound":0,"num_objects_dirty":6080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8994,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":6080}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":67,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494829+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:36:46.494829+0000","last_peered":"2024-08-17T17:36:46.494829+0000","last_clean":"2024-08-17T17:36:46.494829+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:36:46.494829+0000","last_undegraded":"2024-08-17T17:36:46.494829+0000","last_fullsized":"2024-08-17T17:36:46.494829+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":331,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.493712+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:36:46.493712+0000","last_peered":"2024-08-17T17:36:46.493712+0000","last_clean":"2024-08-17T17:36:46.493712+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:36:46.493712+0000","last_undegraded":"2024-08-17T17:36:46.493712+0000","last_fullsized":"2024-08-17T17:36:46.493712+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"360'8743","reported_seq":8804,"reported_epoch":360,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:00.735897+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:37:00.735897+0000","last_peered":"2024-08-17T17:37:00.735897+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:37:00.735897+0000","last_undegraded":"2024-08-17T17:37:00.735897+0000","last_fullsized":"2024-08-17T17:37:00.735897+0000","mapping_epoch":337,"log_start":"324'1853","ondisk_log_start":"324'1853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392626176,"num_objects":5991,"num_object_clones":0,"num_object_copies":17973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5991,"num_objects_unfound":0,"num_objects_dirty":5991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8743,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5991}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":63,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.495020+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:36:46.495020+0000","last_peered":"2024-08-17T17:36:46.495020+0000","last_clean":"2024-08-17T17:36:46.495020+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:36:46.495020+0000","last_undegraded":"2024-08-17T17:36:46.495020+0000","last_fullsized":"2024-08-17T17:36:46.495020+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":341,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.590163+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:36:46.590163+0000","last_peered":"2024-08-17T17:36:46.590163+0000","last_clean":"2024-08-17T17:36:46.590163+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:36:46.590163+0000","last_undegraded":"2024-08-17T17:36:46.590163+0000","last_fullsized":"2024-08-17T17:36:46.590163+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"360'8887","reported_seq":8985,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:58.028112+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:36:58.028112+0000","last_peered":"2024-08-17T17:36:58.028112+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:36:58.028112+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'1813","ondisk_log_start":"324'1813","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397869056,"num_objects":6071,"num_object_clones":0,"num_object_copies":18213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8887,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-698},{"shards":"0,3,7","objects":6071}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":473,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681875+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:36:57.681875+0000","last_peered":"2024-08-17T17:36:57.681875+0000","last_clean":"2024-08-17T17:36:57.681875+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:36:57.681875+0000","last_undegraded":"2024-08-17T17:36:57.681875+0000","last_fullsized":"2024-08-17T17:36:57.681875+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":239,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.675533+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:36:57.675533+0000","last_peered":"2024-08-17T17:36:57.675533+0000","last_clean":"2024-08-17T17:36:57.675533+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:36:57.675533+0000","last_undegraded":"2024-08-17T17:36:57.675533+0000","last_fullsized":"2024-08-17T17:36:57.675533+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"360'8685","reported_seq":8734,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:37:01.731296+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:37:01.731296+0000","last_peered":"2024-08-17T17:37:01.731296+0000","last_clean":"2024-08-17T17:37:01.731296+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:37:01.731296+0000","last_undegraded":"2024-08-17T17:37:01.731296+0000","last_fullsized":"2024-08-17T17:37:01.731296+0000","mapping_epoch":319,"log_start":"324'2074","ondisk_log_start":"324'2074","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387252224,"num_objects":5909,"num_object_clones":0,"num_object_copies":17727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8685,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":562,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.683293+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:36:57.683293+0000","last_peered":"2024-08-17T17:36:57.683293+0000","last_clean":"2024-08-17T17:36:57.683293+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:36:57.683293+0000","last_undegraded":"2024-08-17T17:36:57.683293+0000","last_fullsized":"2024-08-17T17:36:57.683293+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":314,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494002+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:36:46.494002+0000","last_peered":"2024-08-17T17:36:46.494002+0000","last_clean":"2024-08-17T17:36:46.494002+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:36:46.494002+0000","last_undegraded":"2024-08-17T17:36:46.494002+0000","last_fullsized":"2024-08-17T17:36:46.494002+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"360'8862","reported_seq":8921,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:01.867046+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:37:01.867046+0000","last_peered":"2024-08-17T17:37:01.867046+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:37:01.867046+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'2335","ondisk_log_start":"324'2335","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397803520,"num_objects":6070,"num_object_clones":0,"num_object_copies":18210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8862,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":6070},{"shards":"1,6","objects":-1296}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":315,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.590293+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:36:46.590293+0000","last_peered":"2024-08-17T17:36:46.590293+0000","last_clean":"2024-08-17T17:36:46.590293+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:36:46.590293+0000","last_undegraded":"2024-08-17T17:36:46.590293+0000","last_fullsized":"2024-08-17T17:36:46.590293+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":341,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494453+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:36:46.494453+0000","last_peered":"2024-08-17T17:36:46.494453+0000","last_clean":"2024-08-17T17:36:46.494453+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:36:46.494453+0000","last_undegraded":"2024-08-17T17:36:46.494453+0000","last_fullsized":"2024-08-17T17:36:46.494453+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"360'8918","reported_seq":9017,"reported_epoch":360,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:01.671992+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:37:01.671992+0000","last_peered":"2024-08-17T17:37:01.671992+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:37:01.671992+0000","last_undegraded":"2024-08-17T17:37:01.671992+0000","last_fullsized":"2024-08-17T17:37:01.671992+0000","mapping_epoch":337,"log_start":"325'3361","ondisk_log_start":"325'3361","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5557,"log_dups_size":0,"ondisk_log_size":5557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399245312,"num_objects":6092,"num_object_clones":0,"num_object_copies":18276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6092,"num_objects_unfound":0,"num_objects_dirty":6092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8918,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":6092}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":352,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.493823+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:36:46.493823+0000","last_peered":"2024-08-17T17:36:46.493823+0000","last_clean":"2024-08-17T17:36:46.493823+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:36:46.493823+0000","last_undegraded":"2024-08-17T17:36:46.493823+0000","last_fullsized":"2024-08-17T17:36:46.493823+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":314,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494961+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:36:46.494961+0000","last_peered":"2024-08-17T17:36:46.494961+0000","last_clean":"2024-08-17T17:36:46.494961+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:36:46.494961+0000","last_undegraded":"2024-08-17T17:36:46.494961+0000","last_fullsized":"2024-08-17T17:36:46.494961+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"360'8808","reported_seq":10387,"reported_epoch":360,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:01.314273+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:37:01.314273+0000","last_peered":"2024-08-17T17:37:01.314273+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:37:01.314273+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:37:01.314273+0000","mapping_epoch":337,"log_start":"324'1941","ondisk_log_start":"324'1941","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386400256,"num_objects":5896,"num_object_clones":0,"num_object_copies":17688,"num_objects_missing_on_primary":6588,"num_objects_missing":6588,"num_objects_degraded":5896,"num_objects_misplaced":6588,"num_objects_unfound":0,"num_objects_dirty":5896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8808,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":750,"num_bytes_recovered":49086464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":14},{"shards":"1,2,7","objects":5882}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":532,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.692607+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:36:57.692607+0000","last_peered":"2024-08-17T17:36:57.692607+0000","last_clean":"2024-08-17T17:36:57.692607+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:36:57.692607+0000","last_undegraded":"2024-08-17T17:36:57.692607+0000","last_fullsized":"2024-08-17T17:36:57.692607+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":469,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681818+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:36:57.681818+0000","last_peered":"2024-08-17T17:36:57.681818+0000","last_clean":"2024-08-17T17:36:57.681818+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:36:57.681818+0000","last_undegraded":"2024-08-17T17:36:57.681818+0000","last_fullsized":"2024-08-17T17:36:57.681818+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"360'8868","reported_seq":10296,"reported_epoch":360,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:01.741542+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:37:01.741542+0000","last_peered":"2024-08-17T17:37:01.741542+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:37:01.741542+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:37:01.741542+0000","mapping_epoch":337,"log_start":"325'3244","ondisk_log_start":"325'3244","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5624,"log_dups_size":0,"ondisk_log_size":5624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394985496,"num_objects":6028,"num_object_clones":0,"num_object_copies":18084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6028,"num_objects_misplaced":6778,"num_objects_unfound":0,"num_objects_dirty":6028,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8868,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":670,"num_bytes_recovered":43843584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":6028}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":662,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:57.674703+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:36:57.674703+0000","last_peered":"2024-08-17T17:36:57.674703+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:36:57.674703+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":289,"reported_epoch":359,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:36:46.494745+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:36:46.494745+0000","last_peered":"2024-08-17T17:36:46.494745+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:36:46.494745+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:36:46.494745+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":511,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681707+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:36:57.681707+0000","last_peered":"2024-08-17T17:36:57.681707+0000","last_clean":"2024-08-17T17:36:57.681707+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:36:57.681707+0000","last_undegraded":"2024-08-17T17:36:57.681707+0000","last_fullsized":"2024-08-17T17:36:57.681707+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":324,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.591913+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:36:46.591913+0000","last_peered":"2024-08-17T17:36:46.591913+0000","last_clean":"2024-08-17T17:36:46.591913+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:36:46.591913+0000","last_undegraded":"2024-08-17T17:36:46.591913+0000","last_fullsized":"2024-08-17T17:36:46.591913+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"360'8785","reported_seq":8886,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:57.971887+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:36:57.971887+0000","last_peered":"2024-08-17T17:36:57.971887+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:36:57.971887+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1721","ondisk_log_start":"324'1721","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390529024,"num_objects":5959,"num_object_clones":0,"num_object_copies":17877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8785,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-688},{"shards":"0,4,7","objects":5959}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":327,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.670457+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:36:57.670457+0000","last_peered":"2024-08-17T17:36:57.670457+0000","last_clean":"2024-08-17T17:36:57.670457+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:36:57.670457+0000","last_undegraded":"2024-08-17T17:36:57.670457+0000","last_fullsized":"2024-08-17T17:36:57.670457+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":296,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.493182+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:36:46.493182+0000","last_peered":"2024-08-17T17:36:46.493182+0000","last_clean":"2024-08-17T17:36:46.493182+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:36:46.493182+0000","last_undegraded":"2024-08-17T17:36:46.493182+0000","last_fullsized":"2024-08-17T17:36:46.493182+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":253,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.679975+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:36:57.679975+0000","last_peered":"2024-08-17T17:36:57.679975+0000","last_clean":"2024-08-17T17:36:57.679975+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:36:57.679975+0000","last_undegraded":"2024-08-17T17:36:57.679975+0000","last_fullsized":"2024-08-17T17:36:57.679975+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"360'8815","reported_seq":8874,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:01.746150+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:37:01.746150+0000","last_peered":"2024-08-17T17:37:01.746150+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:37:01.746150+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'2217","ondisk_log_start":"324'2217","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391970816,"num_objects":5981,"num_object_clones":0,"num_object_copies":17943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8815,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5981},{"shards":"5,6","objects":-1315}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":419,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.682328+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:36:57.682328+0000","last_peered":"2024-08-17T17:36:57.682328+0000","last_clean":"2024-08-17T17:36:57.682328+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:36:57.682328+0000","last_undegraded":"2024-08-17T17:36:57.682328+0000","last_fullsized":"2024-08-17T17:36:57.682328+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":96,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.589780+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:36:46.589780+0000","last_peered":"2024-08-17T17:36:46.589780+0000","last_clean":"2024-08-17T17:36:46.589780+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:36:46.589780+0000","last_undegraded":"2024-08-17T17:36:46.589780+0000","last_fullsized":"2024-08-17T17:36:46.589780+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":604,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671138+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:36:57.671138+0000","last_peered":"2024-08-17T17:36:57.671138+0000","last_clean":"2024-08-17T17:36:57.671138+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:36:57.671138+0000","last_undegraded":"2024-08-17T17:36:57.671138+0000","last_fullsized":"2024-08-17T17:36:57.671138+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"360'8691","reported_seq":9516,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:36:58.040434+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:36:58.040434+0000","last_peered":"2024-08-17T17:36:58.040434+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:36:58.040434+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1616","ondisk_log_start":"324'1616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390266880,"num_objects":5955,"num_object_clones":0,"num_object_copies":17865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8691,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5955},{"shards":"0,5","objects":-699}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":395,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.670920+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:36:57.670920+0000","last_peered":"2024-08-17T17:36:57.670920+0000","last_clean":"2024-08-17T17:36:57.670920+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:36:57.670920+0000","last_undegraded":"2024-08-17T17:36:57.670920+0000","last_fullsized":"2024-08-17T17:36:57.670920+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":335,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494649+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:36:46.494649+0000","last_peered":"2024-08-17T17:36:46.494649+0000","last_clean":"2024-08-17T17:36:46.494649+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:36:46.494649+0000","last_undegraded":"2024-08-17T17:36:46.494649+0000","last_fullsized":"2024-08-17T17:36:46.494649+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":190,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681620+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:36:57.681620+0000","last_peered":"2024-08-17T17:36:57.681620+0000","last_clean":"2024-08-17T17:36:57.681620+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:36:57.681620+0000","last_undegraded":"2024-08-17T17:36:57.681620+0000","last_fullsized":"2024-08-17T17:36:57.681620+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"360'8831","reported_seq":23775,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.837616+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:36:59.837616+0000","last_peered":"2024-08-17T17:36:59.837616+0000","last_clean":"2024-08-17T17:36:59.837616+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:36:59.837616+0000","last_undegraded":"2024-08-17T17:36:59.837616+0000","last_fullsized":"2024-08-17T17:36:59.837616+0000","mapping_epoch":355,"log_start":"324'2530","ondisk_log_start":"324'2530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393543680,"num_objects":6005,"num_object_clones":0,"num_object_copies":18015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8831,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":274,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494924+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:36:46.494924+0000","last_peered":"2024-08-17T17:36:46.494924+0000","last_clean":"2024-08-17T17:36:46.494924+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:36:46.494924+0000","last_undegraded":"2024-08-17T17:36:46.494924+0000","last_fullsized":"2024-08-17T17:36:46.494924+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":59,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681531+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:36:57.681531+0000","last_peered":"2024-08-17T17:36:57.681531+0000","last_clean":"2024-08-17T17:36:57.681531+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:36:57.681531+0000","last_undegraded":"2024-08-17T17:36:57.681531+0000","last_fullsized":"2024-08-17T17:36:57.681531+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":555,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671277+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:36:57.671277+0000","last_peered":"2024-08-17T17:36:57.671277+0000","last_clean":"2024-08-17T17:36:57.671277+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:36:57.671277+0000","last_undegraded":"2024-08-17T17:36:57.671277+0000","last_fullsized":"2024-08-17T17:36:57.671277+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"360'8590","reported_seq":23102,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:59.895010+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:36:59.895010+0000","last_peered":"2024-08-17T17:36:59.895010+0000","last_clean":"2024-08-17T17:36:59.895010+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:36:59.895010+0000","last_undegraded":"2024-08-17T17:36:59.895010+0000","last_fullsized":"2024-08-17T17:36:59.895010+0000","mapping_epoch":345,"log_start":"324'2272","ondisk_log_start":"324'2272","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6318,"log_dups_size":0,"ondisk_log_size":6318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381026304,"num_objects":5814,"num_object_clones":0,"num_object_copies":17442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8590,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":270,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.495301+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:36:46.495301+0000","last_peered":"2024-08-17T17:36:46.495301+0000","last_clean":"2024-08-17T17:36:46.495301+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:36:46.495301+0000","last_undegraded":"2024-08-17T17:36:46.495301+0000","last_fullsized":"2024-08-17T17:36:46.495301+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":310,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.675069+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:36:57.675069+0000","last_peered":"2024-08-17T17:36:57.675069+0000","last_clean":"2024-08-17T17:36:57.675069+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:36:57.675069+0000","last_undegraded":"2024-08-17T17:36:57.675069+0000","last_fullsized":"2024-08-17T17:36:57.675069+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":412,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494654+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:36:46.494654+0000","last_peered":"2024-08-17T17:36:46.494654+0000","last_clean":"2024-08-17T17:36:46.494654+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:36:46.494654+0000","last_undegraded":"2024-08-17T17:36:46.494654+0000","last_fullsized":"2024-08-17T17:36:46.494654+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"360'8844","reported_seq":8964,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:00.253009+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:37:00.253009+0000","last_peered":"2024-08-17T17:37:00.253009+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:37:00.253009+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2587","ondisk_log_start":"324'2587","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6257,"log_dups_size":0,"ondisk_log_size":6257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389283840,"num_objects":5940,"num_object_clones":0,"num_object_copies":17820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8844,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5940},{"shards":"0,3","objects":-1308}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":585,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681913+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:36:57.681913+0000","last_peered":"2024-08-17T17:36:57.681913+0000","last_clean":"2024-08-17T17:36:57.681913+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:36:57.681913+0000","last_undegraded":"2024-08-17T17:36:57.681913+0000","last_fullsized":"2024-08-17T17:36:57.681913+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":553,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671466+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:36:57.671466+0000","last_peered":"2024-08-17T17:36:57.671466+0000","last_clean":"2024-08-17T17:36:57.671466+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:36:57.671466+0000","last_undegraded":"2024-08-17T17:36:57.671466+0000","last_fullsized":"2024-08-17T17:36:57.671466+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":571,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.692542+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:36:57.692542+0000","last_peered":"2024-08-17T17:36:57.692542+0000","last_clean":"2024-08-17T17:36:57.692542+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:36:57.692542+0000","last_undegraded":"2024-08-17T17:36:57.692542+0000","last_fullsized":"2024-08-17T17:36:57.692542+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"360'8712","reported_seq":23369,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:37:01.434874+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:37:01.434874+0000","last_peered":"2024-08-17T17:37:01.434874+0000","last_clean":"2024-08-17T17:37:01.434874+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:37:01.434874+0000","last_undegraded":"2024-08-17T17:37:01.434874+0000","last_fullsized":"2024-08-17T17:37:01.434874+0000","mapping_epoch":347,"log_start":"324'2172","ondisk_log_start":"324'2172","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384172032,"num_objects":5862,"num_object_clones":0,"num_object_copies":17586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8712,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":243,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.675281+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:36:57.675281+0000","last_peered":"2024-08-17T17:36:57.675281+0000","last_clean":"2024-08-17T17:36:57.675281+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:36:57.675281+0000","last_undegraded":"2024-08-17T17:36:57.675281+0000","last_fullsized":"2024-08-17T17:36:57.675281+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":364,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.590761+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:36:46.590761+0000","last_peered":"2024-08-17T17:36:46.590761+0000","last_clean":"2024-08-17T17:36:46.590761+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:36:46.590761+0000","last_undegraded":"2024-08-17T17:36:46.590761+0000","last_fullsized":"2024-08-17T17:36:46.590761+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":314,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.493789+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:36:46.493789+0000","last_peered":"2024-08-17T17:36:46.493789+0000","last_clean":"2024-08-17T17:36:46.493789+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:36:46.493789+0000","last_undegraded":"2024-08-17T17:36:46.493789+0000","last_fullsized":"2024-08-17T17:36:46.493789+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":314,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494519+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:36:46.494519+0000","last_peered":"2024-08-17T17:36:46.494519+0000","last_clean":"2024-08-17T17:36:46.494519+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:36:46.494519+0000","last_undegraded":"2024-08-17T17:36:46.494519+0000","last_fullsized":"2024-08-17T17:36:46.494519+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":357,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494181+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:36:46.494181+0000","last_peered":"2024-08-17T17:36:46.494181+0000","last_clean":"2024-08-17T17:36:46.494181+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:36:46.494181+0000","last_undegraded":"2024-08-17T17:36:46.494181+0000","last_fullsized":"2024-08-17T17:36:46.494181+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":560,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671010+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:36:57.671010+0000","last_peered":"2024-08-17T17:36:57.671010+0000","last_clean":"2024-08-17T17:36:57.671010+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:36:57.671010+0000","last_undegraded":"2024-08-17T17:36:57.671010+0000","last_fullsized":"2024-08-17T17:36:57.671010+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":100,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.683512+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:36:57.683512+0000","last_peered":"2024-08-17T17:36:57.683512+0000","last_clean":"2024-08-17T17:36:57.683512+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:36:57.683512+0000","last_undegraded":"2024-08-17T17:36:57.683512+0000","last_fullsized":"2024-08-17T17:36:57.683512+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":264,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.495269+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:36:46.495269+0000","last_peered":"2024-08-17T17:36:46.495269+0000","last_clean":"2024-08-17T17:36:46.495269+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:36:46.495269+0000","last_undegraded":"2024-08-17T17:36:46.495269+0000","last_fullsized":"2024-08-17T17:36:46.495269+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":518,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.682013+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:36:57.682013+0000","last_peered":"2024-08-17T17:36:57.682013+0000","last_clean":"2024-08-17T17:36:57.682013+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:36:57.682013+0000","last_undegraded":"2024-08-17T17:36:57.682013+0000","last_fullsized":"2024-08-17T17:36:57.682013+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":408,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.495720+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:36:46.495720+0000","last_peered":"2024-08-17T17:36:46.495720+0000","last_clean":"2024-08-17T17:36:46.495720+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:36:46.495720+0000","last_undegraded":"2024-08-17T17:36:46.495720+0000","last_fullsized":"2024-08-17T17:36:46.495720+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":512,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681423+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:36:57.681423+0000","last_peered":"2024-08-17T17:36:57.681423+0000","last_clean":"2024-08-17T17:36:57.681423+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:36:57.681423+0000","last_undegraded":"2024-08-17T17:36:57.681423+0000","last_fullsized":"2024-08-17T17:36:57.681423+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":331,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.590691+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:36:46.590691+0000","last_peered":"2024-08-17T17:36:46.590691+0000","last_clean":"2024-08-17T17:36:46.590691+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:36:46.590691+0000","last_undegraded":"2024-08-17T17:36:46.590691+0000","last_fullsized":"2024-08-17T17:36:46.590691+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":544,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.692577+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:36:57.692577+0000","last_peered":"2024-08-17T17:36:57.692577+0000","last_clean":"2024-08-17T17:36:57.692577+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:36:57.692577+0000","last_undegraded":"2024-08-17T17:36:57.692577+0000","last_fullsized":"2024-08-17T17:36:57.692577+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":259,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.682207+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:36:57.682207+0000","last_peered":"2024-08-17T17:36:57.682207+0000","last_clean":"2024-08-17T17:36:57.682207+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:36:57.682207+0000","last_undegraded":"2024-08-17T17:36:57.682207+0000","last_fullsized":"2024-08-17T17:36:57.682207+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":552,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671794+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:36:57.671794+0000","last_peered":"2024-08-17T17:36:57.671794+0000","last_clean":"2024-08-17T17:36:57.671794+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:36:57.671794+0000","last_undegraded":"2024-08-17T17:36:57.671794+0000","last_fullsized":"2024-08-17T17:36:57.671794+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":511,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.683323+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:36:57.683323+0000","last_peered":"2024-08-17T17:36:57.683323+0000","last_clean":"2024-08-17T17:36:57.683323+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:36:57.683323+0000","last_undegraded":"2024-08-17T17:36:57.683323+0000","last_fullsized":"2024-08-17T17:36:57.683323+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":343,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494246+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:36:46.494246+0000","last_peered":"2024-08-17T17:36:46.494246+0000","last_clean":"2024-08-17T17:36:46.494246+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:36:46.494246+0000","last_undegraded":"2024-08-17T17:36:46.494246+0000","last_fullsized":"2024-08-17T17:36:46.494246+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":200,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.681791+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:36:57.681791+0000","last_peered":"2024-08-17T17:36:57.681791+0000","last_clean":"2024-08-17T17:36:57.681791+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:36:57.681791+0000","last_undegraded":"2024-08-17T17:36:57.681791+0000","last_fullsized":"2024-08-17T17:36:57.681791+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":256,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.675065+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:36:57.675065+0000","last_peered":"2024-08-17T17:36:57.675065+0000","last_clean":"2024-08-17T17:36:57.675065+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:36:57.675065+0000","last_undegraded":"2024-08-17T17:36:57.675065+0000","last_fullsized":"2024-08-17T17:36:57.675065+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":351,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494218+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:36:46.494218+0000","last_peered":"2024-08-17T17:36:46.494218+0000","last_clean":"2024-08-17T17:36:46.494218+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:36:46.494218+0000","last_undegraded":"2024-08-17T17:36:46.494218+0000","last_fullsized":"2024-08-17T17:36:46.494218+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":214,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494097+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:36:46.494097+0000","last_peered":"2024-08-17T17:36:46.494097+0000","last_clean":"2024-08-17T17:36:46.494097+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:36:46.494097+0000","last_undegraded":"2024-08-17T17:36:46.494097+0000","last_fullsized":"2024-08-17T17:36:46.494097+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":351,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494119+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:36:46.494119+0000","last_peered":"2024-08-17T17:36:46.494119+0000","last_clean":"2024-08-17T17:36:46.494119+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:36:46.494119+0000","last_undegraded":"2024-08-17T17:36:46.494119+0000","last_fullsized":"2024-08-17T17:36:46.494119+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":214,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.494387+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:36:46.494387+0000","last_peered":"2024-08-17T17:36:46.494387+0000","last_clean":"2024-08-17T17:36:46.494387+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:36:46.494387+0000","last_undegraded":"2024-08-17T17:36:46.494387+0000","last_fullsized":"2024-08-17T17:36:46.494387+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":564,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.683244+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:36:57.683244+0000","last_peered":"2024-08-17T17:36:57.683244+0000","last_clean":"2024-08-17T17:36:57.683244+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:36:57.683244+0000","last_undegraded":"2024-08-17T17:36:57.683244+0000","last_fullsized":"2024-08-17T17:36:57.683244+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":537,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.682369+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:36:57.682369+0000","last_peered":"2024-08-17T17:36:57.682369+0000","last_clean":"2024-08-17T17:36:57.682369+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:36:57.682369+0000","last_undegraded":"2024-08-17T17:36:57.682369+0000","last_fullsized":"2024-08-17T17:36:57.682369+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":314,"reported_epoch":359,"state":"active+clean","last_fresh":"2024-08-17T17:36:46.493317+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:36:46.493317+0000","last_peered":"2024-08-17T17:36:46.493317+0000","last_clean":"2024-08-17T17:36:46.493317+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:36:46.493317+0000","last_undegraded":"2024-08-17T17:36:46.493317+0000","last_fullsized":"2024-08-17T17:36:46.493317+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":258,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.675411+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:36:57.675411+0000","last_peered":"2024-08-17T17:36:57.675411+0000","last_clean":"2024-08-17T17:36:57.675411+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:36:57.675411+0000","last_undegraded":"2024-08-17T17:36:57.675411+0000","last_fullsized":"2024-08-17T17:36:57.675411+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6264455192,"num_objects":95589,"num_object_clones":0,"num_object_copies":286767,"num_objects_missing_on_primary":6588,"num_objects_missing":6588,"num_objects_degraded":56343,"num_objects_misplaced":37465,"num_objects_unfound":0,"num_objects_dirty":95589,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140801,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24386,"num_bytes_recovered":1475608576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20359950336,"data_stored":20359938120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104820,"ondisk_log_size":104820,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170047,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463417,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3561368,"kb_used_data":2671824,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90183784,"statfs":{"total":95995035648,"available":92348194816,"internally_reserved":0,"allocated":2735947776,"data_stored":2734042057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492091,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1159292,"kb_used_data":768116,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92585860,"statfs":{"total":95995035648,"available":94807920640,"internally_reserved":0,"allocated":786550784,"data_stored":784667365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986683,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2410252,"kb_used_data":1521796,"kb_used_omap":1,"kb_used_meta":888446,"kb_avail":91334900,"statfs":{"total":95995035648,"available":93526937600,"internally_reserved":0,"allocated":1558319104,"data_stored":1556424325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909769048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2,1,3,4,5],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225510,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3976576,"kb_used_data":3056692,"kb_used_omap":4,"kb_used_meta":919867,"kb_avail":89768576,"statfs":{"total":95995035648,"available":91923021824,"internally_reserved":0,"allocated":3130052608,"data_stored":3128169487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":941944256},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075835,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533728,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3726592,"kb_used_data":2684600,"kb_used_omap":5,"kb_used_meta":1041978,"kb_avail":90018560,"statfs":{"total":95995035648,"available":92179005440,"internally_reserved":0,"allocated":2749030400,"data_stored":2747141957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1066985712},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089704,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3310536,"kb_used_data":2315200,"kb_used_omap":5,"kb_used_meta":995322,"kb_avail":90434616,"statfs":{"total":95995035648,"available":92605046784,"internally_reserved":0,"allocated":2370764800,"data_stored":2368866933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1019209787},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2362966016,"data_stored":2362966016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2742751232,"data_stored":2742747160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4212920320,"data_stored":4212916248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3124756480,"data_stored":3124756480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1553006592,"data_stored":1553006592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":781778944,"data_stored":781778944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2729115648,"data_stored":2729115648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2852655104,"data_stored":2852651032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:37:04.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:37:04.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:04.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:04.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:04.214 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:04.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.328 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:04 smithi142 ceph-mon[103614]: from='client.47051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:04.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:04.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.462 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:04.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:04.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:04.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:04.594 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:04 smithi142 ceph-mon[106729]: from='client.47051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:04.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:04.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:04.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:04 smithi152 ceph-mon[93456]: from='client.47051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:04.836 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:04.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:05.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:05.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:05.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:05.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:05.311 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:05.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:05.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:05.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:05.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:05.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:05 smithi142 ceph-mon[106729]: pgmap v1295: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 56346/286599 objects degraded (19.660%); 37465/286599 objects misplaced (13.072%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:05 smithi142 ceph-mon[103614]: pgmap v1295: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 56346/286599 objects degraded (19.660%); 37465/286599 objects misplaced (13.072%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:05.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:05.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:05.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:05 smithi152 ceph-mon[93456]: pgmap v1295: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 56346/286599 objects degraded (19.660%); 37465/286599 objects misplaced (13.072%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:05.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:05.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:05.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:06.015 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:06.017 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:06.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:06.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:06.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:06.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:06.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:06.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:06.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:06.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:06.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:06.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:06.611 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56346/286599 objects degraded (19.660%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:06.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56346/286599 objects degraded (19.660%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:06.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:06.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:06.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:06.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56346/286599 objects degraded (19.660%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:06.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:06.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:06.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:07.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:07.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:07.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:07.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:07.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:07.276 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:07.276 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:07.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:07.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:07.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:07.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:07.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:07.613 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:07 smithi142 ceph-mon[106729]: pgmap v1296: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 56346/286161 objects degraded (19.690%); 37465/286161 objects misplaced (13.092%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:07.613 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:07 smithi142 ceph-mon[103614]: pgmap v1296: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 56346/286161 objects degraded (19.690%); 37465/286161 objects misplaced (13.092%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:07.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:07.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:07.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:07.750 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:07 smithi152 ceph-mon[93456]: pgmap v1296: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 56346/286161 objects degraded (19.690%); 37465/286161 objects misplaced (13.092%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:37:07.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:07.874 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:07.976 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:07.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:08.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:08.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:08.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:08.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:08.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:08.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:08.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:08.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:08.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:08.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:08.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:08.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:08.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:08.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:08.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:08.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:08.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:09.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:37:09.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:09.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:09.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:09.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:09.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:09.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:09.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:09.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:09.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:09.578 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:09 smithi142 ceph-mon[106729]: pgmap v1297: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 56283/285186 objects degraded (19.736%); 37247/285186 objects misplaced (13.061%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:37:09.578 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:09 smithi142 ceph-mon[103614]: pgmap v1297: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 56283/285186 objects degraded (19.736%); 37247/285186 objects misplaced (13.061%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:37:09.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:09.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:09.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:09.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:09.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:09 smithi152 ceph-mon[93456]: pgmap v1297: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 56283/285186 objects degraded (19.736%); 37247/285186 objects misplaced (13.061%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:37:09.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:09.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:09.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:09.970 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:09.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:09.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:10.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:10.117 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:10.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:10.241 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:10.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:10.365 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:10.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:10.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:10.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:10 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:37:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:37:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:10.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:10.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:10.614 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:10 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:10.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:10 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:10.640 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:37:10.640 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:37:10.640 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:10.640 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:10.648 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:37:10.648 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:10.674 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:37:10.674 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:10.690 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:37:10.690 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:10.707 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:37:10.707 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:10.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:10.722 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:37:10.723 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:10.739 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:37:10.739 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:10.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:10.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:10.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:10.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:10.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:10.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:11.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:11.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:11.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:11.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:11.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:11.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:11.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:11.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:11.501 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-mon[106729]: pgmap v1298: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 56283/284871 objects degraded (19.757%); 37247/284871 objects misplaced (13.075%); 933 KiB/s, 14 objects/s recovering 2024-08-17T17:37:11.501 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1237489079' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:11.501 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56283/284871 objects degraded (19.757%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:11.501 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-mon[103614]: pgmap v1298: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 56283/284871 objects degraded (19.757%); 37247/284871 objects misplaced (13.075%); 933 KiB/s, 14 objects/s recovering 2024-08-17T17:37:11.501 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1237489079' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:11.501 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56283/284871 objects degraded (19.757%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:11.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:11.609 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:11 smithi152 ceph-mon[93456]: pgmap v1298: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 56283/284871 objects degraded (19.757%); 37247/284871 objects misplaced (13.075%); 933 KiB/s, 14 objects/s recovering 2024-08-17T17:37:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1237489079' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56283/284871 objects degraded (19.757%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:11.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:11.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:11.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:11.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:11.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:11.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:11.995 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:37:11.995 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 56283/284871 objects degraded (19.757%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1912,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":360,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":94957,"data_bytes":6221928421,"bytes_used":18488545280,"bytes_avail":557481668608,"bytes_total":575970213888,"degraded_objects":56283,"degraded_total":284871,"degraded_ratio":0.1975736385943111,"misplaced_objects":37247,"misplaced_total":284871,"misplaced_ratio":0.13075040983462691,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":955485,"recovering_keys_per_sec":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":82},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":31,"modified":"2024-08-17T17:36:14.254703+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:37:12.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:12.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:37:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:37:11] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:37:12.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:12.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:12.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:12.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:12.443 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:37:12.459 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:12.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:12.582 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:12.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3683823019' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/696239575' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3683823019' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/696239575' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:12.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:12.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:12.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:12.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:12.800 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:37:12.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:37:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3683823019' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:37:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/696239575' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:12.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:12.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:12.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:12.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:13.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:13.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:13.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:13.235 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:13.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:13.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:13.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:13.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:13.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:13.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:37:13.576 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:13.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:13.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[106729]: pgmap v1299: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 56230/283554 objects degraded (19.830%); 37051/283554 objects misplaced (13.067%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[106729]: osdmap e361: 8 total, 8 up, 6 in 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[103614]: pgmap v1299: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 56230/283554 objects degraded (19.830%); 37051/283554 objects misplaced (13.067%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:13.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:13 smithi142 ceph-mon[103614]: osdmap e361: 8 total, 8 up, 6 in 2024-08-17T17:37:13.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:13.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:13.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:13.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:13 smithi152 ceph-mon[93456]: pgmap v1299: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 56230/283554 objects degraded (19.830%); 37051/283554 objects misplaced (13.067%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:37:13.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:13 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:13.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:13 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:13.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:13 smithi152 ceph-mon[93456]: osdmap e361: 8 total, 8 up, 6 in 2024-08-17T17:37:13.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:13.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:13.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:13.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:13.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:14.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:14.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:14.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:14.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:14.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:14.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:14.453 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:14.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:14.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:14.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:14.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:14.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:14.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:14.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:14.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:14.859 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:14.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:14.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:14.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:15.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:15.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:15.209 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:15.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:15.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:15.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:15.385 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:37:15.385 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:37:15.385 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:37:15.385 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:37:15.385 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:37:15.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:15.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:15.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:37:15.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:15.603 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:15.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:15.611 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:15.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:15.708 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:15 smithi142 ceph-mon[106729]: pgmap v1301: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 56230/283554 objects degraded (19.830%); 37051/283554 objects misplaced (13.067%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:15.709 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:15.709 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:15 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:15.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:15 smithi142 ceph-mon[103614]: pgmap v1301: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 56230/283554 objects degraded (19.830%); 37051/283554 objects misplaced (13.067%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:15.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:15.709 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:15 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:15.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:15 smithi152 ceph-mon[93456]: pgmap v1301: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 56230/283554 objects degraded (19.830%); 37051/283554 objects misplaced (13.067%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:15 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:15.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:15.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:15.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:15.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:15.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:16.079 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:16.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:16.203 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:16.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:16.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:16.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:16.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:16.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:16.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:16.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:16.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56230/283554 objects degraded (19.830%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56230/283554 objects degraded (19.830%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:16.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:16.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:16.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56230/283554 objects degraded (19.830%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:16.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:16.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:16.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:16.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:17.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:17.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:17.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:17.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:17.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:17.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:17.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:17.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:17.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:17 smithi142 ceph-mon[106729]: pgmap v1302: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 56230/283158 objects degraded (19.858%); 37051/283158 objects misplaced (13.085%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:17.538 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:17 smithi142 ceph-mon[103614]: pgmap v1302: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 56230/283158 objects degraded (19.858%); 37051/283158 objects misplaced (13.085%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:17.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:17.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:17.696 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:17.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:17.820 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:17 smithi152 ceph-mon[93456]: pgmap v1302: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 56230/283158 objects degraded (19.858%); 37051/283158 objects misplaced (13.085%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:17.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:17.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:17.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:17.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:18.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:18.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:18.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:18.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:18.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:18.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:18.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:18.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:18.462 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:18.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:18.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:18.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:18.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:18.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:18.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:37:18.812 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:18.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:18.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:18.980 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:19.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:19.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:19.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:19.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:19.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:19.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:19.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:19.476 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:19.479 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:19 smithi142 ceph-mon[106729]: pgmap v1303: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 56176/282378 objects degraded (19.894%); 36886/282378 objects misplaced (13.063%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:19.479 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:19 smithi142 ceph-mon[103614]: pgmap v1303: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 56176/282378 objects degraded (19.894%); 36886/282378 objects misplaced (13.063%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:19.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:19.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:19.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:19.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:19.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:19.735 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:37:19.735 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:19.740 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:37:19.740 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:19.743 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:19.765 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:37:19.765 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:19.785 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:37:19.785 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:19.803 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:37:19.803 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:19.819 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:37:19.819 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:19 smithi152 ceph-mon[93456]: pgmap v1303: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 56176/282378 objects degraded (19.894%); 36886/282378 objects misplaced (13.063%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:19.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:19.837 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:37:19.837 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:19.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:19.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:19.856 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:37:19.856 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:19.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:19.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:20.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:20.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:20.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:20.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:20.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:20.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:20.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:20.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:20.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:20.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:20.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:20.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4283034063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:20.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:20 smithi152 ceph-mon[93456]: pgmap v1304: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 56176/282117 objects degraded (19.912%); 36886/282117 objects misplaced (13.075%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:37:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:37:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:20.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:20.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:20.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4283034063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:20.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:20 smithi142 ceph-mon[106729]: pgmap v1304: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 56176/282117 objects degraded (19.912%); 36886/282117 objects misplaced (13.075%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:20.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4283034063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:20.711 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:20 smithi142 ceph-mon[103614]: pgmap v1304: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 56176/282117 objects degraded (19.912%); 36886/282117 objects misplaced (13.075%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:20.715 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:20.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:20.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:20.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:20.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:20.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:20.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:21.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:21.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:21.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:21.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:21.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:21.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:21.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:21.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:21.468 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:21.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:21.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:21.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:21.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:21.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:21.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:21.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56176/282117 objects degraded (19.912%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:21.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56176/282117 objects degraded (19.912%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:21.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56176/282117 objects degraded (19.912%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:21.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:21.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:21.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:22.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:22.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:37:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:37:21] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:37:22.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:22.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:22.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:22.333 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:37:22.334 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:37:22.338 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:22.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:22.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:22.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:22.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:22.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:22.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:22.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:22.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:22.741 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:22 smithi142 ceph-mon[106729]: from='client.37218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:22.741 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:22 smithi142 ceph-mon[106729]: pgmap v1305: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 56121/280890 objects degraded (19.980%); 36699/280890 objects misplaced (13.065%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:22.742 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:22 smithi142 ceph-mon[103614]: from='client.37218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:22.742 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:22 smithi142 ceph-mon[103614]: pgmap v1305: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 56121/280890 objects degraded (19.980%); 36699/280890 objects misplaced (13.065%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:22 smithi152 ceph-mon[93456]: from='client.37218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:22 smithi152 ceph-mon[93456]: pgmap v1305: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 56121/280890 objects degraded (19.980%); 36699/280890 objects misplaced (13.065%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:22.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:22.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:22.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:22.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:23.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:23.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:23.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:23.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:23.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:23.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:23.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:23.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:23.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:23.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:23.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:23.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:23.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:23.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:24.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:24.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:24.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:24.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:24.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:24.294 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:37:24.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:24.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:24.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:24.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/207114728' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:24.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:24.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:24.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/207114728' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/207114728' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:24.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:24.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:24.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:24.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:24.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:24.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:24.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:25.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:25.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:25.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:25.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:25.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:25.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:25.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:25.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[103614]: pgmap v1306: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 56121/280890 objects degraded (19.980%); 36699/280890 objects misplaced (13.065%); 925 KiB/s, 14 objects/s recovering 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/207114728' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[103614]: osdmap e362: 8 total, 8 up, 6 in 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[106729]: pgmap v1306: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 56121/280890 objects degraded (19.980%); 36699/280890 objects misplaced (13.065%); 925 KiB/s, 14 objects/s recovering 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/207114728' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:25.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:25 smithi142 ceph-mon[106729]: osdmap e362: 8 total, 8 up, 6 in 2024-08-17T17:37:25.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:25 smithi152 ceph-mon[93456]: pgmap v1306: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 56121/280890 objects degraded (19.980%); 36699/280890 objects misplaced (13.065%); 925 KiB/s, 14 objects/s recovering 2024-08-17T17:37:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:25 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:25 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/207114728' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:25 smithi152 ceph-mon[93456]: osdmap e362: 8 total, 8 up, 6 in 2024-08-17T17:37:25.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:25.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:25.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:25.816 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:25.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:25.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:25.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:25.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:26.004 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1304,"stamp":"2024-08-17T17:37:20.273352+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6161766373,"num_objects":94039,"num_object_clones":0,"num_object_copies":282117,"num_objects_missing_on_primary":6511,"num_objects_missing":6511,"num_objects_degraded":56176,"num_objects_misplaced":36886,"num_objects_unfound":0,"num_objects_dirty":94039,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":142655,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24699,"num_bytes_recovered":1495802510,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104903,"ondisk_log_size":104903,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17949656,"kb_used_data":12802468,"kb_used_omap":27,"kb_used_meta":5147108,"kb_avail":544521256,"statfs":{"total":575970213888,"available":557589766144,"internally_reserved":0,"allocated":13109727232,"data_stored":13098319333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5270639336},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2,2,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-60162048,"num_objects":-918,"num_object_clones":0,"num_object_copies":-2754,"num_objects_missing_on_primary":-51,"num_objects_missing":-51,"num_objects_degraded":-107,"num_objects_misplaced":-361,"num_objects_unfound":0,"num_objects_dirty":-918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":918,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":176,"num_bytes_recovered":11534336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002650"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":563,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440656+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:37:12.440656+0000","last_peered":"2024-08-17T17:37:12.440656+0000","last_clean":"2024-08-17T17:37:12.440656+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:37:12.440656+0000","last_undegraded":"2024-08-17T17:37:12.440656+0000","last_fullsized":"2024-08-17T17:37:12.440656+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":70,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673522+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:36:57.673522+0000","last_peered":"2024-08-17T17:36:57.673522+0000","last_clean":"2024-08-17T17:36:57.673522+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:36:57.673522+0000","last_undegraded":"2024-08-17T17:36:57.673522+0000","last_fullsized":"2024-08-17T17:36:57.673522+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":264,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671946+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:36:57.671946+0000","last_peered":"2024-08-17T17:36:57.671946+0000","last_clean":"2024-08-17T17:36:57.671946+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:36:57.671946+0000","last_undegraded":"2024-08-17T17:36:57.671946+0000","last_fullsized":"2024-08-17T17:36:57.671946+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":261,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440437+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:37:12.440437+0000","last_peered":"2024-08-17T17:37:12.440437+0000","last_clean":"2024-08-17T17:37:12.440437+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:37:12.440437+0000","last_undegraded":"2024-08-17T17:37:12.440437+0000","last_fullsized":"2024-08-17T17:37:12.440437+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":438,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440809+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:37:12.440809+0000","last_peered":"2024-08-17T17:37:12.440809+0000","last_clean":"2024-08-17T17:37:12.440809+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:37:12.440809+0000","last_undegraded":"2024-08-17T17:37:12.440809+0000","last_fullsized":"2024-08-17T17:37:12.440809+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":213,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673230+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:36:57.673230+0000","last_peered":"2024-08-17T17:36:57.673230+0000","last_clean":"2024-08-17T17:36:57.673230+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:36:57.673230+0000","last_undegraded":"2024-08-17T17:36:57.673230+0000","last_fullsized":"2024-08-17T17:36:57.673230+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":532,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440040+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:37:12.440040+0000","last_peered":"2024-08-17T17:37:12.440040+0000","last_clean":"2024-08-17T17:37:12.440040+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:37:12.440040+0000","last_undegraded":"2024-08-17T17:37:12.440040+0000","last_fullsized":"2024-08-17T17:37:12.440040+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":381,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.677306+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:36:57.677306+0000","last_peered":"2024-08-17T17:36:57.677306+0000","last_clean":"2024-08-17T17:36:57.677306+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:36:57.677306+0000","last_undegraded":"2024-08-17T17:36:57.677306+0000","last_fullsized":"2024-08-17T17:36:57.677306+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"361'8858","reported_seq":10509,"reported_epoch":361,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:37:16.620150+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:37:16.620150+0000","last_peered":"2024-08-17T17:37:16.620150+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:37:16.620150+0000","last_undegraded":"2024-08-17T17:37:16.620150+0000","last_fullsized":"2024-08-17T17:37:16.620150+0000","mapping_epoch":337,"log_start":"324'1963","ondisk_log_start":"324'1963","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383123456,"num_objects":5846,"num_object_clones":0,"num_object_copies":17538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5834,"num_objects_unfound":0,"num_objects_dirty":5846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8858,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":789,"num_bytes_recovered":51707904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5834},{"shards":"4,6,7","objects":12}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":66,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.715232+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:36:57.715232+0000","last_peered":"2024-08-17T17:36:57.715232+0000","last_clean":"2024-08-17T17:36:57.715232+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:36:57.715232+0000","last_undegraded":"2024-08-17T17:36:57.715232+0000","last_fullsized":"2024-08-17T17:36:57.715232+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671657+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:36:57.671657+0000","last_peered":"2024-08-17T17:36:57.671657+0000","last_clean":"2024-08-17T17:36:57.671657+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:36:57.671657+0000","last_undegraded":"2024-08-17T17:36:57.671657+0000","last_fullsized":"2024-08-17T17:36:57.671657+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"361'9090","reported_seq":9149,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:16.888617+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:37:16.888617+0000","last_peered":"2024-08-17T17:37:16.888617+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:37:16.888617+0000","last_undegraded":"2024-08-17T17:37:16.888617+0000","last_fullsized":"2024-08-17T17:37:16.888617+0000","mapping_epoch":337,"log_start":"324'2482","ondisk_log_start":"324'2482","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392167424,"num_objects":5984,"num_object_clones":0,"num_object_copies":17952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5984,"num_objects_unfound":0,"num_objects_dirty":5984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9090,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5984}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":68,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673434+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:36:57.673434+0000","last_peered":"2024-08-17T17:36:57.673434+0000","last_clean":"2024-08-17T17:36:57.673434+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:36:57.673434+0000","last_undegraded":"2024-08-17T17:36:57.673434+0000","last_fullsized":"2024-08-17T17:36:57.673434+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":332,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673326+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:36:57.673326+0000","last_peered":"2024-08-17T17:36:57.673326+0000","last_clean":"2024-08-17T17:36:57.673326+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:36:57.673326+0000","last_undegraded":"2024-08-17T17:36:57.673326+0000","last_fullsized":"2024-08-17T17:36:57.673326+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"361'8845","reported_seq":8907,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:15.479481+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:37:15.479481+0000","last_peered":"2024-08-17T17:37:15.479481+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:37:15.479481+0000","last_undegraded":"2024-08-17T17:37:15.479481+0000","last_fullsized":"2024-08-17T17:37:15.479481+0000","mapping_epoch":337,"log_start":"324'1953","ondisk_log_start":"324'1953","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385941504,"num_objects":5889,"num_object_clones":0,"num_object_copies":17667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5889,"num_objects_unfound":0,"num_objects_dirty":5889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8845,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5889}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":64,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671793+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:36:57.671793+0000","last_peered":"2024-08-17T17:36:57.671793+0000","last_clean":"2024-08-17T17:36:57.671793+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:36:57.671793+0000","last_undegraded":"2024-08-17T17:36:57.671793+0000","last_fullsized":"2024-08-17T17:36:57.671793+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":342,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.677248+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:36:57.677248+0000","last_peered":"2024-08-17T17:36:57.677248+0000","last_clean":"2024-08-17T17:36:57.677248+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:36:57.677248+0000","last_undegraded":"2024-08-17T17:36:57.677248+0000","last_fullsized":"2024-08-17T17:36:57.677248+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"361'9010","reported_seq":9110,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:19.424639+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:37:19.424639+0000","last_peered":"2024-08-17T17:37:19.424639+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:37:19.424639+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2013","ondisk_log_start":"324'2013","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389808128,"num_objects":5948,"num_object_clones":0,"num_object_copies":17844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9010,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-821},{"shards":"0,3,7","objects":5948}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":475,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440756+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:37:12.440756+0000","last_peered":"2024-08-17T17:37:12.440756+0000","last_clean":"2024-08-17T17:37:12.440756+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:37:12.440756+0000","last_undegraded":"2024-08-17T17:37:12.440756+0000","last_fullsized":"2024-08-17T17:37:12.440756+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":241,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.671035+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:37:12.671035+0000","last_peered":"2024-08-17T17:37:12.671035+0000","last_clean":"2024-08-17T17:37:12.671035+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:37:12.671035+0000","last_undegraded":"2024-08-17T17:37:12.671035+0000","last_fullsized":"2024-08-17T17:37:12.671035+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"361'8779","reported_seq":8829,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:16.905754+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:37:16.905754+0000","last_peered":"2024-08-17T17:37:16.905754+0000","last_clean":"2024-08-17T17:37:16.905754+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:37:16.905754+0000","last_undegraded":"2024-08-17T17:37:16.905754+0000","last_fullsized":"2024-08-17T17:37:16.905754+0000","mapping_epoch":319,"log_start":"324'2174","ondisk_log_start":"324'2174","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381091840,"num_objects":5815,"num_object_clones":0,"num_object_copies":17445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8779,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":564,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440514+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:37:12.440514+0000","last_peered":"2024-08-17T17:37:12.440514+0000","last_clean":"2024-08-17T17:37:12.440514+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:37:12.440514+0000","last_undegraded":"2024-08-17T17:37:12.440514+0000","last_fullsized":"2024-08-17T17:37:12.440514+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":315,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673232+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:36:57.673232+0000","last_peered":"2024-08-17T17:36:57.673232+0000","last_clean":"2024-08-17T17:36:57.673232+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:36:57.673232+0000","last_undegraded":"2024-08-17T17:36:57.673232+0000","last_fullsized":"2024-08-17T17:36:57.673232+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"361'8951","reported_seq":9011,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:17.021069+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:37:17.021069+0000","last_peered":"2024-08-17T17:37:17.021069+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:37:17.021069+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'2335","ondisk_log_start":"324'2335","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391970816,"num_objects":5981,"num_object_clones":0,"num_object_copies":17943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8951,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5981},{"shards":"1,6","objects":-1385}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":316,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.677371+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:36:57.677371+0000","last_peered":"2024-08-17T17:36:57.677371+0000","last_clean":"2024-08-17T17:36:57.677371+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:36:57.677371+0000","last_undegraded":"2024-08-17T17:36:57.677371+0000","last_fullsized":"2024-08-17T17:36:57.677371+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673599+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:36:57.673599+0000","last_peered":"2024-08-17T17:36:57.673599+0000","last_clean":"2024-08-17T17:36:57.673599+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:36:57.673599+0000","last_undegraded":"2024-08-17T17:36:57.673599+0000","last_fullsized":"2024-08-17T17:36:57.673599+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"361'9033","reported_seq":9134,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:16.867968+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:37:16.867968+0000","last_peered":"2024-08-17T17:37:16.867968+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:37:16.867968+0000","last_undegraded":"2024-08-17T17:37:16.867968+0000","last_fullsized":"2024-08-17T17:37:16.867968+0000","mapping_epoch":337,"log_start":"325'3461","ondisk_log_start":"325'3461","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5572,"log_dups_size":0,"ondisk_log_size":5572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391708672,"num_objects":5977,"num_object_clones":0,"num_object_copies":17931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5977,"num_objects_unfound":0,"num_objects_dirty":5977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9033,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5977}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":353,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673835+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:36:57.673835+0000","last_peered":"2024-08-17T17:36:57.673835+0000","last_clean":"2024-08-17T17:36:57.673835+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:36:57.673835+0000","last_undegraded":"2024-08-17T17:36:57.673835+0000","last_fullsized":"2024-08-17T17:36:57.673835+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":315,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673528+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:36:57.673528+0000","last_peered":"2024-08-17T17:36:57.673528+0000","last_clean":"2024-08-17T17:36:57.673528+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:36:57.673528+0000","last_undegraded":"2024-08-17T17:36:57.673528+0000","last_fullsized":"2024-08-17T17:36:57.673528+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"361'8891","reported_seq":10631,"reported_epoch":361,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:16.557306+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:37:16.557306+0000","last_peered":"2024-08-17T17:37:16.557306+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:37:16.557306+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:37:16.557306+0000","mapping_epoch":337,"log_start":"324'2041","ondisk_log_start":"324'2041","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380960768,"num_objects":5813,"num_object_clones":0,"num_object_copies":17439,"num_objects_missing_on_primary":6511,"num_objects_missing":6511,"num_objects_degraded":5813,"num_objects_misplaced":6511,"num_objects_unfound":0,"num_objects_dirty":5813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8891,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":833,"num_bytes_recovered":54525952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":8},{"shards":"1,2,7","objects":5805}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":534,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.441095+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:37:12.441095+0000","last_peered":"2024-08-17T17:37:12.441095+0000","last_clean":"2024-08-17T17:37:12.441095+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:37:12.441095+0000","last_undegraded":"2024-08-17T17:37:12.441095+0000","last_fullsized":"2024-08-17T17:37:12.441095+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":471,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440128+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:37:12.440128+0000","last_peered":"2024-08-17T17:37:12.440128+0000","last_clean":"2024-08-17T17:37:12.440128+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:37:12.440128+0000","last_undegraded":"2024-08-17T17:37:12.440128+0000","last_fullsized":"2024-08-17T17:37:12.440128+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"361'8955","reported_seq":10559,"reported_epoch":361,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:16.962610+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:37:16.962610+0000","last_peered":"2024-08-17T17:37:16.962610+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:37:16.962610+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:37:16.962610+0000","mapping_epoch":337,"log_start":"325'3344","ondisk_log_start":"325'3344","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389283864,"num_objects":5941,"num_object_clones":0,"num_object_copies":17823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5941,"num_objects_misplaced":6691,"num_objects_unfound":0,"num_objects_dirty":5941,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8955,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":757,"num_bytes_recovered":49545216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5941}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":664,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:12.670603+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:37:12.670603+0000","last_peered":"2024-08-17T17:37:12.670603+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:37:12.670603+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":290,"reported_epoch":360,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:36:57.671902+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:36:57.671902+0000","last_peered":"2024-08-17T17:36:57.671902+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:36:57.671902+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:36:57.671902+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":513,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440349+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:37:12.440349+0000","last_peered":"2024-08-17T17:37:12.440349+0000","last_clean":"2024-08-17T17:37:12.440349+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:37:12.440349+0000","last_undegraded":"2024-08-17T17:37:12.440349+0000","last_fullsized":"2024-08-17T17:37:12.440349+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":325,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.677664+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:36:57.677664+0000","last_peered":"2024-08-17T17:36:57.677664+0000","last_clean":"2024-08-17T17:36:57.677664+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:36:57.677664+0000","last_undegraded":"2024-08-17T17:36:57.677664+0000","last_fullsized":"2024-08-17T17:36:57.677664+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"361'8926","reported_seq":9029,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:19.526207+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:37:19.526207+0000","last_peered":"2024-08-17T17:37:19.526207+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:37:19.526207+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'1921","ondisk_log_start":"324'1921","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381288448,"num_objects":5818,"num_object_clones":0,"num_object_copies":17454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8926,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-829},{"shards":"0,4,7","objects":5818}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.441320+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:37:12.441320+0000","last_peered":"2024-08-17T17:37:12.441320+0000","last_clean":"2024-08-17T17:37:12.441320+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:37:12.441320+0000","last_undegraded":"2024-08-17T17:37:12.441320+0000","last_fullsized":"2024-08-17T17:37:12.441320+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":297,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673050+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:36:57.673050+0000","last_peered":"2024-08-17T17:36:57.673050+0000","last_clean":"2024-08-17T17:36:57.673050+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:36:57.673050+0000","last_undegraded":"2024-08-17T17:36:57.673050+0000","last_fullsized":"2024-08-17T17:36:57.673050+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":255,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.672673+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:37:12.672673+0000","last_peered":"2024-08-17T17:37:12.672673+0000","last_clean":"2024-08-17T17:37:12.672673+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:37:12.672673+0000","last_undegraded":"2024-08-17T17:37:12.672673+0000","last_fullsized":"2024-08-17T17:37:12.672673+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"361'8903","reported_seq":8963,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:16.996650+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:37:16.996650+0000","last_peered":"2024-08-17T17:37:16.996650+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:37:16.996650+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'2317","ondisk_log_start":"324'2317","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386203648,"num_objects":5893,"num_object_clones":0,"num_object_copies":17679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8903,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5893},{"shards":"5,6","objects":-1403}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":421,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440041+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:37:12.440041+0000","last_peered":"2024-08-17T17:37:12.440041+0000","last_clean":"2024-08-17T17:37:12.440041+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:37:12.440041+0000","last_undegraded":"2024-08-17T17:37:12.440041+0000","last_fullsized":"2024-08-17T17:37:12.440041+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":97,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.677109+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:36:57.677109+0000","last_peered":"2024-08-17T17:36:57.677109+0000","last_clean":"2024-08-17T17:36:57.677109+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:36:57.677109+0000","last_undegraded":"2024-08-17T17:36:57.677109+0000","last_fullsized":"2024-08-17T17:36:57.677109+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":606,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440304+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:37:12.440304+0000","last_peered":"2024-08-17T17:37:12.440304+0000","last_clean":"2024-08-17T17:37:12.440304+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:37:12.440304+0000","last_undegraded":"2024-08-17T17:37:12.440304+0000","last_fullsized":"2024-08-17T17:37:12.440304+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"361'8817","reported_seq":9644,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:19.738264+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:37:19.738264+0000","last_peered":"2024-08-17T17:37:19.738264+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:37:19.738264+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1816","ondisk_log_start":"324'1816","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382009344,"num_objects":5829,"num_object_clones":0,"num_object_copies":17487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8817,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5829},{"shards":"0,5","objects":-825}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":398,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440672+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:37:12.440672+0000","last_peered":"2024-08-17T17:37:12.440672+0000","last_clean":"2024-08-17T17:37:12.440672+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:37:12.440672+0000","last_undegraded":"2024-08-17T17:37:12.440672+0000","last_fullsized":"2024-08-17T17:37:12.440672+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":336,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673876+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:36:57.673876+0000","last_peered":"2024-08-17T17:36:57.673876+0000","last_clean":"2024-08-17T17:36:57.673876+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:36:57.673876+0000","last_undegraded":"2024-08-17T17:36:57.673876+0000","last_fullsized":"2024-08-17T17:36:57.673876+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":192,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440601+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:37:12.440601+0000","last_peered":"2024-08-17T17:37:12.440601+0000","last_clean":"2024-08-17T17:37:12.440601+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:37:12.440601+0000","last_undegraded":"2024-08-17T17:37:12.440601+0000","last_fullsized":"2024-08-17T17:37:12.440601+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"361'8924","reported_seq":23870,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:15.272755+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:37:15.272755+0000","last_peered":"2024-08-17T17:37:15.272755+0000","last_clean":"2024-08-17T17:37:15.272755+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:37:15.272755+0000","last_undegraded":"2024-08-17T17:37:15.272755+0000","last_fullsized":"2024-08-17T17:37:15.272755+0000","mapping_epoch":355,"log_start":"324'2630","ondisk_log_start":"324'2630","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6294,"log_dups_size":0,"ondisk_log_size":6294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387448832,"num_objects":5912,"num_object_clones":0,"num_object_copies":17736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8924,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":275,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671860+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:36:57.671860+0000","last_peered":"2024-08-17T17:36:57.671860+0000","last_clean":"2024-08-17T17:36:57.671860+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:36:57.671860+0000","last_undegraded":"2024-08-17T17:36:57.671860+0000","last_fullsized":"2024-08-17T17:36:57.671860+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440549+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:37:12.440549+0000","last_peered":"2024-08-17T17:37:12.440549+0000","last_clean":"2024-08-17T17:37:12.440549+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:37:12.440549+0000","last_undegraded":"2024-08-17T17:37:12.440549+0000","last_fullsized":"2024-08-17T17:37:12.440549+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":557,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440963+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:37:12.440963+0000","last_peered":"2024-08-17T17:37:12.440963+0000","last_clean":"2024-08-17T17:37:12.440963+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:37:12.440963+0000","last_undegraded":"2024-08-17T17:37:12.440963+0000","last_fullsized":"2024-08-17T17:37:12.440963+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"361'8666","reported_seq":23180,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:14.449847+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:37:14.449847+0000","last_peered":"2024-08-17T17:37:14.449847+0000","last_clean":"2024-08-17T17:37:14.449847+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:37:14.449847+0000","last_undegraded":"2024-08-17T17:37:14.449847+0000","last_fullsized":"2024-08-17T17:37:14.449847+0000","mapping_epoch":345,"log_start":"324'2372","ondisk_log_start":"324'2372","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6294,"log_dups_size":0,"ondisk_log_size":6294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376045568,"num_objects":5738,"num_object_clones":0,"num_object_copies":17214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8666,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":271,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.672059+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:36:57.672059+0000","last_peered":"2024-08-17T17:36:57.672059+0000","last_clean":"2024-08-17T17:36:57.672059+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:36:57.672059+0000","last_undegraded":"2024-08-17T17:36:57.672059+0000","last_fullsized":"2024-08-17T17:36:57.672059+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":312,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.671096+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:37:12.671096+0000","last_peered":"2024-08-17T17:37:12.671096+0000","last_clean":"2024-08-17T17:37:12.671096+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:37:12.671096+0000","last_undegraded":"2024-08-17T17:37:12.671096+0000","last_fullsized":"2024-08-17T17:37:12.671096+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":413,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671765+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:36:57.671765+0000","last_peered":"2024-08-17T17:36:57.671765+0000","last_clean":"2024-08-17T17:36:57.671765+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:36:57.671765+0000","last_undegraded":"2024-08-17T17:36:57.671765+0000","last_fullsized":"2024-08-17T17:36:57.671765+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"361'8927","reported_seq":9049,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:15.438673+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:37:15.438673+0000","last_peered":"2024-08-17T17:37:15.438673+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:37:15.438673+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"324'2587","ondisk_log_start":"324'2587","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6340,"log_dups_size":0,"ondisk_log_size":6340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383844352,"num_objects":5857,"num_object_clones":0,"num_object_copies":17571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8927,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5857},{"shards":"0,3","objects":-1391}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":587,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440208+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:37:12.440208+0000","last_peered":"2024-08-17T17:37:12.440208+0000","last_clean":"2024-08-17T17:37:12.440208+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:37:12.440208+0000","last_undegraded":"2024-08-17T17:37:12.440208+0000","last_fullsized":"2024-08-17T17:37:12.440208+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":555,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.441696+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:37:12.441696+0000","last_peered":"2024-08-17T17:37:12.441696+0000","last_clean":"2024-08-17T17:37:12.441696+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:37:12.441696+0000","last_undegraded":"2024-08-17T17:37:12.441696+0000","last_fullsized":"2024-08-17T17:37:12.441696+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":573,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.439608+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:37:12.439608+0000","last_peered":"2024-08-17T17:37:12.439608+0000","last_clean":"2024-08-17T17:37:12.439608+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:37:12.439608+0000","last_undegraded":"2024-08-17T17:37:12.439608+0000","last_fullsized":"2024-08-17T17:37:12.439608+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"361'8802","reported_seq":23460,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:16.910328+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:37:16.910328+0000","last_peered":"2024-08-17T17:37:16.910328+0000","last_clean":"2024-08-17T17:37:16.910328+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:37:16.910328+0000","last_undegraded":"2024-08-17T17:37:16.910328+0000","last_fullsized":"2024-08-17T17:37:16.910328+0000","mapping_epoch":347,"log_start":"324'2272","ondisk_log_start":"324'2272","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378273792,"num_objects":5772,"num_object_clones":0,"num_object_copies":17316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8802,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":245,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.672590+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:37:12.672590+0000","last_peered":"2024-08-17T17:37:12.672590+0000","last_clean":"2024-08-17T17:37:12.672590+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:37:12.672590+0000","last_undegraded":"2024-08-17T17:37:12.672590+0000","last_fullsized":"2024-08-17T17:37:12.672590+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":365,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.678008+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:36:57.678008+0000","last_peered":"2024-08-17T17:36:57.678008+0000","last_clean":"2024-08-17T17:36:57.678008+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:36:57.678008+0000","last_undegraded":"2024-08-17T17:36:57.678008+0000","last_fullsized":"2024-08-17T17:36:57.678008+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":315,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673833+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:36:57.673833+0000","last_peered":"2024-08-17T17:36:57.673833+0000","last_clean":"2024-08-17T17:36:57.673833+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:36:57.673833+0000","last_undegraded":"2024-08-17T17:36:57.673833+0000","last_fullsized":"2024-08-17T17:36:57.673833+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":315,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673750+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:36:57.673750+0000","last_peered":"2024-08-17T17:36:57.673750+0000","last_clean":"2024-08-17T17:36:57.673750+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:36:57.673750+0000","last_undegraded":"2024-08-17T17:36:57.673750+0000","last_fullsized":"2024-08-17T17:36:57.673750+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":358,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673319+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:36:57.673319+0000","last_peered":"2024-08-17T17:36:57.673319+0000","last_clean":"2024-08-17T17:36:57.673319+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:36:57.673319+0000","last_undegraded":"2024-08-17T17:36:57.673319+0000","last_fullsized":"2024-08-17T17:36:57.673319+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":562,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440760+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:37:12.440760+0000","last_peered":"2024-08-17T17:37:12.440760+0000","last_clean":"2024-08-17T17:37:12.440760+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:37:12.440760+0000","last_undegraded":"2024-08-17T17:37:12.440760+0000","last_fullsized":"2024-08-17T17:37:12.440760+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":102,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.441058+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:37:12.441058+0000","last_peered":"2024-08-17T17:37:12.441058+0000","last_clean":"2024-08-17T17:37:12.441058+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:37:12.441058+0000","last_undegraded":"2024-08-17T17:37:12.441058+0000","last_fullsized":"2024-08-17T17:37:12.441058+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":265,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.671878+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:36:57.671878+0000","last_peered":"2024-08-17T17:36:57.671878+0000","last_clean":"2024-08-17T17:36:57.671878+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:36:57.671878+0000","last_undegraded":"2024-08-17T17:36:57.671878+0000","last_fullsized":"2024-08-17T17:36:57.671878+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":520,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.439691+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:37:12.439691+0000","last_peered":"2024-08-17T17:37:12.439691+0000","last_clean":"2024-08-17T17:37:12.439691+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:37:12.439691+0000","last_undegraded":"2024-08-17T17:37:12.439691+0000","last_fullsized":"2024-08-17T17:37:12.439691+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":409,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.672253+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:36:57.672253+0000","last_peered":"2024-08-17T17:36:57.672253+0000","last_clean":"2024-08-17T17:36:57.672253+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:36:57.672253+0000","last_undegraded":"2024-08-17T17:36:57.672253+0000","last_fullsized":"2024-08-17T17:36:57.672253+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":514,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.439503+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:37:12.439503+0000","last_peered":"2024-08-17T17:37:12.439503+0000","last_clean":"2024-08-17T17:37:12.439503+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:37:12.439503+0000","last_undegraded":"2024-08-17T17:37:12.439503+0000","last_fullsized":"2024-08-17T17:37:12.439503+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":332,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.677936+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:36:57.677936+0000","last_peered":"2024-08-17T17:36:57.677936+0000","last_clean":"2024-08-17T17:36:57.677936+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:36:57.677936+0000","last_undegraded":"2024-08-17T17:36:57.677936+0000","last_fullsized":"2024-08-17T17:36:57.677936+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":546,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.439694+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:37:12.439694+0000","last_peered":"2024-08-17T17:37:12.439694+0000","last_clean":"2024-08-17T17:37:12.439694+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:37:12.439694+0000","last_undegraded":"2024-08-17T17:37:12.439694+0000","last_fullsized":"2024-08-17T17:37:12.439694+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":261,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440729+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:37:12.440729+0000","last_peered":"2024-08-17T17:37:12.440729+0000","last_clean":"2024-08-17T17:37:12.440729+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:37:12.440729+0000","last_undegraded":"2024-08-17T17:37:12.440729+0000","last_fullsized":"2024-08-17T17:37:12.440729+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":554,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440057+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:37:12.440057+0000","last_peered":"2024-08-17T17:37:12.440057+0000","last_clean":"2024-08-17T17:37:12.440057+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:37:12.440057+0000","last_undegraded":"2024-08-17T17:37:12.440057+0000","last_fullsized":"2024-08-17T17:37:12.440057+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":513,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440898+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:37:12.440898+0000","last_peered":"2024-08-17T17:37:12.440898+0000","last_clean":"2024-08-17T17:37:12.440898+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:37:12.440898+0000","last_undegraded":"2024-08-17T17:37:12.440898+0000","last_fullsized":"2024-08-17T17:37:12.440898+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":344,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.674025+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:36:57.674025+0000","last_peered":"2024-08-17T17:36:57.674025+0000","last_clean":"2024-08-17T17:36:57.674025+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:36:57.674025+0000","last_undegraded":"2024-08-17T17:36:57.674025+0000","last_fullsized":"2024-08-17T17:36:57.674025+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":202,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440693+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:37:12.440693+0000","last_peered":"2024-08-17T17:37:12.440693+0000","last_clean":"2024-08-17T17:37:12.440693+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:37:12.440693+0000","last_undegraded":"2024-08-17T17:37:12.440693+0000","last_fullsized":"2024-08-17T17:37:12.440693+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":259,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.670920+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:37:12.670920+0000","last_peered":"2024-08-17T17:37:12.670920+0000","last_clean":"2024-08-17T17:37:12.670920+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:37:12.670920+0000","last_undegraded":"2024-08-17T17:37:12.670920+0000","last_fullsized":"2024-08-17T17:37:12.670920+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":352,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673688+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:36:57.673688+0000","last_peered":"2024-08-17T17:36:57.673688+0000","last_clean":"2024-08-17T17:36:57.673688+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:36:57.673688+0000","last_undegraded":"2024-08-17T17:36:57.673688+0000","last_fullsized":"2024-08-17T17:36:57.673688+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":215,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673317+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:36:57.673317+0000","last_peered":"2024-08-17T17:36:57.673317+0000","last_clean":"2024-08-17T17:36:57.673317+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:36:57.673317+0000","last_undegraded":"2024-08-17T17:36:57.673317+0000","last_fullsized":"2024-08-17T17:36:57.673317+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":352,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673967+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:36:57.673967+0000","last_peered":"2024-08-17T17:36:57.673967+0000","last_clean":"2024-08-17T17:36:57.673967+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:36:57.673967+0000","last_undegraded":"2024-08-17T17:36:57.673967+0000","last_fullsized":"2024-08-17T17:36:57.673967+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":215,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.673462+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:36:57.673462+0000","last_peered":"2024-08-17T17:36:57.673462+0000","last_clean":"2024-08-17T17:36:57.673462+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:36:57.673462+0000","last_undegraded":"2024-08-17T17:36:57.673462+0000","last_fullsized":"2024-08-17T17:36:57.673462+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":566,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440087+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:37:12.440087+0000","last_peered":"2024-08-17T17:37:12.440087+0000","last_clean":"2024-08-17T17:37:12.440087+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:37:12.440087+0000","last_undegraded":"2024-08-17T17:37:12.440087+0000","last_fullsized":"2024-08-17T17:37:12.440087+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":539,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.440894+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:37:12.440894+0000","last_peered":"2024-08-17T17:37:12.440894+0000","last_clean":"2024-08-17T17:37:12.440894+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:37:12.440894+0000","last_undegraded":"2024-08-17T17:37:12.440894+0000","last_fullsized":"2024-08-17T17:37:12.440894+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":315,"reported_epoch":360,"state":"active+clean","last_fresh":"2024-08-17T17:36:57.678521+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:36:57.678521+0000","last_peered":"2024-08-17T17:36:57.678521+0000","last_clean":"2024-08-17T17:36:57.678521+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:36:57.678521+0000","last_undegraded":"2024-08-17T17:36:57.678521+0000","last_fullsized":"2024-08-17T17:36:57.678521+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":261,"reported_epoch":361,"state":"active+clean","last_fresh":"2024-08-17T17:37:12.671237+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:37:12.671237+0000","last_peered":"2024-08-17T17:37:12.671237+0000","last_clean":"2024-08-17T17:37:12.671237+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:37:12.671237+0000","last_undegraded":"2024-08-17T17:37:12.671237+0000","last_fullsized":"2024-08-17T17:37:12.671237+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6161170456,"num_objects":94013,"num_object_clones":0,"num_object_copies":282039,"num_objects_missing_on_primary":6511,"num_objects_missing":6511,"num_objects_degraded":56173,"num_objects_misplaced":36886,"num_objects_unfound":0,"num_objects_dirty":94013,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142377,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24658,"num_bytes_recovered":1493434368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19988033536,"data_stored":19988021320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104696,"ondisk_log_size":104696,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170050,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463420,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3526888,"kb_used_data":2633184,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90218264,"statfs":{"total":95995035648,"available":92383502336,"internally_reserved":0,"allocated":2696380416,"data_stored":2694466904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492095,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1149004,"kb_used_data":753668,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92596148,"statfs":{"total":95995035648,"available":94818455552,"internally_reserved":0,"allocated":771756032,"data_stored":769864820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986686,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2390044,"kb_used_data":1497428,"kb_used_omap":1,"kb_used_meta":892606,"kb_avail":91355108,"statfs":{"total":95995035648,"available":93547630592,"internally_reserved":0,"allocated":1533366272,"data_stored":1531463700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914028888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2,2,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225513,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3936016,"kb_used_data":3011972,"kb_used_omap":4,"kb_used_meta":924027,"kb_avail":89809136,"statfs":{"total":95995035648,"available":91964555264,"internally_reserved":0,"allocated":3084259328,"data_stored":3082368414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":946204096},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075839,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533731,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3690000,"kb_used_data":2643848,"kb_used_omap":5,"kb_used_meta":1046138,"kb_avail":90055152,"statfs":{"total":95995035648,"available":92216475648,"internally_reserved":0,"allocated":2707300352,"data_stored":2705404116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1071245552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089709,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3257704,"kb_used_data":2262368,"kb_used_omap":5,"kb_used_meta":995322,"kb_avail":90487448,"statfs":{"total":95995035648,"available":92659146752,"internally_reserved":0,"allocated":2316664832,"data_stored":2314751379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1019209787},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2310930432,"data_stored":2310930432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2700152832,"data_stored":2700148760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4094955520,"data_stored":4094951448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3078094848,"data_stored":3078094848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1527971840,"data_stored":1527971840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":766771200,"data_stored":766771200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2687369216,"data_stored":2687369216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2821787648,"data_stored":2821783576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:37:26.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:26.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:26.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:26.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:26.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:26.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:26.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:26.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:26.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:26.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56121/280890 objects degraded (19.980%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:26.583 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:26.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56121/280890 objects degraded (19.980%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56121/280890 objects degraded (19.980%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:26.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:26.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:26.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:26.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:26.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:26.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:26.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:26.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:27.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:27.082 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:27.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:27.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:27.208 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:27.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:27.315 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:27.315 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:27.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:27.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:27.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:27.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:27.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:37:27.519 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:27 smithi142 ceph-mon[103614]: pgmap v1308: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 56121/280245 objects degraded (20.026%); 36699/280245 objects misplaced (13.095%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:27.519 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:27.519 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:27.520 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:27 smithi142 ceph-mon[106729]: pgmap v1308: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 56121/280245 objects degraded (20.026%); 36699/280245 objects misplaced (13.095%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:27.520 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:27.520 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:27.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:27.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:27.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:27.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:27.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:27.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:27 smithi152 ceph-mon[93456]: pgmap v1308: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 56121/280245 objects degraded (20.026%); 36699/280245 objects misplaced (13.095%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:27.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:27.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:27.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:27.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:28.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:28.075 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:28.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:37:28.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:28.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:28.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:28.340 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:28.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:28.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:28.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:28.545 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:28.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:28.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:28.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:28.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:28.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:28.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:28.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:28.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:29.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:37:29.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:29.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:29.090 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:29.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:29.209 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:29.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:29.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:29.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:29.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:29.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:29.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:29.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:29.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:29 smithi142 ceph-mon[106729]: pgmap v1309: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 56051/279231 objects degraded (20.073%); 36479/279231 objects misplaced (13.064%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:29.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:29 smithi142 ceph-mon[103614]: pgmap v1309: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 56051/279231 objects degraded (20.073%); 36479/279231 objects misplaced (13.064%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:29.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:29.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:29.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:29 smithi152 ceph-mon[93456]: pgmap v1309: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 56051/279231 objects degraded (20.073%); 36479/279231 objects misplaced (13.064%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:29.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:29.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:29.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:29.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:30.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:30.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:30.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:30.209 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:30.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:30.334 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:30.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:30.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:30.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:37:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:37:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:30.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:30.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:30.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:30.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:30.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:30.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:30.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:30.935 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:30.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:31.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:31.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:31.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:31.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:31.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:31.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:31.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:31.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:31.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:31.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:31.577 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:31.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: pgmap v1310: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 56051/279015 objects degraded (20.089%); 36479/279015 objects misplaced (13.074%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 56051/279015 objects degraded (20.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: pgmap v1310: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 56051/279015 objects degraded (20.089%); 36479/279015 objects misplaced (13.074%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 56051/279015 objects degraded (20.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:31.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:31.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:31.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: pgmap v1310: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 56051/279015 objects degraded (20.089%); 36479/279015 objects misplaced (13.074%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 56051/279015 objects degraded (20.089%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:37:31.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:31.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:31.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:32.007 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:37:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:37:31] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:37:32.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:32.073 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:32.175 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:32.198 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:32.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:32.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:32.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:32.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:32.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:32.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:32.548 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:32.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:32.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:32.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:32.694 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:32.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:32.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:32.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:32.941 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:33.043 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:33.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:33.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:33.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:33.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:33.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:33.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:33.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:33.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:33.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:33.568 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:33 smithi142 ceph-mon[106729]: pgmap v1311: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55998/278073 objects degraded (20.138%); 36330/278073 objects misplaced (13.065%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:33.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:33 smithi142 ceph-mon[103614]: pgmap v1311: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55998/278073 objects degraded (20.138%); 36330/278073 objects misplaced (13.065%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:33.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:33.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:33 smithi152 ceph-mon[93456]: pgmap v1311: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55998/278073 objects degraded (20.138%); 36330/278073 objects misplaced (13.065%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:33.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.941 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:34.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:34.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:34.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:34.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:34.217 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:37:34.218 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:34.224 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:37:34.224 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:34.224 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:34.241 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:37:34.241 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:34.261 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:37:34.261 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:34.278 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:37:34.278 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:34.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:34.294 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:37:34.294 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:34.312 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:37:34.312 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:34.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:34.329 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:37:34.330 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:34.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.447 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:37:34.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2015571047' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:34.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3271622342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:34.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2015571047' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:34.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3271622342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:34.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:34.565 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:34.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:34.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:34.791 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:34.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2015571047' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:37:34.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3271622342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:34.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:34.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:35.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:35.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:35.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:35.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:35.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:35.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:35.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:35.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:35.535 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:35.557 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:35.659 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:35.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:35.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[106729]: pgmap v1312: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55998/278073 objects degraded (20.138%); 36330/278073 objects misplaced (13.065%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:35.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:35.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2015571047' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:37:35.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[106729]: osdmap e363: 8 total, 8 up, 6 in 2024-08-17T17:37:35.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[103614]: pgmap v1312: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55998/278073 objects degraded (20.138%); 36330/278073 objects misplaced (13.065%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:35.730 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:35.730 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2015571047' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:37:35.730 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:35 smithi142 ceph-mon[103614]: osdmap e363: 8 total, 8 up, 6 in 2024-08-17T17:37:35.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:35.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:35.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:35 smithi152 ceph-mon[93456]: pgmap v1312: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55998/278073 objects degraded (20.138%); 36330/278073 objects misplaced (13.065%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:35 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2015571047' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:37:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:35 smithi152 ceph-mon[93456]: osdmap e363: 8 total, 8 up, 6 in 2024-08-17T17:37:35.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:35.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:36.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:36.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:36.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:36.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:36.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:36.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:36.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:36.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:36.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:36.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:36.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:36.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:36.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:36.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:36.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55998/278073 objects degraded (20.138%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55998/278073 objects degraded (20.138%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55998/278073 objects degraded (20.138%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:36.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:36.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:36.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:37.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:37.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:37.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:37.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:37.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:37.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:37.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:37.294 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:37.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:37.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:37.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:37.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:37.607 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:37:37.607 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:37:37.608 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:37:37.608 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:37:37.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:37 smithi142 ceph-mon[106729]: pgmap v1314: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55998/277623 objects degraded (20.171%); 36330/277623 objects misplaced (13.086%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:37.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:37 smithi142 ceph-mon[103614]: pgmap v1314: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55998/277623 objects degraded (20.171%); 36330/277623 objects misplaced (13.086%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:37.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:37.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:37.767 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:37.789 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:37 smithi152 ceph-mon[93456]: pgmap v1314: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55998/277623 objects degraded (20.171%); 36330/277623 objects misplaced (13.086%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:37.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:37.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:38.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:38.036 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:38.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:38.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:38.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:37:38.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:38.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:38.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:38.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:38.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:38.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:37:38.585 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:37:38.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:38.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:38.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:38.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:38.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:38.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:37:38.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:38.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:37:39.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:39.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:39.144 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:39.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:39.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:39.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:39.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:39.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:39.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:39.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:39.429 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:39.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:39.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:39.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:39.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:39 smithi142 ceph-mon[103614]: pgmap v1315: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 55943/276906 objects degraded (20.203%); 36182/276906 objects misplaced (13.067%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:39 smithi142 ceph-mon[106729]: pgmap v1315: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 55943/276906 objects degraded (20.203%); 36182/276906 objects misplaced (13.067%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:39.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:39.680 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:39.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:39.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:39.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:39 smithi152 ceph-mon[93456]: pgmap v1315: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 55943/276906 objects degraded (20.203%); 36182/276906 objects misplaced (13.067%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:39.864 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1315,"stamp":"2024-08-17T17:37:38.278500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6047930341,"num_objects":92302,"num_object_clones":0,"num_object_copies":276906,"num_objects_missing_on_primary":6391,"num_objects_missing":6391,"num_objects_degraded":55943,"num_objects_misplaced":36182,"num_objects_unfound":0,"num_objects_dirty":92302,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":144392,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25043,"num_bytes_recovered":1518346894,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105040,"ondisk_log_size":105040,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17696800,"kb_used_data":12566244,"kb_used_omap":27,"kb_used_meta":5130468,"kb_avail":544774112,"statfs":{"total":575970213888,"available":557848690688,"internally_reserved":0,"allocated":12867833856,"data_stored":12856332441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5253599976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,0,0,0,1,1,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-50790400,"num_objects":-775,"num_object_clones":0,"num_object_copies":-2325,"num_objects_missing_on_primary":-60,"num_objects_missing":-60,"num_objects_degraded":-108,"num_objects_misplaced":-297,"num_objects_unfound":0,"num_objects_dirty":-775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":775,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002793"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":567,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452600+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:37:34.452600+0000","last_peered":"2024-08-17T17:37:34.452600+0000","last_clean":"2024-08-17T17:37:34.452600+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:37:34.452600+0000","last_undegraded":"2024-08-17T17:37:34.452600+0000","last_fullsized":"2024-08-17T17:37:34.452600+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":72,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307051+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:37:24.307051+0000","last_peered":"2024-08-17T17:37:24.307051+0000","last_clean":"2024-08-17T17:37:24.307051+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:37:24.307051+0000","last_undegraded":"2024-08-17T17:37:24.307051+0000","last_fullsized":"2024-08-17T17:37:24.307051+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.335093+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:37:24.335093+0000","last_peered":"2024-08-17T17:37:24.335093+0000","last_clean":"2024-08-17T17:37:24.335093+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:37:24.335093+0000","last_undegraded":"2024-08-17T17:37:24.335093+0000","last_fullsized":"2024-08-17T17:37:24.335093+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":265,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452293+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:37:34.452293+0000","last_peered":"2024-08-17T17:37:34.452293+0000","last_clean":"2024-08-17T17:37:34.452293+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:37:34.452293+0000","last_undegraded":"2024-08-17T17:37:34.452293+0000","last_fullsized":"2024-08-17T17:37:34.452293+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":442,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.453376+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:37:34.453376+0000","last_peered":"2024-08-17T17:37:34.453376+0000","last_clean":"2024-08-17T17:37:34.453376+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:37:34.453376+0000","last_undegraded":"2024-08-17T17:37:34.453376+0000","last_fullsized":"2024-08-17T17:37:34.453376+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306789+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:37:24.306789+0000","last_peered":"2024-08-17T17:37:24.306789+0000","last_clean":"2024-08-17T17:37:24.306789+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:37:24.306789+0000","last_undegraded":"2024-08-17T17:37:24.306789+0000","last_fullsized":"2024-08-17T17:37:24.306789+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":536,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.453192+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:37:34.453192+0000","last_peered":"2024-08-17T17:37:34.453192+0000","last_clean":"2024-08-17T17:37:34.453192+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:37:34.453192+0000","last_undegraded":"2024-08-17T17:37:34.453192+0000","last_fullsized":"2024-08-17T17:37:34.453192+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":383,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.314859+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:37:24.314859+0000","last_peered":"2024-08-17T17:37:24.314859+0000","last_clean":"2024-08-17T17:37:24.314859+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:37:24.314859+0000","last_undegraded":"2024-08-17T17:37:24.314859+0000","last_fullsized":"2024-08-17T17:37:24.314859+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"363'8981","reported_seq":10856,"reported_epoch":363,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:37:36.732562+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:37:36.732562+0000","last_peered":"2024-08-17T17:37:36.732562+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:37:36.732562+0000","last_undegraded":"2024-08-17T17:37:36.732562+0000","last_fullsized":"2024-08-17T17:37:36.732562+0000","mapping_epoch":337,"log_start":"324'2063","ondisk_log_start":"324'2063","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6918,"log_dups_size":0,"ondisk_log_size":6918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375062528,"num_objects":5723,"num_object_clones":0,"num_object_copies":17169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5723,"num_objects_unfound":0,"num_objects_dirty":5723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8981,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":900,"num_bytes_recovered":58982400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5723}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":68,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.354059+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:37:24.354059+0000","last_peered":"2024-08-17T17:37:24.354059+0000","last_clean":"2024-08-17T17:37:24.354059+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:37:24.354059+0000","last_undegraded":"2024-08-17T17:37:24.354059+0000","last_fullsized":"2024-08-17T17:37:24.354059+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.335904+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:37:24.335904+0000","last_peered":"2024-08-17T17:37:24.335904+0000","last_clean":"2024-08-17T17:37:24.335904+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:37:24.335904+0000","last_undegraded":"2024-08-17T17:37:24.335904+0000","last_fullsized":"2024-08-17T17:37:24.335904+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"363'9201","reported_seq":9262,"reported_epoch":363,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:36.703563+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:37:36.703563+0000","last_peered":"2024-08-17T17:37:36.703563+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:37:36.703563+0000","last_undegraded":"2024-08-17T17:37:36.703563+0000","last_fullsized":"2024-08-17T17:37:36.703563+0000","mapping_epoch":337,"log_start":"324'2582","ondisk_log_start":"324'2582","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384892928,"num_objects":5873,"num_object_clones":0,"num_object_copies":17619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5873,"num_objects_unfound":0,"num_objects_dirty":5873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9201,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5873}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":70,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307406+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:37:24.307406+0000","last_peered":"2024-08-17T17:37:24.307406+0000","last_clean":"2024-08-17T17:37:24.307406+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:37:24.307406+0000","last_undegraded":"2024-08-17T17:37:24.307406+0000","last_fullsized":"2024-08-17T17:37:24.307406+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":334,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306726+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:37:24.306726+0000","last_peered":"2024-08-17T17:37:24.306726+0000","last_clean":"2024-08-17T17:37:24.306726+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:37:24.306726+0000","last_undegraded":"2024-08-17T17:37:24.306726+0000","last_fullsized":"2024-08-17T17:37:24.306726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"363'8966","reported_seq":9030,"reported_epoch":363,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:35.517903+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:37:35.517903+0000","last_peered":"2024-08-17T17:37:35.517903+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:37:35.517903+0000","last_undegraded":"2024-08-17T17:37:35.517903+0000","last_fullsized":"2024-08-17T17:37:35.517903+0000","mapping_epoch":337,"log_start":"324'2053","ondisk_log_start":"324'2053","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378011648,"num_objects":5768,"num_object_clones":0,"num_object_copies":17304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5768,"num_objects_unfound":0,"num_objects_dirty":5768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8966,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5768}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":66,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.335022+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:37:24.335022+0000","last_peered":"2024-08-17T17:37:24.335022+0000","last_clean":"2024-08-17T17:37:24.335022+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:37:24.335022+0000","last_undegraded":"2024-08-17T17:37:24.335022+0000","last_fullsized":"2024-08-17T17:37:24.335022+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":344,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.314800+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:37:24.314800+0000","last_peered":"2024-08-17T17:37:24.314800+0000","last_clean":"2024-08-17T17:37:24.314800+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:37:24.314800+0000","last_undegraded":"2024-08-17T17:37:24.314800+0000","last_fullsized":"2024-08-17T17:37:24.314800+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"363'9091","reported_seq":9195,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:34.856104+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:37:34.856104+0000","last_peered":"2024-08-17T17:37:34.856104+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:37:34.856104+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2113","ondisk_log_start":"324'2113","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384499712,"num_objects":5867,"num_object_clones":0,"num_object_copies":17601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9091,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-902},{"shards":"0,3,7","objects":5867}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":479,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452756+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:37:34.452756+0000","last_peered":"2024-08-17T17:37:34.452756+0000","last_clean":"2024-08-17T17:37:34.452756+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:37:34.452756+0000","last_undegraded":"2024-08-17T17:37:34.452756+0000","last_fullsized":"2024-08-17T17:37:34.452756+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":245,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.504330+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:37:34.504330+0000","last_peered":"2024-08-17T17:37:34.504330+0000","last_clean":"2024-08-17T17:37:34.504330+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:37:34.504330+0000","last_undegraded":"2024-08-17T17:37:34.504330+0000","last_fullsized":"2024-08-17T17:37:34.504330+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"363'8883","reported_seq":8935,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:35.656460+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:37:35.656460+0000","last_peered":"2024-08-17T17:37:35.656460+0000","last_clean":"2024-08-17T17:37:35.656460+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:37:35.656460+0000","last_undegraded":"2024-08-17T17:37:35.656460+0000","last_fullsized":"2024-08-17T17:37:35.656460+0000","mapping_epoch":319,"log_start":"324'2274","ondisk_log_start":"324'2274","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374276096,"num_objects":5711,"num_object_clones":0,"num_object_copies":17133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8883,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":569,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452510+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:37:34.452510+0000","last_peered":"2024-08-17T17:37:34.452510+0000","last_clean":"2024-08-17T17:37:34.452510+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:37:34.452510+0000","last_undegraded":"2024-08-17T17:37:34.452510+0000","last_fullsized":"2024-08-17T17:37:34.452510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":317,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307339+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:37:24.307339+0000","last_peered":"2024-08-17T17:37:24.307339+0000","last_clean":"2024-08-17T17:37:24.307339+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:37:24.307339+0000","last_undegraded":"2024-08-17T17:37:24.307339+0000","last_fullsized":"2024-08-17T17:37:24.307339+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"363'9040","reported_seq":9102,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:35.555070+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:37:35.555070+0000","last_peered":"2024-08-17T17:37:35.555070+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:37:35.555070+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"324'2435","ondisk_log_start":"324'2435","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386138112,"num_objects":5892,"num_object_clones":0,"num_object_copies":17676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9040,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5892},{"shards":"1,6","objects":-1474}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":318,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.314970+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:37:24.314970+0000","last_peered":"2024-08-17T17:37:24.314970+0000","last_clean":"2024-08-17T17:37:24.314970+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:37:24.314970+0000","last_undegraded":"2024-08-17T17:37:24.314970+0000","last_fullsized":"2024-08-17T17:37:24.314970+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":344,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307892+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:37:24.307892+0000","last_peered":"2024-08-17T17:37:24.307892+0000","last_clean":"2024-08-17T17:37:24.307892+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:37:24.307892+0000","last_undegraded":"2024-08-17T17:37:24.307892+0000","last_fullsized":"2024-08-17T17:37:24.307892+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"363'9154","reported_seq":9259,"reported_epoch":363,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:35.959302+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:37:35.959302+0000","last_peered":"2024-08-17T17:37:35.959302+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:37:35.959302+0000","last_undegraded":"2024-08-17T17:37:35.959302+0000","last_fullsized":"2024-08-17T17:37:35.959302+0000","mapping_epoch":337,"log_start":"325'3561","ondisk_log_start":"325'3561","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5593,"log_dups_size":0,"ondisk_log_size":5593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383778816,"num_objects":5856,"num_object_clones":0,"num_object_copies":17568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5856,"num_objects_unfound":0,"num_objects_dirty":5856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9154,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5856}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":355,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306919+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:37:24.306919+0000","last_peered":"2024-08-17T17:37:24.306919+0000","last_clean":"2024-08-17T17:37:24.306919+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:37:24.306919+0000","last_undegraded":"2024-08-17T17:37:24.306919+0000","last_fullsized":"2024-08-17T17:37:24.306919+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":317,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307524+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:37:24.307524+0000","last_peered":"2024-08-17T17:37:24.307524+0000","last_clean":"2024-08-17T17:37:24.307524+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:37:24.307524+0000","last_undegraded":"2024-08-17T17:37:24.307524+0000","last_fullsized":"2024-08-17T17:37:24.307524+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"363'9004","reported_seq":10980,"reported_epoch":363,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:36.721352+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:37:36.721352+0000","last_peered":"2024-08-17T17:37:36.721352+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:37:36.721352+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:37:36.721352+0000","mapping_epoch":337,"log_start":"324'2141","ondisk_log_start":"324'2141","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6863,"log_dups_size":0,"ondisk_log_size":6863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373555200,"num_objects":5700,"num_object_clones":0,"num_object_copies":17100,"num_objects_missing_on_primary":6391,"num_objects_missing":6391,"num_objects_degraded":5700,"num_objects_misplaced":6391,"num_objects_unfound":0,"num_objects_dirty":5700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9004,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":946,"num_bytes_recovered":61931520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":15},{"shards":"1,2,7","objects":5685}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":538,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.451929+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:37:34.451929+0000","last_peered":"2024-08-17T17:37:34.451929+0000","last_clean":"2024-08-17T17:37:34.451929+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:37:34.451929+0000","last_undegraded":"2024-08-17T17:37:34.451929+0000","last_fullsized":"2024-08-17T17:37:34.451929+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":475,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.453486+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:37:34.453486+0000","last_peered":"2024-08-17T17:37:34.453486+0000","last_clean":"2024-08-17T17:37:34.453486+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:37:34.453486+0000","last_undegraded":"2024-08-17T17:37:34.453486+0000","last_fullsized":"2024-08-17T17:37:34.453486+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"363'9075","reported_seq":10923,"reported_epoch":363,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:36.712652+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:37:36.712652+0000","last_peered":"2024-08-17T17:37:36.712652+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:37:36.712652+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:37:36.712652+0000","mapping_epoch":337,"log_start":"325'3444","ondisk_log_start":"325'3444","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381419544,"num_objects":5821,"num_object_clones":0,"num_object_copies":17463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5821,"num_objects_misplaced":6571,"num_objects_unfound":0,"num_objects_dirty":5821,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9075,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":877,"num_bytes_recovered":57409536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5821}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":668,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:34.503911+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:37:34.503911+0000","last_peered":"2024-08-17T17:37:34.503911+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:37:34.503911+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":292,"reported_epoch":362,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:37:24.334927+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:37:24.334927+0000","last_peered":"2024-08-17T17:37:24.334927+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:37:24.334927+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:37:24.334927+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":517,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452249+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:37:34.452249+0000","last_peered":"2024-08-17T17:37:34.452249+0000","last_clean":"2024-08-17T17:37:34.452249+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:37:34.452249+0000","last_undegraded":"2024-08-17T17:37:34.452249+0000","last_fullsized":"2024-08-17T17:37:34.452249+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.315343+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:37:24.315343+0000","last_peered":"2024-08-17T17:37:24.315343+0000","last_clean":"2024-08-17T17:37:24.315343+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:37:24.315343+0000","last_undegraded":"2024-08-17T17:37:24.315343+0000","last_fullsized":"2024-08-17T17:37:24.315343+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"363'9008","reported_seq":9115,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:34.610164+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:37:34.610164+0000","last_peered":"2024-08-17T17:37:34.610164+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:37:34.610164+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2021","ondisk_log_start":"324'2021","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375914496,"num_objects":5736,"num_object_clones":0,"num_object_copies":17208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9008,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-911},{"shards":"0,4,7","objects":5736}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":333,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.449409+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:37:34.449409+0000","last_peered":"2024-08-17T17:37:34.449409+0000","last_clean":"2024-08-17T17:37:34.449409+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:37:34.449409+0000","last_undegraded":"2024-08-17T17:37:34.449409+0000","last_fullsized":"2024-08-17T17:37:34.449409+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":299,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306604+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:37:24.306604+0000","last_peered":"2024-08-17T17:37:24.306604+0000","last_clean":"2024-08-17T17:37:24.306604+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:37:24.306604+0000","last_undegraded":"2024-08-17T17:37:24.306604+0000","last_fullsized":"2024-08-17T17:37:24.306604+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":260,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.504540+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:37:34.504540+0000","last_peered":"2024-08-17T17:37:34.504540+0000","last_clean":"2024-08-17T17:37:34.504540+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:37:34.504540+0000","last_undegraded":"2024-08-17T17:37:34.504540+0000","last_fullsized":"2024-08-17T17:37:34.504540+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"363'9021","reported_seq":9083,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:37.015085+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:37:37.015085+0000","last_peered":"2024-08-17T17:37:37.015085+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:37:37.015085+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"324'2417","ondisk_log_start":"324'2417","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378470400,"num_objects":5775,"num_object_clones":0,"num_object_copies":17325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9021,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5775},{"shards":"5,6","objects":-1521}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":426,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.451985+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:37:34.451985+0000","last_peered":"2024-08-17T17:37:34.451985+0000","last_clean":"2024-08-17T17:37:34.451985+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:37:34.451985+0000","last_undegraded":"2024-08-17T17:37:34.451985+0000","last_fullsized":"2024-08-17T17:37:34.451985+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":99,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.314663+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:37:24.314663+0000","last_peered":"2024-08-17T17:37:24.314663+0000","last_clean":"2024-08-17T17:37:24.314663+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:37:24.314663+0000","last_undegraded":"2024-08-17T17:37:24.314663+0000","last_fullsized":"2024-08-17T17:37:24.314663+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":610,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.449969+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:37:34.449969+0000","last_peered":"2024-08-17T17:37:34.449969+0000","last_clean":"2024-08-17T17:37:34.449969+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:37:34.449969+0000","last_undegraded":"2024-08-17T17:37:34.449969+0000","last_fullsized":"2024-08-17T17:37:34.449969+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"363'8905","reported_seq":9736,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:34.824662+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:37:34.824662+0000","last_peered":"2024-08-17T17:37:34.824662+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:37:34.824662+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'1916","ondisk_log_start":"324'1916","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376242176,"num_objects":5741,"num_object_clones":0,"num_object_copies":17223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8905,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5741},{"shards":"0,5","objects":-913}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":402,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.449866+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:37:34.449866+0000","last_peered":"2024-08-17T17:37:34.449866+0000","last_clean":"2024-08-17T17:37:34.449866+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:37:34.449866+0000","last_undegraded":"2024-08-17T17:37:34.449866+0000","last_fullsized":"2024-08-17T17:37:34.449866+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":338,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307470+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:37:24.307470+0000","last_peered":"2024-08-17T17:37:24.307470+0000","last_clean":"2024-08-17T17:37:24.307470+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:37:24.307470+0000","last_undegraded":"2024-08-17T17:37:24.307470+0000","last_fullsized":"2024-08-17T17:37:24.307470+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":197,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452567+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:37:34.452567+0000","last_peered":"2024-08-17T17:37:34.452567+0000","last_clean":"2024-08-17T17:37:34.452567+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:37:34.452567+0000","last_undegraded":"2024-08-17T17:37:34.452567+0000","last_fullsized":"2024-08-17T17:37:34.452567+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"363'9057","reported_seq":24007,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:35.428149+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:37:35.428149+0000","last_peered":"2024-08-17T17:37:35.428149+0000","last_clean":"2024-08-17T17:37:35.428149+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:37:35.428149+0000","last_undegraded":"2024-08-17T17:37:35.428149+0000","last_fullsized":"2024-08-17T17:37:35.428149+0000","mapping_epoch":355,"log_start":"325'2730","ondisk_log_start":"325'2730","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378732544,"num_objects":5779,"num_object_clones":0,"num_object_copies":17337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9057,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":277,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.334940+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:37:24.334940+0000","last_peered":"2024-08-17T17:37:24.334940+0000","last_clean":"2024-08-17T17:37:24.334940+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:37:24.334940+0000","last_undegraded":"2024-08-17T17:37:24.334940+0000","last_fullsized":"2024-08-17T17:37:24.334940+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":65,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452483+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:37:34.452483+0000","last_peered":"2024-08-17T17:37:34.452483+0000","last_clean":"2024-08-17T17:37:34.452483+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:37:34.452483+0000","last_undegraded":"2024-08-17T17:37:34.452483+0000","last_fullsized":"2024-08-17T17:37:34.452483+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":561,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.450135+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:37:34.450135+0000","last_peered":"2024-08-17T17:37:34.450135+0000","last_clean":"2024-08-17T17:37:34.450135+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:37:34.450135+0000","last_undegraded":"2024-08-17T17:37:34.450135+0000","last_fullsized":"2024-08-17T17:37:34.450135+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"363'8783","reported_seq":23301,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:35.415973+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:37:35.415973+0000","last_peered":"2024-08-17T17:37:35.415973+0000","last_clean":"2024-08-17T17:37:35.415973+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:37:35.415973+0000","last_undegraded":"2024-08-17T17:37:35.415973+0000","last_fullsized":"2024-08-17T17:37:35.415973+0000","mapping_epoch":345,"log_start":"324'2472","ondisk_log_start":"324'2472","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6311,"log_dups_size":0,"ondisk_log_size":6311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368377856,"num_objects":5621,"num_object_clones":0,"num_object_copies":16863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8783,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":273,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.335285+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:37:24.335285+0000","last_peered":"2024-08-17T17:37:24.335285+0000","last_clean":"2024-08-17T17:37:24.335285+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:37:24.335285+0000","last_undegraded":"2024-08-17T17:37:24.335285+0000","last_fullsized":"2024-08-17T17:37:24.335285+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":316,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.504402+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:37:34.504402+0000","last_peered":"2024-08-17T17:37:34.504402+0000","last_clean":"2024-08-17T17:37:34.504402+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:37:34.504402+0000","last_undegraded":"2024-08-17T17:37:34.504402+0000","last_fullsized":"2024-08-17T17:37:34.504402+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":415,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.334846+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:37:24.334846+0000","last_peered":"2024-08-17T17:37:24.334846+0000","last_clean":"2024-08-17T17:37:24.334846+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:37:24.334846+0000","last_undegraded":"2024-08-17T17:37:24.334846+0000","last_fullsized":"2024-08-17T17:37:24.334846+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"363'9024","reported_seq":9150,"reported_epoch":363,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:35.473156+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:37:35.473156+0000","last_peered":"2024-08-17T17:37:35.473156+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:37:35.473156+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'2687","ondisk_log_start":"325'2687","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377487360,"num_objects":5760,"num_object_clones":0,"num_object_copies":17280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9024,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5760},{"shards":"0,3","objects":-1488}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":591,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.453559+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:37:34.453559+0000","last_peered":"2024-08-17T17:37:34.453559+0000","last_clean":"2024-08-17T17:37:34.453559+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:37:34.453559+0000","last_undegraded":"2024-08-17T17:37:34.453559+0000","last_fullsized":"2024-08-17T17:37:34.453559+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":559,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.450277+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:37:34.450277+0000","last_peered":"2024-08-17T17:37:34.450277+0000","last_clean":"2024-08-17T17:37:34.450277+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:37:34.450277+0000","last_undegraded":"2024-08-17T17:37:34.450277+0000","last_fullsized":"2024-08-17T17:37:34.450277+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":577,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.451635+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:37:34.451635+0000","last_peered":"2024-08-17T17:37:34.451635+0000","last_clean":"2024-08-17T17:37:34.451635+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:37:34.451635+0000","last_undegraded":"2024-08-17T17:37:34.451635+0000","last_fullsized":"2024-08-17T17:37:34.451635+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"363'8921","reported_seq":23581,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:37.018981+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:37:37.018981+0000","last_peered":"2024-08-17T17:37:37.018981+0000","last_clean":"2024-08-17T17:37:37.018981+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:37:37.018981+0000","last_undegraded":"2024-08-17T17:37:37.018981+0000","last_fullsized":"2024-08-17T17:37:37.018981+0000","mapping_epoch":347,"log_start":"324'2372","ondisk_log_start":"324'2372","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370475008,"num_objects":5653,"num_object_clones":0,"num_object_copies":16959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8921,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":250,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.504424+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:37:34.504424+0000","last_peered":"2024-08-17T17:37:34.504424+0000","last_clean":"2024-08-17T17:37:34.504424+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:37:34.504424+0000","last_undegraded":"2024-08-17T17:37:34.504424+0000","last_fullsized":"2024-08-17T17:37:34.504424+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":367,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.315478+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:37:24.315478+0000","last_peered":"2024-08-17T17:37:24.315478+0000","last_clean":"2024-08-17T17:37:24.315478+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:37:24.315478+0000","last_undegraded":"2024-08-17T17:37:24.315478+0000","last_fullsized":"2024-08-17T17:37:24.315478+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":317,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306846+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:37:24.306846+0000","last_peered":"2024-08-17T17:37:24.306846+0000","last_clean":"2024-08-17T17:37:24.306846+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:37:24.306846+0000","last_undegraded":"2024-08-17T17:37:24.306846+0000","last_fullsized":"2024-08-17T17:37:24.306846+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":317,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307333+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:37:24.307333+0000","last_peered":"2024-08-17T17:37:24.307333+0000","last_clean":"2024-08-17T17:37:24.307333+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:37:24.307333+0000","last_undegraded":"2024-08-17T17:37:24.307333+0000","last_fullsized":"2024-08-17T17:37:24.307333+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":360,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307496+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:37:24.307496+0000","last_peered":"2024-08-17T17:37:24.307496+0000","last_clean":"2024-08-17T17:37:24.307496+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:37:24.307496+0000","last_undegraded":"2024-08-17T17:37:24.307496+0000","last_fullsized":"2024-08-17T17:37:24.307496+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":566,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.449950+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:37:34.449950+0000","last_peered":"2024-08-17T17:37:34.449950+0000","last_clean":"2024-08-17T17:37:34.449950+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:37:34.449950+0000","last_undegraded":"2024-08-17T17:37:34.449950+0000","last_fullsized":"2024-08-17T17:37:34.449950+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":106,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452985+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:37:34.452985+0000","last_peered":"2024-08-17T17:37:34.452985+0000","last_clean":"2024-08-17T17:37:34.452985+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:37:34.452985+0000","last_undegraded":"2024-08-17T17:37:34.452985+0000","last_fullsized":"2024-08-17T17:37:34.452985+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":267,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.335279+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:37:24.335279+0000","last_peered":"2024-08-17T17:37:24.335279+0000","last_clean":"2024-08-17T17:37:24.335279+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:37:24.335279+0000","last_undegraded":"2024-08-17T17:37:24.335279+0000","last_fullsized":"2024-08-17T17:37:24.335279+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":524,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.451634+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:37:34.451634+0000","last_peered":"2024-08-17T17:37:34.451634+0000","last_clean":"2024-08-17T17:37:34.451634+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:37:34.451634+0000","last_undegraded":"2024-08-17T17:37:34.451634+0000","last_fullsized":"2024-08-17T17:37:34.451634+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":411,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.335651+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:37:24.335651+0000","last_peered":"2024-08-17T17:37:24.335651+0000","last_clean":"2024-08-17T17:37:24.335651+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:37:24.335651+0000","last_undegraded":"2024-08-17T17:37:24.335651+0000","last_fullsized":"2024-08-17T17:37:24.335651+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":518,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.451536+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:37:34.451536+0000","last_peered":"2024-08-17T17:37:34.451536+0000","last_clean":"2024-08-17T17:37:34.451536+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:37:34.451536+0000","last_undegraded":"2024-08-17T17:37:34.451536+0000","last_fullsized":"2024-08-17T17:37:34.451536+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":334,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.315415+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:37:24.315415+0000","last_peered":"2024-08-17T17:37:24.315415+0000","last_clean":"2024-08-17T17:37:24.315415+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:37:24.315415+0000","last_undegraded":"2024-08-17T17:37:24.315415+0000","last_fullsized":"2024-08-17T17:37:24.315415+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":550,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.451822+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:37:34.451822+0000","last_peered":"2024-08-17T17:37:34.451822+0000","last_clean":"2024-08-17T17:37:34.451822+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:37:34.451822+0000","last_undegraded":"2024-08-17T17:37:34.451822+0000","last_fullsized":"2024-08-17T17:37:34.451822+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":265,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.453297+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:37:34.453297+0000","last_peered":"2024-08-17T17:37:34.453297+0000","last_clean":"2024-08-17T17:37:34.453297+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:37:34.453297+0000","last_undegraded":"2024-08-17T17:37:34.453297+0000","last_fullsized":"2024-08-17T17:37:34.453297+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":558,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.449662+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:37:34.449662+0000","last_peered":"2024-08-17T17:37:34.449662+0000","last_clean":"2024-08-17T17:37:34.449662+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:37:34.449662+0000","last_undegraded":"2024-08-17T17:37:34.449662+0000","last_fullsized":"2024-08-17T17:37:34.449662+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":517,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452737+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:37:34.452737+0000","last_peered":"2024-08-17T17:37:34.452737+0000","last_clean":"2024-08-17T17:37:34.452737+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:37:34.452737+0000","last_undegraded":"2024-08-17T17:37:34.452737+0000","last_fullsized":"2024-08-17T17:37:34.452737+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":346,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307067+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:37:24.307067+0000","last_peered":"2024-08-17T17:37:24.307067+0000","last_clean":"2024-08-17T17:37:24.307067+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:37:24.307067+0000","last_undegraded":"2024-08-17T17:37:24.307067+0000","last_fullsized":"2024-08-17T17:37:24.307067+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":206,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452680+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:37:34.452680+0000","last_peered":"2024-08-17T17:37:34.452680+0000","last_clean":"2024-08-17T17:37:34.452680+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:37:34.452680+0000","last_undegraded":"2024-08-17T17:37:34.452680+0000","last_fullsized":"2024-08-17T17:37:34.452680+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":263,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.504221+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:37:34.504221+0000","last_peered":"2024-08-17T17:37:34.504221+0000","last_clean":"2024-08-17T17:37:34.504221+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:37:34.504221+0000","last_undegraded":"2024-08-17T17:37:34.504221+0000","last_fullsized":"2024-08-17T17:37:34.504221+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":354,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307265+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:37:24.307265+0000","last_peered":"2024-08-17T17:37:24.307265+0000","last_clean":"2024-08-17T17:37:24.307265+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:37:24.307265+0000","last_undegraded":"2024-08-17T17:37:24.307265+0000","last_fullsized":"2024-08-17T17:37:24.307265+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":217,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307419+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:37:24.307419+0000","last_peered":"2024-08-17T17:37:24.307419+0000","last_clean":"2024-08-17T17:37:24.307419+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:37:24.307419+0000","last_undegraded":"2024-08-17T17:37:24.307419+0000","last_fullsized":"2024-08-17T17:37:24.307419+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":354,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306937+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:37:24.306937+0000","last_peered":"2024-08-17T17:37:24.306937+0000","last_clean":"2024-08-17T17:37:24.306937+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:37:24.306937+0000","last_undegraded":"2024-08-17T17:37:24.306937+0000","last_fullsized":"2024-08-17T17:37:24.306937+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":217,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.307774+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:37:24.307774+0000","last_peered":"2024-08-17T17:37:24.307774+0000","last_clean":"2024-08-17T17:37:24.307774+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:37:24.307774+0000","last_undegraded":"2024-08-17T17:37:24.307774+0000","last_fullsized":"2024-08-17T17:37:24.307774+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":570,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.452059+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:37:34.452059+0000","last_peered":"2024-08-17T17:37:34.452059+0000","last_clean":"2024-08-17T17:37:34.452059+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:37:34.452059+0000","last_undegraded":"2024-08-17T17:37:34.452059+0000","last_fullsized":"2024-08-17T17:37:34.452059+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":543,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.453397+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:37:34.453397+0000","last_peered":"2024-08-17T17:37:34.453397+0000","last_clean":"2024-08-17T17:37:34.453397+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:37:34.453397+0000","last_undegraded":"2024-08-17T17:37:34.453397+0000","last_fullsized":"2024-08-17T17:37:34.453397+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":317,"reported_epoch":362,"state":"active+clean","last_fresh":"2024-08-17T17:37:24.306591+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:37:24.306591+0000","last_peered":"2024-08-17T17:37:24.306591+0000","last_clean":"2024-08-17T17:37:24.306591+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:37:24.306591+0000","last_undegraded":"2024-08-17T17:37:24.306591+0000","last_fullsized":"2024-08-17T17:37:24.306591+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":265,"reported_epoch":363,"state":"active+clean","last_fresh":"2024-08-17T17:37:34.504424+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:37:34.504424+0000","last_peered":"2024-08-17T17:37:34.504424+0000","last_clean":"2024-08-17T17:37:34.504424+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:37:34.504424+0000","last_undegraded":"2024-08-17T17:37:34.504424+0000","last_fullsized":"2024-08-17T17:37:34.504424+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6047334424,"num_objects":92276,"num_object_clones":0,"num_object_copies":276828,"num_objects_missing_on_primary":6391,"num_objects_missing":6391,"num_objects_degraded":55940,"num_objects_misplaced":36182,"num_objects_unfound":0,"num_objects_dirty":92276,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144114,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25002,"num_bytes_recovered":1515978752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19623456768,"data_stored":19623444552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104833,"ondisk_log_size":104833,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170054,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463424,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3479948,"kb_used_data":2582080,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":90265204,"statfs":{"total":95995035648,"available":92431568896,"internally_reserved":0,"allocated":2644049920,"data_stored":2642120822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492098,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1137580,"kb_used_data":742244,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92607572,"statfs":{"total":95995035648,"available":94830153728,"internally_reserved":0,"allocated":760057856,"data_stored":758151058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986690,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2364476,"kb_used_data":1467700,"kb_used_omap":1,"kb_used_meta":896766,"kb_avail":91380676,"statfs":{"total":95995035648,"available":93573812224,"internally_reserved":0,"allocated":1502924800,"data_stored":1501006642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":918288728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,1,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225517,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3864880,"kb_used_data":2953316,"kb_used_omap":4,"kb_used_meta":911547,"kb_avail":89880272,"statfs":{"total":95995035648,"available":92037398528,"internally_reserved":0,"allocated":3024195584,"data_stored":3022289084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":933424576},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075842,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533735,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3637360,"kb_used_data":2591208,"kb_used_omap":5,"kb_used_meta":1046138,"kb_avail":90107792,"statfs":{"total":95995035648,"available":92270379008,"internally_reserved":0,"allocated":2653396992,"data_stored":2651485170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1071245552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089712,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3212556,"kb_used_data":2229696,"kb_used_omap":5,"kb_used_meta":982842,"kb_avail":90532596,"statfs":{"total":95995035648,"available":92705378304,"internally_reserved":0,"allocated":2283208704,"data_stored":2281279665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1006430267},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2277638144,"data_stored":2277638144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2648248320,"data_stored":2648244248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4006481920,"data_stored":4006477848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3017670656,"data_stored":3017670656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1498021888,"data_stored":1498021888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":755236864,"data_stored":755236864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2637561856,"data_stored":2637561856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2782597120,"data_stored":2782593048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:37:39.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:37:39.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:39.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:39.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:40.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:40.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:40.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:40.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:40.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:40.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:40.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:40.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:40.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:40.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:40.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:40.560 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:40.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:40 smithi152 ceph-mon[93456]: from='client.37242 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:37:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:37:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:40.595 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:40.629 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:40.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:40.684 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:40.719 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:40 smithi142 ceph-mon[106729]: from='client.37242 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:40.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:40 smithi142 ceph-mon[103614]: from='client.37242 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:37:40.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:40.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:40.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:40.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:41.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:41.155 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:41.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:41.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:41.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:41.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:41.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-mon[103614]: pgmap v1316: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 55943/276696 objects degraded (20.218%); 36182/276696 objects misplaced (13.076%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55943/276696 objects degraded (20.218%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-mon[106729]: pgmap v1316: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 55943/276696 objects degraded (20.218%); 36182/276696 objects misplaced (13.076%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55943/276696 objects degraded (20.218%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:41.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:41.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:41.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:41.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:41 smithi152 ceph-mon[93456]: pgmap v1316: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 55943/276696 objects degraded (20.218%); 36182/276696 objects misplaced (13.076%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:37:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55943/276696 objects degraded (20.218%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:41.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:41.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:42.029 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:42.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:42.101 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:37:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:37:41] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:37:42.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:42.174 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:42.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:42.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:42.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:42.450 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:42.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:42.533 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:42.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:42.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:42.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:42.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:42.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:42.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:42 smithi152 ceph-mon[93456]: pgmap v1317: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275592 objects degraded (20.278%); 35997/275592 objects misplaced (13.062%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:42.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:42 smithi142 ceph-mon[106729]: pgmap v1317: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275592 objects degraded (20.278%); 35997/275592 objects misplaced (13.062%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:42.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:42 smithi142 ceph-mon[103614]: pgmap v1317: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275592 objects degraded (20.278%); 35997/275592 objects misplaced (13.062%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:42.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:42.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:42.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:43.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:43.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:43.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:43.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:43.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:43.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:43.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:43.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:43.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:43.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:43.550 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:43.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:43.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:43.695 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:43.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:43.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:43.864 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:37:43.864 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:37:43.864 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:37:43.864 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:37:43.864 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:37:43.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:43.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:43.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:44.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:44.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:44.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:44.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:44.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:44.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:44.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:44.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:44.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:44.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:44.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:44.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:44.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:44.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:44.668 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:44.690 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:44.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:44.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:44.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:44.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:37:44.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:44.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:44.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:45.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:45.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:45.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:45.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:45.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:45.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:45.309 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:45.340 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:37:45.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:45.417 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:45 smithi142 ceph-mon[106729]: pgmap v1318: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275592 objects degraded (20.278%); 35997/275592 objects misplaced (13.062%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:45.417 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2911868622' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:45.417 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:45 smithi142 ceph-mon[103614]: pgmap v1318: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275592 objects degraded (20.278%); 35997/275592 objects misplaced (13.062%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:45.417 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2911868622' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:45.433 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:45.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:45.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:45.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:45.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:45.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:45.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:45 smithi152 ceph-mon[93456]: pgmap v1318: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275592 objects degraded (20.278%); 35997/275592 objects misplaced (13.062%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2911868622' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:45.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:45.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:46.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:46.053 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:46.155 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:46.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:46.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:46.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:46.300 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:46.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:46.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:46.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:46.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2911868622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[106729]: osdmap e364: 8 total, 8 up, 6 in 2024-08-17T17:37:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55884/275592 objects degraded (20.278%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2911868622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[103614]: osdmap e364: 8 total, 8 up, 6 in 2024-08-17T17:37:46.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55884/275592 objects degraded (20.278%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:46.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:46.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:46.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:46.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:46 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2911868622' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:46 smithi152 ceph-mon[93456]: osdmap e364: 8 total, 8 up, 6 in 2024-08-17T17:37:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55884/275592 objects degraded (20.278%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:46.899 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:46.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:47.023 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:47.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:47.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:47.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:47.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:47.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:47.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:47.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:47.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:47.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:47.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:47.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:47.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:47.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:47 smithi142 ceph-mon[106729]: pgmap v1320: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275145 objects degraded (20.311%); 35997/275145 objects misplaced (13.083%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2591982075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:47.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:47 smithi142 ceph-mon[103614]: pgmap v1320: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275145 objects degraded (20.311%); 35997/275145 objects misplaced (13.083%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:47.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2591982075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:47.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:47.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:47.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:47.783 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:37:47.783 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:37:47.783 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:37:47.783 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:37:47.783 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:47.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:47.785 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:37:47.785 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:37:47.785 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:47.785 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:47.791 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:47 smithi152 ceph-mon[93456]: pgmap v1320: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55884/275145 objects degraded (20.311%); 35997/275145 objects misplaced (13.083%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:37:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2591982075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:47.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:47.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:47.982 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:48.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:48.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:48.162 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:48.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:48.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:48.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:48.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:48.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:48.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:48.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:48.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:48.660 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:48.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:48.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:37:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:48.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:48.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:48.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:49.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:49.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:49.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:49.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:49.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:49.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:37:49.282 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 55827/274416 objects degraded (20.344%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1949,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":364,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":91472,"data_bytes":5993535461,"bytes_used":18005630976,"bytes_avail":557964582912,"bytes_total":575970213888,"degraded_objects":55827,"degraded_total":274416,"degraded_ratio":0.20343930383068043,"misplaced_objects":35848,"misplaced_total":274416,"misplaced_ratio":0.13063378228674713,"recovering_objects_per_sec":17,"recovering_bytes_per_sec":1120337,"recovering_keys_per_sec":0,"num_objects_recovered":171,"num_bytes_recovered":11206656,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":82},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":32,"modified":"2024-08-17T17:37:30.276395+0000","services":{"osd":{"daemons":{"summary":"","0":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:37:49.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:49.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:49.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:49.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:49.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:49.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:49.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:49.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:49 smithi142 ceph-mon[106729]: pgmap v1321: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55827/274416 objects degraded (20.344%); 35848/274416 objects misplaced (13.063%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:49.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/469937862' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:37:49.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:49 smithi142 ceph-mon[103614]: pgmap v1321: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55827/274416 objects degraded (20.344%); 35848/274416 objects misplaced (13.063%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:49.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/469937862' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:37:49.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:49.660 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:49.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:49.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:49.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:49.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:49 smithi152 ceph-mon[93456]: pgmap v1321: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 55827/274416 objects degraded (20.344%); 35848/274416 objects misplaced (13.063%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/469937862' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:37:49.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:49.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:49.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:50.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:50.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:50.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:50.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:50.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:50.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:50.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:50.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:50.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:50.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:50.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:50.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:37:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:37:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:50.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:50.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:50.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:50.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:50.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:50.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:50.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:51.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:51.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:51.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:51.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:51.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:51.224 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:37:51.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:37:51.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:51.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:51.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:51.468 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:51 smithi142 ceph-mon[106729]: pgmap v1322: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55816/274077 objects degraded (20.365%); 35827/274077 objects misplaced (13.072%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:51.468 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55816/274077 objects degraded (20.365%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:51.468 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:51 smithi142 ceph-mon[103614]: pgmap v1322: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55816/274077 objects degraded (20.365%); 35827/274077 objects misplaced (13.072%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:51.469 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55816/274077 objects degraded (20.365%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:51.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:51.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:51.714 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:51.714 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:51.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:51.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:51.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:51 smithi152 ceph-mon[93456]: pgmap v1322: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55816/274077 objects degraded (20.365%); 35827/274077 objects misplaced (13.072%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:37:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55816/274077 objects degraded (20.365%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:51.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:51.913 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:52.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:52.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:52.048 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:37:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:37:51] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:37:52.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:52.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:52.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:52.262 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:52.284 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:52.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:52.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:52.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:52.510 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:52.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:37:52.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:52.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:52.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:52.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:52.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:52.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:52.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:53.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:53.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:53.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:53.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:53.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:53.275 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:53.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:53.398 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:53.496 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:53.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:53.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:53.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:53 smithi142 ceph-mon[106729]: pgmap v1323: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 55785/273174 objects degraded (20.421%); 35712/273174 objects misplaced (13.073%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:37:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:53 smithi142 ceph-mon[103614]: pgmap v1323: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 55785/273174 objects degraded (20.421%); 35712/273174 objects misplaced (13.073%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:37:53.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:53.653 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:53.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:53.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:53 smithi152 ceph-mon[93456]: pgmap v1323: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 55785/273174 objects degraded (20.421%); 35712/273174 objects misplaced (13.073%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:37:53.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:53.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:53.991 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:54.006 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:54.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:54.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:54.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:54.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:54.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:54.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:54.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:54.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:54.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:54.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:54.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:54.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:54.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:54.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:54.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:54.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:54.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:54.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:55.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:55.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:55.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:55.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:55.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:55.273 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:55.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:55.375 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:55.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:55.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:55.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:55 smithi142 ceph-mon[106729]: pgmap v1324: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55752/272973 objects degraded (20.424%); 35645/272973 objects misplaced (13.058%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:55 smithi142 ceph-mon[103614]: pgmap v1324: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55752/272973 objects degraded (20.424%); 35645/272973 objects misplaced (13.058%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:55.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:55.650 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:55.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:55.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:55 smithi152 ceph-mon[93456]: pgmap v1324: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 55752/272973 objects degraded (20.424%); 35645/272973 objects misplaced (13.058%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:37:55.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:55.896 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:55.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:55.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:56.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:56.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:56.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:56.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:56.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:56.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:56.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:56.359 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:37:56.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:56.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:56.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:56.524 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1511349434' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55752/272973 objects degraded (20.424%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1511349434' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55752/272973 objects degraded (20.424%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:56.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:56.650 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:56.752 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:56.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1511349434' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55752/272973 objects degraded (20.424%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:56.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:56.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:56.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:57.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:57.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:57.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:57.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.270 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:57.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:57.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:57.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:57.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:57.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:57.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[106729]: pgmap v1325: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55752/272430 objects degraded (20.465%); 35645/272430 objects misplaced (13.084%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1511349434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[106729]: osdmap e365: 8 total, 8 up, 6 in 2024-08-17T17:37:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[103614]: pgmap v1325: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55752/272430 objects degraded (20.465%); 35645/272430 objects misplaced (13.084%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1511349434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[103614]: osdmap e365: 8 total, 8 up, 6 in 2024-08-17T17:37:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:57.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:57.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:57.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:57 smithi152 ceph-mon[93456]: pgmap v1325: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55752/272430 objects degraded (20.465%); 35645/272430 objects misplaced (13.084%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:37:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:57 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1511349434' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:57 smithi152 ceph-mon[93456]: osdmap e365: 8 total, 8 up, 6 in 2024-08-17T17:37:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:57.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:57.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:57.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:58.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:58.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:58.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:58.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:58.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:58.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:58.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:58.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:58.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:58.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:58.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:58.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:58.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:58.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:58.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:58.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:58.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:58.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:59.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:59.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:37:59.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:59.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:59.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:59.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:59.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:59.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:59.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:59.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:59.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:59.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:37:59 smithi142 ceph-mon[103614]: pgmap v1327: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 55723/271581 objects degraded (20.518%); 35514/271581 objects misplaced (13.077%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:37:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:37:59 smithi142 ceph-mon[106729]: pgmap v1327: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 55723/271581 objects degraded (20.518%); 35514/271581 objects misplaced (13.077%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:37:59.624 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:37:59 smithi152 ceph-mon[93456]: pgmap v1327: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 55723/271581 objects degraded (20.518%); 35514/271581 objects misplaced (13.077%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:37:59.658 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:37:59.658 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:59.672 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:37:59.672 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:59.689 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:37:59.689 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:59.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:59.707 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:37:59.707 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:59.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:59.724 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:37:59.724 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:59.742 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:37:59.742 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:59.763 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:37:59.763 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:59.781 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:37:59.781 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:59.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:59.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:37:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:59.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:59.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:37:59.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:00.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:00.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:00.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:00.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:00.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:00.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:00.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:00.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/703601820' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:38:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:38:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:00.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:00.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:00.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/703601820' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:00.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/703601820' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:00.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:00.705 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:38:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:38:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:38:00.706 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:38:00.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:00.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:00.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:00.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:00.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:00.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:00.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:00.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:01.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:01.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:01.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:01.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:01.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:01.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:01.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:01.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:38:01.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:01.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:01.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:01 smithi152 ceph-mon[93456]: pgmap v1328: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55683/270939 objects degraded (20.552%); 35431/270939 objects misplaced (13.077%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55683/270939 objects degraded (20.552%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:01.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:01.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:01 smithi142 ceph-mon[106729]: pgmap v1328: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55683/270939 objects degraded (20.552%); 35431/270939 objects misplaced (13.077%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55683/270939 objects degraded (20.552%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:01.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:01 smithi142 ceph-mon[103614]: pgmap v1328: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55683/270939 objects degraded (20.552%); 35431/270939 objects misplaced (13.077%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55683/270939 objects degraded (20.552%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:01.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:01.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:01.839 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:01.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:01.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:01.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:02.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.098 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:38:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:38:01] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:38:02.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.143 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:02.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:02.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:02.296 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:38:02.298 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:38:02.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:02.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:02.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:02.482 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:02.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:02.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:02.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:02.838 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:02.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:02.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:02.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:02.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:03.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:03.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:03.222 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:03.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:03.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:03.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:03.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:03.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:03 smithi142 ceph-mon[106729]: pgmap v1329: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 55657/269790 objects degraded (20.630%); 35303/269790 objects misplaced (13.085%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:03.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:03 smithi142 ceph-mon[106729]: from='client.37278 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:03 smithi142 ceph-mon[103614]: pgmap v1329: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 55657/269790 objects degraded (20.630%); 35303/269790 objects misplaced (13.085%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:03.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:03 smithi142 ceph-mon[103614]: from='client.37278 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:03.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:03.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:03 smithi152 ceph-mon[93456]: pgmap v1329: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 55657/269790 objects degraded (20.630%); 35303/269790 objects misplaced (13.085%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:03 smithi152 ceph-mon[93456]: from='client.37278 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:03.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:03.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:03.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:03.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:03.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:03.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:04.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:04.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:04.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:04.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:04.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:04.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:04.465 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:04.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:04.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:04.490 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1329,"stamp":"2024-08-17T17:38:02.285365+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5892478949,"num_objects":89930,"num_object_clones":0,"num_object_copies":269790,"num_objects_missing_on_primary":6245,"num_objects_missing":6245,"num_objects_degraded":55657,"num_objects_misplaced":35303,"num_objects_unfound":0,"num_objects_dirty":89930,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":146764,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25484,"num_bytes_recovered":1547248270,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104812,"ondisk_log_size":104812,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17384152,"kb_used_data":12249444,"kb_used_omap":27,"kb_used_meta":5134628,"kb_avail":545086760,"statfs":{"total":575970213888,"available":558168842240,"internally_reserved":0,"allocated":12543430656,"data_stored":12531835725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5257859816},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-73924608,"num_objects":-1128,"num_object_clones":0,"num_object_copies":-3384,"num_objects_missing_on_primary":-58,"num_objects_missing":-58,"num_objects_degraded":-128,"num_objects_misplaced":-409,"num_objects_unfound":0,"num_objects_dirty":-1128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1128,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":196,"num_bytes_recovered":12845056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002859"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":571,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364067+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:37:56.364067+0000","last_peered":"2024-08-17T17:37:56.364067+0000","last_clean":"2024-08-17T17:37:56.364067+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:37:56.364067+0000","last_undegraded":"2024-08-17T17:37:56.364067+0000","last_fullsized":"2024-08-17T17:37:56.364067+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":74,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347223+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:37:45.347223+0000","last_peered":"2024-08-17T17:37:45.347223+0000","last_clean":"2024-08-17T17:37:45.347223+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:37:45.347223+0000","last_undegraded":"2024-08-17T17:37:45.347223+0000","last_fullsized":"2024-08-17T17:37:45.347223+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":268,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.395254+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:37:45.395254+0000","last_peered":"2024-08-17T17:37:45.395254+0000","last_clean":"2024-08-17T17:37:45.395254+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:37:45.395254+0000","last_undegraded":"2024-08-17T17:37:45.395254+0000","last_fullsized":"2024-08-17T17:37:45.395254+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":269,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364716+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:37:56.364716+0000","last_peered":"2024-08-17T17:37:56.364716+0000","last_clean":"2024-08-17T17:37:56.364716+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:37:56.364716+0000","last_undegraded":"2024-08-17T17:37:56.364716+0000","last_fullsized":"2024-08-17T17:37:56.364716+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":446,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364993+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:37:56.364993+0000","last_peered":"2024-08-17T17:37:56.364993+0000","last_clean":"2024-08-17T17:37:56.364993+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:37:56.364993+0000","last_undegraded":"2024-08-17T17:37:56.364993+0000","last_fullsized":"2024-08-17T17:37:56.364993+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":217,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345862+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:37:45.345862+0000","last_peered":"2024-08-17T17:37:45.345862+0000","last_clean":"2024-08-17T17:37:45.345862+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:37:45.345862+0000","last_undegraded":"2024-08-17T17:37:45.345862+0000","last_fullsized":"2024-08-17T17:37:45.345862+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":540,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364508+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:37:56.364508+0000","last_peered":"2024-08-17T17:37:56.364508+0000","last_clean":"2024-08-17T17:37:56.364508+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:37:56.364508+0000","last_undegraded":"2024-08-17T17:37:56.364508+0000","last_fullsized":"2024-08-17T17:37:56.364508+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":385,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346098+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:37:45.346098+0000","last_peered":"2024-08-17T17:37:45.346098+0000","last_clean":"2024-08-17T17:37:45.346098+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:37:45.346098+0000","last_undegraded":"2024-08-17T17:37:45.346098+0000","last_fullsized":"2024-08-17T17:37:45.346098+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"365'9136","reported_seq":11323,"reported_epoch":365,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:38:02.007663+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:38:02.007663+0000","last_peered":"2024-08-17T17:38:02.007663+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:38:02.007663+0000","last_undegraded":"2024-08-17T17:38:02.007663+0000","last_fullsized":"2024-08-17T17:38:02.007663+0000","mapping_epoch":337,"log_start":"324'2263","ondisk_log_start":"324'2263","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6873,"log_dups_size":0,"ondisk_log_size":6873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364904448,"num_objects":5568,"num_object_clones":0,"num_object_copies":16704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5568,"num_objects_unfound":0,"num_objects_dirty":5568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9136,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1055,"num_bytes_recovered":69140480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5568}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":70,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.439943+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:37:45.439943+0000","last_peered":"2024-08-17T17:37:45.439943+0000","last_clean":"2024-08-17T17:37:45.439943+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:37:45.439943+0000","last_undegraded":"2024-08-17T17:37:45.439943+0000","last_fullsized":"2024-08-17T17:37:45.439943+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.394897+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:37:45.394897+0000","last_peered":"2024-08-17T17:37:45.394897+0000","last_clean":"2024-08-17T17:37:45.394897+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:37:45.394897+0000","last_undegraded":"2024-08-17T17:37:45.394897+0000","last_fullsized":"2024-08-17T17:37:45.394897+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"365'9350","reported_seq":9413,"reported_epoch":365,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:00.777375+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:38:00.777375+0000","last_peered":"2024-08-17T17:38:00.777375+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:38:00.777375+0000","last_undegraded":"2024-08-17T17:38:00.777375+0000","last_fullsized":"2024-08-17T17:38:00.777375+0000","mapping_epoch":337,"log_start":"325'2782","ondisk_log_start":"325'2782","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375128064,"num_objects":5724,"num_object_clones":0,"num_object_copies":17172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5724,"num_objects_unfound":0,"num_objects_dirty":5724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9350,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5724}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":72,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347496+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:37:45.347496+0000","last_peered":"2024-08-17T17:37:45.347496+0000","last_clean":"2024-08-17T17:37:45.347496+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:37:45.347496+0000","last_undegraded":"2024-08-17T17:37:45.347496+0000","last_fullsized":"2024-08-17T17:37:45.347496+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":336,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345780+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:37:45.345780+0000","last_peered":"2024-08-17T17:37:45.345780+0000","last_clean":"2024-08-17T17:37:45.345780+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:37:45.345780+0000","last_undegraded":"2024-08-17T17:37:45.345780+0000","last_fullsized":"2024-08-17T17:37:45.345780+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"365'9114","reported_seq":9180,"reported_epoch":365,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:01.968996+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:38:01.968996+0000","last_peered":"2024-08-17T17:38:01.968996+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:38:01.968996+0000","last_undegraded":"2024-08-17T17:38:01.968996+0000","last_fullsized":"2024-08-17T17:38:01.968996+0000","mapping_epoch":337,"log_start":"324'2253","ondisk_log_start":"324'2253","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6861,"log_dups_size":0,"ondisk_log_size":6861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368312320,"num_objects":5620,"num_object_clones":0,"num_object_copies":16860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5620,"num_objects_unfound":0,"num_objects_dirty":5620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9114,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5620}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":68,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.395177+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:37:45.395177+0000","last_peered":"2024-08-17T17:37:45.395177+0000","last_clean":"2024-08-17T17:37:45.395177+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:37:45.395177+0000","last_undegraded":"2024-08-17T17:37:45.395177+0000","last_fullsized":"2024-08-17T17:37:45.395177+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":346,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345908+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:37:45.345908+0000","last_peered":"2024-08-17T17:37:45.345908+0000","last_clean":"2024-08-17T17:37:45.345908+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:37:45.345908+0000","last_undegraded":"2024-08-17T17:37:45.345908+0000","last_fullsized":"2024-08-17T17:37:45.345908+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"365'9239","reported_seq":9348,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:59.835817+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:37:59.835817+0000","last_peered":"2024-08-17T17:37:59.835817+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:37:59.835817+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2213","ondisk_log_start":"324'2213","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374800384,"num_objects":5719,"num_object_clones":0,"num_object_copies":17157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9239,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1050},{"shards":"0,3,7","objects":5719}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":483,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364235+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:37:56.364235+0000","last_peered":"2024-08-17T17:37:56.364235+0000","last_clean":"2024-08-17T17:37:56.364235+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:37:56.364235+0000","last_undegraded":"2024-08-17T17:37:56.364235+0000","last_fullsized":"2024-08-17T17:37:56.364235+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":249,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.397995+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:37:56.397995+0000","last_peered":"2024-08-17T17:37:56.397995+0000","last_clean":"2024-08-17T17:37:56.397995+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:37:56.397995+0000","last_undegraded":"2024-08-17T17:37:56.397995+0000","last_fullsized":"2024-08-17T17:37:56.397995+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"365'9035","reported_seq":9089,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:38:00.802628+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:38:00.802628+0000","last_peered":"2024-08-17T17:38:00.802628+0000","last_clean":"2024-08-17T17:38:00.802628+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:38:00.802628+0000","last_undegraded":"2024-08-17T17:38:00.802628+0000","last_fullsized":"2024-08-17T17:38:00.802628+0000","mapping_epoch":319,"log_start":"324'2474","ondisk_log_start":"324'2474","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364314624,"num_objects":5559,"num_object_clones":0,"num_object_copies":16677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9035,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":573,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365960+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:37:56.365960+0000","last_peered":"2024-08-17T17:37:56.365960+0000","last_clean":"2024-08-17T17:37:56.365960+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:37:56.365960+0000","last_undegraded":"2024-08-17T17:37:56.365960+0000","last_fullsized":"2024-08-17T17:37:56.365960+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":319,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346628+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:37:45.346628+0000","last_peered":"2024-08-17T17:37:45.346628+0000","last_clean":"2024-08-17T17:37:45.346628+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:37:45.346628+0000","last_undegraded":"2024-08-17T17:37:45.346628+0000","last_fullsized":"2024-08-17T17:37:45.346628+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"365'9190","reported_seq":9254,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:01.284075+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:38:01.284075+0000","last_peered":"2024-08-17T17:38:01.284075+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:38:01.284075+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'2635","ondisk_log_start":"325'2635","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376307712,"num_objects":5742,"num_object_clones":0,"num_object_copies":17226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9190,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5742},{"shards":"1,6","objects":-1624}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":320,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346041+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:37:45.346041+0000","last_peered":"2024-08-17T17:37:45.346041+0000","last_clean":"2024-08-17T17:37:45.346041+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:37:45.346041+0000","last_undegraded":"2024-08-17T17:37:45.346041+0000","last_fullsized":"2024-08-17T17:37:45.346041+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":346,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347288+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:37:45.347288+0000","last_peered":"2024-08-17T17:37:45.347288+0000","last_clean":"2024-08-17T17:37:45.347288+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:37:45.347288+0000","last_undegraded":"2024-08-17T17:37:45.347288+0000","last_fullsized":"2024-08-17T17:37:45.347288+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"365'9297","reported_seq":9406,"reported_epoch":365,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:37:58.787524+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:37:58.787524+0000","last_peered":"2024-08-17T17:37:58.787524+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:37:58.787524+0000","last_undegraded":"2024-08-17T17:37:58.787524+0000","last_fullsized":"2024-08-17T17:37:58.787524+0000","mapping_epoch":337,"log_start":"326'3661","ondisk_log_start":"326'3661","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374407168,"num_objects":5713,"num_object_clones":0,"num_object_copies":17139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5713,"num_objects_unfound":0,"num_objects_dirty":5713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9297,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5713}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":357,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346038+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:37:45.346038+0000","last_peered":"2024-08-17T17:37:45.346038+0000","last_clean":"2024-08-17T17:37:45.346038+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:37:45.346038+0000","last_undegraded":"2024-08-17T17:37:45.346038+0000","last_fullsized":"2024-08-17T17:37:45.346038+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":319,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347622+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:37:45.347622+0000","last_peered":"2024-08-17T17:37:45.347622+0000","last_clean":"2024-08-17T17:37:45.347622+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:37:45.347622+0000","last_undegraded":"2024-08-17T17:37:45.347622+0000","last_fullsized":"2024-08-17T17:37:45.347622+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"365'9152","reported_seq":11424,"reported_epoch":365,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:01.924608+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:38:01.924608+0000","last_peered":"2024-08-17T17:38:01.924608+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:38:01.924608+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:38:01.924608+0000","mapping_epoch":337,"log_start":"324'2241","ondisk_log_start":"324'2241","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363855872,"num_objects":5552,"num_object_clones":0,"num_object_copies":16656,"num_objects_missing_on_primary":6245,"num_objects_missing":6245,"num_objects_degraded":5552,"num_objects_misplaced":6245,"num_objects_unfound":0,"num_objects_dirty":5552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9152,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1094,"num_bytes_recovered":71630848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":5539}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":542,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.366771+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:37:56.366771+0000","last_peered":"2024-08-17T17:37:56.366771+0000","last_clean":"2024-08-17T17:37:56.366771+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:37:56.366771+0000","last_undegraded":"2024-08-17T17:37:56.366771+0000","last_fullsized":"2024-08-17T17:37:56.366771+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":479,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364595+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:37:56.364595+0000","last_peered":"2024-08-17T17:37:56.364595+0000","last_clean":"2024-08-17T17:37:56.364595+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:37:56.364595+0000","last_undegraded":"2024-08-17T17:37:56.364595+0000","last_fullsized":"2024-08-17T17:37:56.364595+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"365'9213","reported_seq":11341,"reported_epoch":365,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:37:58.870806+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:37:58.870806+0000","last_peered":"2024-08-17T17:37:58.870806+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:37:58.870806+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:37:58.870806+0000","mapping_epoch":337,"log_start":"326'3644","ondisk_log_start":"326'3644","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5569,"log_dups_size":0,"ondisk_log_size":5569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372375576,"num_objects":5683,"num_object_clones":0,"num_object_copies":17049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5683,"num_objects_misplaced":6433,"num_objects_unfound":0,"num_objects_dirty":5683,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9213,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1015,"num_bytes_recovered":66453504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5683}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":673,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:56.395802+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:37:56.395802+0000","last_peered":"2024-08-17T17:37:56.395802+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:37:56.395802+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":294,"reported_epoch":364,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:37:45.395157+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:37:45.395157+0000","last_peered":"2024-08-17T17:37:45.395157+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:37:45.395157+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:37:45.395157+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":522,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364628+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:37:56.364628+0000","last_peered":"2024-08-17T17:37:56.364628+0000","last_clean":"2024-08-17T17:37:56.364628+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:37:56.364628+0000","last_undegraded":"2024-08-17T17:37:56.364628+0000","last_fullsized":"2024-08-17T17:37:56.364628+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":329,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.357607+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:37:45.357607+0000","last_peered":"2024-08-17T17:37:45.357607+0000","last_clean":"2024-08-17T17:37:45.357607+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:37:45.357607+0000","last_undegraded":"2024-08-17T17:37:45.357607+0000","last_fullsized":"2024-08-17T17:37:45.357607+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"365'9150","reported_seq":9261,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:59.807212+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:37:59.807212+0000","last_peered":"2024-08-17T17:37:59.807212+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:37:59.807212+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2121","ondisk_log_start":"324'2121","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366608384,"num_objects":5594,"num_object_clones":0,"num_object_copies":16782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9150,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1053},{"shards":"0,4,7","objects":5594}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":338,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364322+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:37:56.364322+0000","last_peered":"2024-08-17T17:37:56.364322+0000","last_clean":"2024-08-17T17:37:56.364322+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:37:56.364322+0000","last_undegraded":"2024-08-17T17:37:56.364322+0000","last_fullsized":"2024-08-17T17:37:56.364322+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":301,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345650+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:37:45.345650+0000","last_peered":"2024-08-17T17:37:45.345650+0000","last_clean":"2024-08-17T17:37:45.345650+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:37:45.345650+0000","last_undegraded":"2024-08-17T17:37:45.345650+0000","last_fullsized":"2024-08-17T17:37:45.345650+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":264,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.398204+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:37:56.398204+0000","last_peered":"2024-08-17T17:37:56.398204+0000","last_clean":"2024-08-17T17:37:56.398204+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:37:56.398204+0000","last_undegraded":"2024-08-17T17:37:56.398204+0000","last_fullsized":"2024-08-17T17:37:56.398204+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"365'9142","reported_seq":9206,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:01.942849+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:38:01.942849+0000","last_peered":"2024-08-17T17:38:01.942849+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:38:01.942849+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'2617","ondisk_log_start":"325'2617","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9142,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5654},{"shards":"5,6","objects":-1642}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":430,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364237+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:37:56.364237+0000","last_peered":"2024-08-17T17:37:56.364237+0000","last_clean":"2024-08-17T17:37:56.364237+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:37:56.364237+0000","last_undegraded":"2024-08-17T17:37:56.364237+0000","last_fullsized":"2024-08-17T17:37:56.364237+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345923+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:37:45.345923+0000","last_peered":"2024-08-17T17:37:45.345923+0000","last_clean":"2024-08-17T17:37:45.345923+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:37:45.345923+0000","last_undegraded":"2024-08-17T17:37:45.345923+0000","last_fullsized":"2024-08-17T17:37:45.345923+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":614,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364399+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:37:56.364399+0000","last_peered":"2024-08-17T17:37:56.364399+0000","last_clean":"2024-08-17T17:37:56.364399+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:37:56.364399+0000","last_undegraded":"2024-08-17T17:37:56.364399+0000","last_fullsized":"2024-08-17T17:37:56.364399+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"365'9070","reported_seq":9905,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:59.696164+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:37:59.696164+0000","last_peered":"2024-08-17T17:37:59.696164+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:37:59.696164+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2016","ondisk_log_start":"324'2016","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365428736,"num_objects":5576,"num_object_clones":0,"num_object_copies":16728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9070,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5576},{"shards":"0,5","objects":-1078}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":406,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365383+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:37:56.365383+0000","last_peered":"2024-08-17T17:37:56.365383+0000","last_clean":"2024-08-17T17:37:56.365383+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:37:56.365383+0000","last_undegraded":"2024-08-17T17:37:56.365383+0000","last_fullsized":"2024-08-17T17:37:56.365383+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":340,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347571+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:37:45.347571+0000","last_peered":"2024-08-17T17:37:45.347571+0000","last_clean":"2024-08-17T17:37:45.347571+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:37:45.347571+0000","last_undegraded":"2024-08-17T17:37:45.347571+0000","last_fullsized":"2024-08-17T17:37:45.347571+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":201,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.363966+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:37:56.363966+0000","last_peered":"2024-08-17T17:37:56.363966+0000","last_clean":"2024-08-17T17:37:56.363966+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:37:56.363966+0000","last_undegraded":"2024-08-17T17:37:56.363966+0000","last_fullsized":"2024-08-17T17:37:56.363966+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"365'9202","reported_seq":24157,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:38:00.199245+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:38:00.199245+0000","last_peered":"2024-08-17T17:38:00.199245+0000","last_clean":"2024-08-17T17:38:00.199245+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:38:00.199245+0000","last_undegraded":"2024-08-17T17:38:00.199245+0000","last_fullsized":"2024-08-17T17:38:00.199245+0000","mapping_epoch":355,"log_start":"325'2930","ondisk_log_start":"325'2930","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6272,"log_dups_size":0,"ondisk_log_size":6272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369229824,"num_objects":5634,"num_object_clones":0,"num_object_copies":16902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9202,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":279,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.395091+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:37:45.395091+0000","last_peered":"2024-08-17T17:37:45.395091+0000","last_clean":"2024-08-17T17:37:45.395091+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:37:45.395091+0000","last_undegraded":"2024-08-17T17:37:45.395091+0000","last_fullsized":"2024-08-17T17:37:45.395091+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.363962+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:37:56.363962+0000","last_peered":"2024-08-17T17:37:56.363962+0000","last_clean":"2024-08-17T17:37:56.363962+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:37:56.363962+0000","last_undegraded":"2024-08-17T17:37:56.363962+0000","last_fullsized":"2024-08-17T17:37:56.363962+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":565,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365665+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:37:56.365665+0000","last_peered":"2024-08-17T17:37:56.365665+0000","last_clean":"2024-08-17T17:37:56.365665+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:37:56.365665+0000","last_undegraded":"2024-08-17T17:37:56.365665+0000","last_fullsized":"2024-08-17T17:37:56.365665+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"365'8934","reported_seq":23456,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:38:00.102591+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:38:00.102591+0000","last_peered":"2024-08-17T17:38:00.102591+0000","last_clean":"2024-08-17T17:38:00.102591+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:38:00.102591+0000","last_undegraded":"2024-08-17T17:38:00.102591+0000","last_fullsized":"2024-08-17T17:38:00.102591+0000","mapping_epoch":345,"log_start":"325'2672","ondisk_log_start":"325'2672","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6262,"log_dups_size":0,"ondisk_log_size":6262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358481920,"num_objects":5470,"num_object_clones":0,"num_object_copies":16410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8934,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":275,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.395638+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:37:45.395638+0000","last_peered":"2024-08-17T17:37:45.395638+0000","last_clean":"2024-08-17T17:37:45.395638+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:37:45.395638+0000","last_undegraded":"2024-08-17T17:37:45.395638+0000","last_fullsized":"2024-08-17T17:37:45.395638+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":321,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.396375+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:37:56.396375+0000","last_peered":"2024-08-17T17:37:56.396375+0000","last_clean":"2024-08-17T17:37:56.396375+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:37:56.396375+0000","last_undegraded":"2024-08-17T17:37:56.396375+0000","last_fullsized":"2024-08-17T17:37:56.396375+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":417,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.395316+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:37:45.395316+0000","last_peered":"2024-08-17T17:37:45.395316+0000","last_clean":"2024-08-17T17:37:45.395316+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:37:45.395316+0000","last_undegraded":"2024-08-17T17:37:45.395316+0000","last_fullsized":"2024-08-17T17:37:45.395316+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"365'9180","reported_seq":9310,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:00.207149+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:38:00.207149+0000","last_peered":"2024-08-17T17:38:00.207149+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:38:00.207149+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'2887","ondisk_log_start":"325'2887","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6293,"log_dups_size":0,"ondisk_log_size":6293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367263744,"num_objects":5604,"num_object_clones":0,"num_object_copies":16812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9180,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5604},{"shards":"0,3","objects":-1644}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":595,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364685+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:37:56.364685+0000","last_peered":"2024-08-17T17:37:56.364685+0000","last_clean":"2024-08-17T17:37:56.364685+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:37:56.364685+0000","last_undegraded":"2024-08-17T17:37:56.364685+0000","last_fullsized":"2024-08-17T17:37:56.364685+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":563,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.366375+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:37:56.366375+0000","last_peered":"2024-08-17T17:37:56.366375+0000","last_clean":"2024-08-17T17:37:56.366375+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:37:56.366375+0000","last_undegraded":"2024-08-17T17:37:56.366375+0000","last_fullsized":"2024-08-17T17:37:56.366375+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":581,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.366614+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:37:56.366614+0000","last_peered":"2024-08-17T17:37:56.366614+0000","last_clean":"2024-08-17T17:37:56.366614+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:37:56.366614+0000","last_undegraded":"2024-08-17T17:37:56.366614+0000","last_fullsized":"2024-08-17T17:37:56.366614+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"365'9082","reported_seq":23744,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:38:00.700650+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:38:00.700650+0000","last_peered":"2024-08-17T17:38:00.700650+0000","last_clean":"2024-08-17T17:38:00.700650+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:38:00.700650+0000","last_undegraded":"2024-08-17T17:38:00.700650+0000","last_fullsized":"2024-08-17T17:38:00.700650+0000","mapping_epoch":347,"log_start":"324'2472","ondisk_log_start":"324'2472","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359923712,"num_objects":5492,"num_object_clones":0,"num_object_copies":16476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9082,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":254,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.397616+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:37:56.397616+0000","last_peered":"2024-08-17T17:37:56.397616+0000","last_clean":"2024-08-17T17:37:56.397616+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:37:56.397616+0000","last_undegraded":"2024-08-17T17:37:56.397616+0000","last_fullsized":"2024-08-17T17:37:56.397616+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":369,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346880+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:37:45.346880+0000","last_peered":"2024-08-17T17:37:45.346880+0000","last_clean":"2024-08-17T17:37:45.346880+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:37:45.346880+0000","last_undegraded":"2024-08-17T17:37:45.346880+0000","last_fullsized":"2024-08-17T17:37:45.346880+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":319,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345930+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:37:45.345930+0000","last_peered":"2024-08-17T17:37:45.345930+0000","last_clean":"2024-08-17T17:37:45.345930+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:37:45.345930+0000","last_undegraded":"2024-08-17T17:37:45.345930+0000","last_fullsized":"2024-08-17T17:37:45.345930+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":319,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347451+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:37:45.347451+0000","last_peered":"2024-08-17T17:37:45.347451+0000","last_clean":"2024-08-17T17:37:45.347451+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:37:45.347451+0000","last_undegraded":"2024-08-17T17:37:45.347451+0000","last_fullsized":"2024-08-17T17:37:45.347451+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":362,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346833+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:37:45.346833+0000","last_peered":"2024-08-17T17:37:45.346833+0000","last_clean":"2024-08-17T17:37:45.346833+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:37:45.346833+0000","last_undegraded":"2024-08-17T17:37:45.346833+0000","last_fullsized":"2024-08-17T17:37:45.346833+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":570,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365474+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:37:56.365474+0000","last_peered":"2024-08-17T17:37:56.365474+0000","last_clean":"2024-08-17T17:37:56.365474+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:37:56.365474+0000","last_undegraded":"2024-08-17T17:37:56.365474+0000","last_fullsized":"2024-08-17T17:37:56.365474+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":110,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365144+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:37:56.365144+0000","last_peered":"2024-08-17T17:37:56.365144+0000","last_clean":"2024-08-17T17:37:56.365144+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:37:56.365144+0000","last_undegraded":"2024-08-17T17:37:56.365144+0000","last_fullsized":"2024-08-17T17:37:56.365144+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.395217+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:37:45.395217+0000","last_peered":"2024-08-17T17:37:45.395217+0000","last_clean":"2024-08-17T17:37:45.395217+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:37:45.395217+0000","last_undegraded":"2024-08-17T17:37:45.395217+0000","last_fullsized":"2024-08-17T17:37:45.395217+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":528,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.363810+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:37:56.363810+0000","last_peered":"2024-08-17T17:37:56.363810+0000","last_clean":"2024-08-17T17:37:56.363810+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:37:56.363810+0000","last_undegraded":"2024-08-17T17:37:56.363810+0000","last_fullsized":"2024-08-17T17:37:56.363810+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":413,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.396026+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:37:45.396026+0000","last_peered":"2024-08-17T17:37:45.396026+0000","last_clean":"2024-08-17T17:37:45.396026+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:37:45.396026+0000","last_undegraded":"2024-08-17T17:37:45.396026+0000","last_fullsized":"2024-08-17T17:37:45.396026+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":522,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365211+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:37:56.365211+0000","last_peered":"2024-08-17T17:37:56.365211+0000","last_clean":"2024-08-17T17:37:56.365211+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:37:56.365211+0000","last_undegraded":"2024-08-17T17:37:56.365211+0000","last_fullsized":"2024-08-17T17:37:56.365211+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":336,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346596+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:37:45.346596+0000","last_peered":"2024-08-17T17:37:45.346596+0000","last_clean":"2024-08-17T17:37:45.346596+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:37:45.346596+0000","last_undegraded":"2024-08-17T17:37:45.346596+0000","last_fullsized":"2024-08-17T17:37:45.346596+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":554,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.366694+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:37:56.366694+0000","last_peered":"2024-08-17T17:37:56.366694+0000","last_clean":"2024-08-17T17:37:56.366694+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:37:56.366694+0000","last_undegraded":"2024-08-17T17:37:56.366694+0000","last_fullsized":"2024-08-17T17:37:56.366694+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":270,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364976+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:37:56.364976+0000","last_peered":"2024-08-17T17:37:56.364976+0000","last_clean":"2024-08-17T17:37:56.364976+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:37:56.364976+0000","last_undegraded":"2024-08-17T17:37:56.364976+0000","last_fullsized":"2024-08-17T17:37:56.364976+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":562,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364992+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:37:56.364992+0000","last_peered":"2024-08-17T17:37:56.364992+0000","last_clean":"2024-08-17T17:37:56.364992+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:37:56.364992+0000","last_undegraded":"2024-08-17T17:37:56.364992+0000","last_fullsized":"2024-08-17T17:37:56.364992+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":522,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364998+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:37:56.364998+0000","last_peered":"2024-08-17T17:37:56.364998+0000","last_clean":"2024-08-17T17:37:56.364998+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:37:56.364998+0000","last_undegraded":"2024-08-17T17:37:56.364998+0000","last_fullsized":"2024-08-17T17:37:56.364998+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":348,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347707+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:37:45.347707+0000","last_peered":"2024-08-17T17:37:45.347707+0000","last_clean":"2024-08-17T17:37:45.347707+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:37:45.347707+0000","last_undegraded":"2024-08-17T17:37:45.347707+0000","last_fullsized":"2024-08-17T17:37:45.347707+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":210,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364152+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:37:56.364152+0000","last_peered":"2024-08-17T17:37:56.364152+0000","last_clean":"2024-08-17T17:37:56.364152+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:37:56.364152+0000","last_undegraded":"2024-08-17T17:37:56.364152+0000","last_fullsized":"2024-08-17T17:37:56.364152+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":267,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.396268+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:37:56.396268+0000","last_peered":"2024-08-17T17:37:56.396268+0000","last_clean":"2024-08-17T17:37:56.396268+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:37:56.396268+0000","last_undegraded":"2024-08-17T17:37:56.396268+0000","last_fullsized":"2024-08-17T17:37:56.396268+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":356,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347391+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:37:45.347391+0000","last_peered":"2024-08-17T17:37:45.347391+0000","last_clean":"2024-08-17T17:37:45.347391+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:37:45.347391+0000","last_undegraded":"2024-08-17T17:37:45.347391+0000","last_fullsized":"2024-08-17T17:37:45.347391+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":219,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346711+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:37:45.346711+0000","last_peered":"2024-08-17T17:37:45.346711+0000","last_clean":"2024-08-17T17:37:45.346711+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:37:45.346711+0000","last_undegraded":"2024-08-17T17:37:45.346711+0000","last_fullsized":"2024-08-17T17:37:45.346711+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":356,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.346158+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:37:45.346158+0000","last_peered":"2024-08-17T17:37:45.346158+0000","last_clean":"2024-08-17T17:37:45.346158+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:37:45.346158+0000","last_undegraded":"2024-08-17T17:37:45.346158+0000","last_fullsized":"2024-08-17T17:37:45.346158+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":219,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.347110+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:37:45.347110+0000","last_peered":"2024-08-17T17:37:45.347110+0000","last_clean":"2024-08-17T17:37:45.347110+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:37:45.347110+0000","last_undegraded":"2024-08-17T17:37:45.347110+0000","last_fullsized":"2024-08-17T17:37:45.347110+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":574,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.364320+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:37:56.364320+0000","last_peered":"2024-08-17T17:37:56.364320+0000","last_clean":"2024-08-17T17:37:56.364320+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:37:56.364320+0000","last_undegraded":"2024-08-17T17:37:56.364320+0000","last_fullsized":"2024-08-17T17:37:56.364320+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":547,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.365072+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:37:56.365072+0000","last_peered":"2024-08-17T17:37:56.365072+0000","last_clean":"2024-08-17T17:37:56.365072+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:37:56.365072+0000","last_undegraded":"2024-08-17T17:37:56.365072+0000","last_fullsized":"2024-08-17T17:37:56.365072+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":319,"reported_epoch":364,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.345650+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:37:45.345650+0000","last_peered":"2024-08-17T17:37:45.345650+0000","last_clean":"2024-08-17T17:37:45.345650+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:37:45.345650+0000","last_undegraded":"2024-08-17T17:37:45.345650+0000","last_fullsized":"2024-08-17T17:37:45.345650+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":269,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.396714+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:37:56.396714+0000","last_peered":"2024-08-17T17:37:56.396714+0000","last_clean":"2024-08-17T17:37:56.396714+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:37:56.396714+0000","last_undegraded":"2024-08-17T17:37:56.396714+0000","last_fullsized":"2024-08-17T17:37:56.396714+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5891883032,"num_objects":89904,"num_object_clones":0,"num_object_copies":269712,"num_objects_missing_on_primary":6245,"num_objects_missing":6245,"num_objects_degraded":55654,"num_objects_misplaced":35303,"num_objects_unfound":0,"num_objects_dirty":89904,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146486,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25443,"num_bytes_recovered":1544880128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19096743936,"data_stored":19096731720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104605,"ondisk_log_size":104605,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170059,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463429,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3404520,"kb_used_data":2514976,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90340632,"statfs":{"total":95995035648,"available":92508807168,"internally_reserved":0,"allocated":2575335424,"data_stored":2573390740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492103,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1120524,"kb_used_data":725188,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92624628,"statfs":{"total":95995035648,"available":94847619072,"internally_reserved":0,"allocated":742592512,"data_stored":740670128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986695,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2334492,"kb_used_data":1429396,"kb_used_omap":1,"kb_used_meta":905086,"kb_avail":91410660,"statfs":{"total":95995035648,"available":93604515840,"internally_reserved":0,"allocated":1463701504,"data_stored":1461767760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926808408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225522,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3796172,"kb_used_data":2876292,"kb_used_omap":4,"kb_used_meta":919867,"kb_avail":89948980,"statfs":{"total":95995035648,"available":92107755520,"internally_reserved":0,"allocated":2945323008,"data_stored":2943400922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":941944256},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075847,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533740,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3569876,"kb_used_data":2532040,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90175276,"statfs":{"total":95995035648,"available":92339482624,"internally_reserved":0,"allocated":2592808960,"data_stored":2590881552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089717,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3158568,"kb_used_data":2171552,"kb_used_omap":5,"kb_used_meta":987002,"kb_avail":90586584,"statfs":{"total":95995035648,"available":92760662016,"internally_reserved":0,"allocated":2223669248,"data_stored":2221724623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1010690107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2217345024,"data_stored":2217345024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2586644480,"data_stored":2586640408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3859025920,"data_stored":3859021848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2938241024,"data_stored":2938241024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1458110464,"data_stored":1458110464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":736755712,"data_stored":736755712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2569273344,"data_stored":2569273344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2731347968,"data_stored":2731343896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:04.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:04.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:04.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:04.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:04.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:04.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:04.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:04.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:04.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:04.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:05.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:05.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:05.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:05.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:05.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:05.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:05.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:05.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:38:05.423 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:38:05.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:05.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:05.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:05.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:05 smithi142 ceph-mon[106729]: pgmap v1330: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 55607/269493 objects degraded (20.634%); 35204/269493 objects misplaced (13.063%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:05 smithi142 ceph-mon[103614]: pgmap v1330: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 55607/269493 objects degraded (20.634%); 35204/269493 objects misplaced (13.063%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:05.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:05.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:05 smithi152 ceph-mon[93456]: pgmap v1330: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 55607/269493 objects degraded (20.634%); 35204/269493 objects misplaced (13.063%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:05.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:05.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:05.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:05.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:06.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:06.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:06.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:06.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:06.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:06.358 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:06.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:06.484 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:06.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:38:06.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:06.607 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:06.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55607/269493 objects degraded (20.634%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:06.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55607/269493 objects degraded (20.634%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:06.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:06.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55607/269493 objects degraded (20.634%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:06.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:06.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:06.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:06.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:06.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:06.996 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:07.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:07.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:07.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:07.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:07.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:07.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:07.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:07.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:07.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:07.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:07.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:07.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:07.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:07.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:07 smithi142 ceph-mon[106729]: pgmap v1331: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 55607/268842 objects degraded (20.684%); 35204/268842 objects misplaced (13.095%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:07.538 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:07 smithi142 ceph-mon[103614]: pgmap v1331: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 55607/268842 objects degraded (20.684%); 35204/268842 objects misplaced (13.095%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:07.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:07.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:07.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:07.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:07 smithi152 ceph-mon[93456]: pgmap v1331: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 55607/268842 objects degraded (20.684%); 35204/268842 objects misplaced (13.095%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:07.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:07.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.851 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:07.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:07.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:07.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:07.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:08.077 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:08.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:08.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:08.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:08.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:08.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:08.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:08.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:08.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:08.597 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:08.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:08.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:08.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:08.816 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:08.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:08.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:08.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:08.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:09.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.098 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:09.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:09.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:09.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:09.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:09.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:09.383 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:38:09.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:09.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:09.500 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:09 smithi142 ceph-mon[106729]: pgmap v1332: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 55576/267993 objects degraded (20.738%); 35057/267993 objects misplaced (13.081%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:38:09.500 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3488787943' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:09.500 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:09.500 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:09 smithi142 ceph-mon[103614]: pgmap v1332: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 55576/267993 objects degraded (20.738%); 35057/267993 objects misplaced (13.081%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:38:09.500 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3488787943' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:09.501 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:09.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:09.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:09.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:09.719 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:38:09.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:09.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:09.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:09 smithi152 ceph-mon[93456]: pgmap v1332: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 55576/267993 objects degraded (20.738%); 35057/267993 objects misplaced (13.081%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:38:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3488787943' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:09.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:09.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:09.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:10.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:10.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:10.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:10.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:10.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:10.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:10.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:38:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:38:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:10 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:10 smithi152 ceph-mon[93456]: osdmap e366: 8 total, 8 up, 6 in 2024-08-17T17:38:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3630890004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:10.590 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:10.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:10.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:10.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[106729]: osdmap e366: 8 total, 8 up, 6 in 2024-08-17T17:38:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3630890004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[103614]: osdmap e366: 8 total, 8 up, 6 in 2024-08-17T17:38:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3630890004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:10.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:10.964 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:11.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:11.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:11.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:11.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:11.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:11.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:11.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:11.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:11.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:11.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:11.579 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:11.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:11.675 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:11 smithi142 ceph-mon[103614]: pgmap v1334: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 55554/267597 objects degraded (20.760%); 35006/267597 objects misplaced (13.082%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:11.675 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55554/267597 objects degraded (20.760%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:11.675 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:11 smithi142 ceph-mon[106729]: pgmap v1334: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 55554/267597 objects degraded (20.760%); 35006/267597 objects misplaced (13.082%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:11.675 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55554/267597 objects degraded (20.760%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:11.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:11.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:11.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:11 smithi152 ceph-mon[93456]: pgmap v1334: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 55554/267597 objects degraded (20.760%); 35006/267597 objects misplaced (13.082%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55554/267597 objects degraded (20.760%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:11.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:11.931 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:38:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:38:11] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:38:11.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:11.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:11.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:12.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:12.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:12.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:12.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:12.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:12.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:12.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:12.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:12.504 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:12.504 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:12.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:12.586 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:12.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:12.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:12.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:12.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:12.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:13.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:13.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:13.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:13.209 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:38:13.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:13.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:13.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:13.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:13.501 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:13 smithi142 ceph-mon[106729]: pgmap v1335: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55525/266646 objects degraded (20.823%); 34891/266646 objects misplaced (13.085%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:13.501 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:13 smithi142 ceph-mon[103614]: pgmap v1335: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55525/266646 objects degraded (20.823%); 34891/266646 objects misplaced (13.085%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:13.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:13.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:13.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:38:13.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:13.648 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:13.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:13.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:13.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:13 smithi152 ceph-mon[93456]: pgmap v1335: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55525/266646 objects degraded (20.823%); 34891/266646 objects misplaced (13.085%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:13.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:13.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:14.000 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:14.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:14.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:14.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:14.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:14.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:14.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:14.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:38:14.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:14.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:14.508 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:14.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:14.590 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:14.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:14.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:14.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:14.780 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:14.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:14.904 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:15.006 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:15.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:15.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:15.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:15.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:15.161 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:15.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:15.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:15.361 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:38:15.362 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:38:15.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:15.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:15.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:15.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:15.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:15.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:15.728 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:15 smithi142 ceph-mon[106729]: pgmap v1336: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55488/266442 objects degraded (20.826%); 34823/266442 objects misplaced (13.070%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:38:15.728 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:15 smithi142 ceph-mon[103614]: pgmap v1336: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55488/266442 objects degraded (20.826%); 34823/266442 objects misplaced (13.070%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:38:15.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:15.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:15 smithi152 ceph-mon[93456]: pgmap v1336: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55488/266442 objects degraded (20.826%); 34823/266442 objects misplaced (13.070%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:38:15.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:15.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:16.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:16.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:16.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:16.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:16.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:16.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:16.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:16.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:16.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:16.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:16.524 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:16 smithi142 ceph-mon[106729]: from='client.47159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55488/266442 objects degraded (20.826%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:16 smithi142 ceph-mon[103614]: from='client.47159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:16.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55488/266442 objects degraded (20.826%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:16.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:16.648 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:16.748 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:16.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:16.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:16 smithi152 ceph-mon[93456]: from='client.47159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55488/266442 objects degraded (20.826%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:16.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:16.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:16.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:17.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:17.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:17.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:17.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:17.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:17.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:17.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:17.509 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1336,"stamp":"2024-08-17T17:38:14.289020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5819340773,"num_objects":88814,"num_object_clones":0,"num_object_copies":266442,"num_objects_missing_on_primary":6186,"num_objects_missing":6186,"num_objects_degraded":55488,"num_objects_misplaced":34823,"num_objects_unfound":0,"num_objects_dirty":88814,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":147880,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25723,"num_bytes_recovered":1562911374,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104728,"ondisk_log_size":104728,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17219760,"kb_used_data":12093364,"kb_used_omap":27,"kb_used_meta":5126308,"kb_avail":545251152,"statfs":{"total":575970213888,"available":558337179648,"internally_reserved":0,"allocated":12383604736,"data_stored":12371970840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5249340136},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-66650112,"num_objects":-1017,"num_object_clones":0,"num_object_copies":-3051,"num_objects_missing_on_primary":-59,"num_objects_missing":-59,"num_objects_degraded":-119,"num_objects_misplaced":-381,"num_objects_unfound":0,"num_objects_dirty":-1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1017,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":12386304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002981"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":573,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525758+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:38:09.525758+0000","last_peered":"2024-08-17T17:38:09.525758+0000","last_clean":"2024-08-17T17:38:09.525758+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:38:09.525758+0000","last_undegraded":"2024-08-17T17:38:09.525758+0000","last_fullsized":"2024-08-17T17:38:09.525758+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370161+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:37:56.370161+0000","last_peered":"2024-08-17T17:37:56.370161+0000","last_clean":"2024-08-17T17:37:56.370161+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:37:56.370161+0000","last_undegraded":"2024-08-17T17:37:56.370161+0000","last_fullsized":"2024-08-17T17:37:56.370161+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":269,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384550+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:37:56.384550+0000","last_peered":"2024-08-17T17:37:56.384550+0000","last_clean":"2024-08-17T17:37:56.384550+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:37:56.384550+0000","last_undegraded":"2024-08-17T17:37:56.384550+0000","last_fullsized":"2024-08-17T17:37:56.384550+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":271,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525615+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:38:09.525615+0000","last_peered":"2024-08-17T17:38:09.525615+0000","last_clean":"2024-08-17T17:38:09.525615+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:38:09.525615+0000","last_undegraded":"2024-08-17T17:38:09.525615+0000","last_fullsized":"2024-08-17T17:38:09.525615+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":448,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525800+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:38:09.525800+0000","last_peered":"2024-08-17T17:38:09.525800+0000","last_clean":"2024-08-17T17:38:09.525800+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:38:09.525800+0000","last_undegraded":"2024-08-17T17:38:09.525800+0000","last_fullsized":"2024-08-17T17:38:09.525800+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":218,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370097+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:37:56.370097+0000","last_peered":"2024-08-17T17:37:56.370097+0000","last_clean":"2024-08-17T17:37:56.370097+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:37:56.370097+0000","last_undegraded":"2024-08-17T17:37:56.370097+0000","last_fullsized":"2024-08-17T17:37:56.370097+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":542,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525791+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:38:09.525791+0000","last_peered":"2024-08-17T17:38:09.525791+0000","last_clean":"2024-08-17T17:38:09.525791+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:38:09.525791+0000","last_undegraded":"2024-08-17T17:38:09.525791+0000","last_fullsized":"2024-08-17T17:38:09.525791+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":386,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.388263+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:37:56.388263+0000","last_peered":"2024-08-17T17:37:56.388263+0000","last_clean":"2024-08-17T17:37:56.388263+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:37:56.388263+0000","last_undegraded":"2024-08-17T17:37:56.388263+0000","last_fullsized":"2024-08-17T17:37:56.388263+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"366'9206","reported_seq":11534,"reported_epoch":366,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:38:12.074675+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:38:12.074675+0000","last_peered":"2024-08-17T17:38:12.074675+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:38:12.074675+0000","last_undegraded":"2024-08-17T17:38:12.074675+0000","last_fullsized":"2024-08-17T17:38:12.074675+0000","mapping_epoch":337,"log_start":"324'2363","ondisk_log_start":"324'2363","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360316928,"num_objects":5498,"num_object_clones":0,"num_object_copies":16494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5498,"num_objects_unfound":0,"num_objects_dirty":5498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9206,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1125,"num_bytes_recovered":73728000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5498}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":71,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.473268+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:37:56.473268+0000","last_peered":"2024-08-17T17:37:56.473268+0000","last_clean":"2024-08-17T17:37:56.473268+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:37:56.473268+0000","last_undegraded":"2024-08-17T17:37:56.473268+0000","last_fullsized":"2024-08-17T17:37:56.473268+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.385551+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:37:56.385551+0000","last_peered":"2024-08-17T17:37:56.385551+0000","last_clean":"2024-08-17T17:37:56.385551+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:37:56.385551+0000","last_undegraded":"2024-08-17T17:37:56.385551+0000","last_fullsized":"2024-08-17T17:37:56.385551+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"366'9418","reported_seq":9482,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:11.922608+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:38:11.922608+0000","last_peered":"2024-08-17T17:38:11.922608+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:38:11.922608+0000","last_undegraded":"2024-08-17T17:38:11.922608+0000","last_fullsized":"2024-08-17T17:38:11.922608+0000","mapping_epoch":337,"log_start":"325'2882","ondisk_log_start":"325'2882","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370671616,"num_objects":5656,"num_object_clones":0,"num_object_copies":16968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5656,"num_objects_unfound":0,"num_objects_dirty":5656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9418,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5656}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370667+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:37:56.370667+0000","last_peered":"2024-08-17T17:37:56.370667+0000","last_clean":"2024-08-17T17:37:56.370667+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:37:56.370667+0000","last_undegraded":"2024-08-17T17:37:56.370667+0000","last_fullsized":"2024-08-17T17:37:56.370667+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":337,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370019+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:37:56.370019+0000","last_peered":"2024-08-17T17:37:56.370019+0000","last_clean":"2024-08-17T17:37:56.370019+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:37:56.370019+0000","last_undegraded":"2024-08-17T17:37:56.370019+0000","last_fullsized":"2024-08-17T17:37:56.370019+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"366'9179","reported_seq":9246,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:11.888312+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:38:11.888312+0000","last_peered":"2024-08-17T17:38:11.888312+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:38:11.888312+0000","last_undegraded":"2024-08-17T17:38:11.888312+0000","last_fullsized":"2024-08-17T17:38:11.888312+0000","mapping_epoch":337,"log_start":"324'2353","ondisk_log_start":"324'2353","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6826,"log_dups_size":0,"ondisk_log_size":6826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364052480,"num_objects":5555,"num_object_clones":0,"num_object_copies":16665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5555,"num_objects_unfound":0,"num_objects_dirty":5555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9179,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5555}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":69,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384478+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:37:56.384478+0000","last_peered":"2024-08-17T17:37:56.384478+0000","last_clean":"2024-08-17T17:37:56.384478+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:37:56.384478+0000","last_undegraded":"2024-08-17T17:37:56.384478+0000","last_fullsized":"2024-08-17T17:37:56.384478+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":347,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.385040+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:37:56.385040+0000","last_peered":"2024-08-17T17:37:56.385040+0000","last_clean":"2024-08-17T17:37:56.385040+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:37:56.385040+0000","last_undegraded":"2024-08-17T17:37:56.385040+0000","last_fullsized":"2024-08-17T17:37:56.385040+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"366'9317","reported_seq":9428,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:09.537982+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:38:09.537982+0000","last_peered":"2024-08-17T17:38:09.537982+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:38:09.537982+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2313","ondisk_log_start":"324'2313","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369688576,"num_objects":5641,"num_object_clones":0,"num_object_copies":16923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9317,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1128},{"shards":"0,3,7","objects":5641}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":485,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525922+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:38:09.525922+0000","last_peered":"2024-08-17T17:38:09.525922+0000","last_clean":"2024-08-17T17:38:09.525922+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:38:09.525922+0000","last_undegraded":"2024-08-17T17:38:09.525922+0000","last_fullsized":"2024-08-17T17:38:09.525922+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":252,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.530473+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:38:09.530473+0000","last_peered":"2024-08-17T17:38:09.530473+0000","last_clean":"2024-08-17T17:38:09.530473+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:38:09.530473+0000","last_undegraded":"2024-08-17T17:38:09.530473+0000","last_fullsized":"2024-08-17T17:38:09.530473+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"366'9091","reported_seq":9146,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:11.872725+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:38:11.872725+0000","last_peered":"2024-08-17T17:38:11.872725+0000","last_clean":"2024-08-17T17:38:11.872725+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:38:11.872725+0000","last_undegraded":"2024-08-17T17:38:11.872725+0000","last_fullsized":"2024-08-17T17:38:11.872725+0000","mapping_epoch":319,"log_start":"324'2474","ondisk_log_start":"324'2474","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360644608,"num_objects":5503,"num_object_clones":0,"num_object_copies":16509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9091,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":575,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525557+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:38:09.525557+0000","last_peered":"2024-08-17T17:38:09.525557+0000","last_clean":"2024-08-17T17:38:09.525557+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:38:09.525557+0000","last_undegraded":"2024-08-17T17:38:09.525557+0000","last_fullsized":"2024-08-17T17:38:09.525557+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":320,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370251+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:37:56.370251+0000","last_peered":"2024-08-17T17:37:56.370251+0000","last_clean":"2024-08-17T17:37:56.370251+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:37:56.370251+0000","last_undegraded":"2024-08-17T17:37:56.370251+0000","last_fullsized":"2024-08-17T17:37:56.370251+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"366'9260","reported_seq":9325,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:12.063167+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:38:12.063167+0000","last_peered":"2024-08-17T17:38:12.063167+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:38:12.063167+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'2735","ondisk_log_start":"325'2735","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371720192,"num_objects":5672,"num_object_clones":0,"num_object_copies":17016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9260,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5672},{"shards":"1,6","objects":-1694}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":321,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.385164+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:37:56.385164+0000","last_peered":"2024-08-17T17:37:56.385164+0000","last_clean":"2024-08-17T17:37:56.385164+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:37:56.385164+0000","last_undegraded":"2024-08-17T17:37:56.385164+0000","last_fullsized":"2024-08-17T17:37:56.385164+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":347,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370581+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:37:56.370581+0000","last_peered":"2024-08-17T17:37:56.370581+0000","last_clean":"2024-08-17T17:37:56.370581+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:37:56.370581+0000","last_undegraded":"2024-08-17T17:37:56.370581+0000","last_fullsized":"2024-08-17T17:37:56.370581+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"366'9406","reported_seq":9518,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:13.554626+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:38:13.554626+0000","last_peered":"2024-08-17T17:38:13.554626+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:38:13.554626+0000","last_undegraded":"2024-08-17T17:38:13.554626+0000","last_fullsized":"2024-08-17T17:38:13.554626+0000","mapping_epoch":337,"log_start":"326'3761","ondisk_log_start":"326'3761","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367263744,"num_objects":5604,"num_object_clones":0,"num_object_copies":16812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5604,"num_objects_unfound":0,"num_objects_dirty":5604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9406,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5604}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":358,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370236+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:37:56.370236+0000","last_peered":"2024-08-17T17:37:56.370236+0000","last_clean":"2024-08-17T17:37:56.370236+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:37:56.370236+0000","last_undegraded":"2024-08-17T17:37:56.370236+0000","last_fullsized":"2024-08-17T17:37:56.370236+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":320,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370745+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:37:56.370745+0000","last_peered":"2024-08-17T17:37:56.370745+0000","last_clean":"2024-08-17T17:37:56.370745+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:37:56.370745+0000","last_undegraded":"2024-08-17T17:37:56.370745+0000","last_fullsized":"2024-08-17T17:37:56.370745+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"366'9212","reported_seq":11604,"reported_epoch":366,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:12.064559+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:38:12.064559+0000","last_peered":"2024-08-17T17:38:12.064559+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:38:12.064559+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:38:12.064559+0000","mapping_epoch":337,"log_start":"324'2341","ondisk_log_start":"324'2341","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6871,"log_dups_size":0,"ondisk_log_size":6871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359923712,"num_objects":5492,"num_object_clones":0,"num_object_copies":16476,"num_objects_missing_on_primary":6186,"num_objects_missing":6186,"num_objects_degraded":5492,"num_objects_misplaced":6186,"num_objects_unfound":0,"num_objects_dirty":5492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9212,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1154,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":5480}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":544,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525546+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:38:09.525546+0000","last_peered":"2024-08-17T17:38:09.525546+0000","last_clean":"2024-08-17T17:38:09.525546+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:38:09.525546+0000","last_undegraded":"2024-08-17T17:38:09.525546+0000","last_fullsized":"2024-08-17T17:38:09.525546+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":481,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525874+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:38:09.525874+0000","last_peered":"2024-08-17T17:38:09.525874+0000","last_clean":"2024-08-17T17:38:09.525874+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:38:09.525874+0000","last_undegraded":"2024-08-17T17:38:09.525874+0000","last_fullsized":"2024-08-17T17:38:09.525874+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"366'9322","reported_seq":11671,"reported_epoch":366,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:13.799612+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:38:13.799612+0000","last_peered":"2024-08-17T17:38:13.799612+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:38:13.799612+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:38:13.799612+0000","mapping_epoch":337,"log_start":"326'3744","ondisk_log_start":"326'3744","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232152,"num_objects":5574,"num_object_clones":0,"num_object_copies":16722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5574,"num_objects_misplaced":6324,"num_objects_unfound":0,"num_objects_dirty":5574,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9322,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1124,"num_bytes_recovered":73596928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5574}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":675,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:09.529867+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:38:09.529867+0000","last_peered":"2024-08-17T17:38:09.529867+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:38:09.529867+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":295,"reported_epoch":365,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:37:56.384243+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:37:56.384243+0000","last_peered":"2024-08-17T17:37:56.384243+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:37:56.384243+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:37:56.384243+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":524,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525526+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:38:09.525526+0000","last_peered":"2024-08-17T17:38:09.525526+0000","last_clean":"2024-08-17T17:38:09.525526+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:38:09.525526+0000","last_undegraded":"2024-08-17T17:38:09.525526+0000","last_fullsized":"2024-08-17T17:38:09.525526+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":330,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.388158+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:37:56.388158+0000","last_peered":"2024-08-17T17:37:56.388158+0000","last_clean":"2024-08-17T17:37:56.388158+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:37:56.388158+0000","last_undegraded":"2024-08-17T17:37:56.388158+0000","last_fullsized":"2024-08-17T17:37:56.388158+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"366'9207","reported_seq":9320,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:09.537025+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:38:09.537025+0000","last_peered":"2024-08-17T17:38:09.537025+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:38:09.537025+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2221","ondisk_log_start":"324'2221","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362872832,"num_objects":5537,"num_object_clones":0,"num_object_copies":16611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9207,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1110},{"shards":"0,4,7","objects":5537}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":340,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381524+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:38:09.381524+0000","last_peered":"2024-08-17T17:38:09.381524+0000","last_clean":"2024-08-17T17:38:09.381524+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:38:09.381524+0000","last_undegraded":"2024-08-17T17:38:09.381524+0000","last_fullsized":"2024-08-17T17:38:09.381524+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":302,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.369940+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:37:56.369940+0000","last_peered":"2024-08-17T17:37:56.369940+0000","last_clean":"2024-08-17T17:37:56.369940+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:37:56.369940+0000","last_undegraded":"2024-08-17T17:37:56.369940+0000","last_fullsized":"2024-08-17T17:37:56.369940+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.530426+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:38:09.530426+0000","last_peered":"2024-08-17T17:38:09.530426+0000","last_clean":"2024-08-17T17:38:09.530426+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:38:09.530426+0000","last_undegraded":"2024-08-17T17:38:09.530426+0000","last_fullsized":"2024-08-17T17:38:09.530426+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"366'9204","reported_seq":9268,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:12.046682+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:38:12.046682+0000","last_peered":"2024-08-17T17:38:12.046682+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:38:12.046682+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'2617","ondisk_log_start":"325'2617","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366477312,"num_objects":5592,"num_object_clones":0,"num_object_copies":16776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9204,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5592},{"shards":"5,6","objects":-1704}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":432,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.527797+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:38:09.527797+0000","last_peered":"2024-08-17T17:38:09.527797+0000","last_clean":"2024-08-17T17:38:09.527797+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:38:09.527797+0000","last_undegraded":"2024-08-17T17:38:09.527797+0000","last_fullsized":"2024-08-17T17:38:09.527797+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.386666+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:37:56.386666+0000","last_peered":"2024-08-17T17:37:56.386666+0000","last_clean":"2024-08-17T17:37:56.386666+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:37:56.386666+0000","last_undegraded":"2024-08-17T17:37:56.386666+0000","last_fullsized":"2024-08-17T17:37:56.386666+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":617,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.382340+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:38:09.382340+0000","last_peered":"2024-08-17T17:38:09.382340+0000","last_clean":"2024-08-17T17:38:09.382340+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:38:09.382340+0000","last_undegraded":"2024-08-17T17:38:09.382340+0000","last_fullsized":"2024-08-17T17:38:09.382340+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"366'9133","reported_seq":9970,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:09.539786+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:38:09.539786+0000","last_peered":"2024-08-17T17:38:09.539786+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:38:09.539786+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2116","ondisk_log_start":"324'2116","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9133,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5513},{"shards":"0,5","objects":-1141}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":408,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381760+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:38:09.381760+0000","last_peered":"2024-08-17T17:38:09.381760+0000","last_clean":"2024-08-17T17:38:09.381760+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:38:09.381760+0000","last_undegraded":"2024-08-17T17:38:09.381760+0000","last_fullsized":"2024-08-17T17:38:09.381760+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":341,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370481+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:37:56.370481+0000","last_peered":"2024-08-17T17:37:56.370481+0000","last_clean":"2024-08-17T17:37:56.370481+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:37:56.370481+0000","last_undegraded":"2024-08-17T17:37:56.370481+0000","last_fullsized":"2024-08-17T17:37:56.370481+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":203,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525677+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:38:09.525677+0000","last_peered":"2024-08-17T17:38:09.525677+0000","last_clean":"2024-08-17T17:38:09.525677+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:38:09.525677+0000","last_undegraded":"2024-08-17T17:38:09.525677+0000","last_fullsized":"2024-08-17T17:38:09.525677+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"366'9269","reported_seq":24226,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:10.491874+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:38:10.491874+0000","last_peered":"2024-08-17T17:38:10.491874+0000","last_clean":"2024-08-17T17:38:10.491874+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:38:10.491874+0000","last_undegraded":"2024-08-17T17:38:10.491874+0000","last_fullsized":"2024-08-17T17:38:10.491874+0000","mapping_epoch":355,"log_start":"325'2930","ondisk_log_start":"325'2930","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364838912,"num_objects":5567,"num_object_clones":0,"num_object_copies":16701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9269,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":280,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384396+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:37:56.384396+0000","last_peered":"2024-08-17T17:37:56.384396+0000","last_clean":"2024-08-17T17:37:56.384396+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:37:56.384396+0000","last_undegraded":"2024-08-17T17:37:56.384396+0000","last_fullsized":"2024-08-17T17:37:56.384396+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525435+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:38:09.525435+0000","last_peered":"2024-08-17T17:38:09.525435+0000","last_clean":"2024-08-17T17:38:09.525435+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:38:09.525435+0000","last_undegraded":"2024-08-17T17:38:09.525435+0000","last_fullsized":"2024-08-17T17:38:09.525435+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":567,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.382042+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:38:09.382042+0000","last_peered":"2024-08-17T17:38:09.382042+0000","last_clean":"2024-08-17T17:38:09.382042+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:38:09.382042+0000","last_undegraded":"2024-08-17T17:38:09.382042+0000","last_fullsized":"2024-08-17T17:38:09.382042+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"366'8981","reported_seq":23505,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:10.442538+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:38:10.442538+0000","last_peered":"2024-08-17T17:38:10.442538+0000","last_clean":"2024-08-17T17:38:10.442538+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:38:10.442538+0000","last_undegraded":"2024-08-17T17:38:10.442538+0000","last_fullsized":"2024-08-17T17:38:10.442538+0000","mapping_epoch":345,"log_start":"325'2672","ondisk_log_start":"325'2672","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6309,"log_dups_size":0,"ondisk_log_size":6309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355401728,"num_objects":5423,"num_object_clones":0,"num_object_copies":16269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8981,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":276,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384487+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:37:56.384487+0000","last_peered":"2024-08-17T17:37:56.384487+0000","last_clean":"2024-08-17T17:37:56.384487+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:37:56.384487+0000","last_undegraded":"2024-08-17T17:37:56.384487+0000","last_fullsized":"2024-08-17T17:37:56.384487+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":323,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.530060+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:38:09.530060+0000","last_peered":"2024-08-17T17:38:09.530060+0000","last_clean":"2024-08-17T17:38:09.530060+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:38:09.530060+0000","last_undegraded":"2024-08-17T17:38:09.530060+0000","last_fullsized":"2024-08-17T17:38:09.530060+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":418,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384127+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:37:56.384127+0000","last_peered":"2024-08-17T17:37:56.384127+0000","last_clean":"2024-08-17T17:37:56.384127+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:37:56.384127+0000","last_undegraded":"2024-08-17T17:37:56.384127+0000","last_fullsized":"2024-08-17T17:37:56.384127+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"366'9244","reported_seq":9377,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:10.550871+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:38:10.550871+0000","last_peered":"2024-08-17T17:38:10.550871+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:38:10.550871+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'2987","ondisk_log_start":"325'2987","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6257,"log_dups_size":0,"ondisk_log_size":6257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363069440,"num_objects":5540,"num_object_clones":0,"num_object_copies":16620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9244,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5540},{"shards":"0,3","objects":-1708}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":597,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525966+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:38:09.525966+0000","last_peered":"2024-08-17T17:38:09.525966+0000","last_clean":"2024-08-17T17:38:09.525966+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:38:09.525966+0000","last_undegraded":"2024-08-17T17:38:09.525966+0000","last_fullsized":"2024-08-17T17:38:09.525966+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":566,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.383017+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:38:09.383017+0000","last_peered":"2024-08-17T17:38:09.383017+0000","last_clean":"2024-08-17T17:38:09.383017+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:38:09.383017+0000","last_undegraded":"2024-08-17T17:38:09.383017+0000","last_fullsized":"2024-08-17T17:38:09.383017+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":583,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525349+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:38:09.525349+0000","last_peered":"2024-08-17T17:38:09.525349+0000","last_clean":"2024-08-17T17:38:09.525349+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:38:09.525349+0000","last_undegraded":"2024-08-17T17:38:09.525349+0000","last_fullsized":"2024-08-17T17:38:09.525349+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"366'9153","reported_seq":23816,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:11.884286+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:38:11.884286+0000","last_peered":"2024-08-17T17:38:11.884286+0000","last_clean":"2024-08-17T17:38:11.884286+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:38:11.884286+0000","last_undegraded":"2024-08-17T17:38:11.884286+0000","last_fullsized":"2024-08-17T17:38:11.884286+0000","mapping_epoch":347,"log_start":"324'2572","ondisk_log_start":"324'2572","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6581,"log_dups_size":0,"ondisk_log_size":6581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355270656,"num_objects":5421,"num_object_clones":0,"num_object_copies":16263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9153,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":256,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.530237+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:38:09.530237+0000","last_peered":"2024-08-17T17:38:09.530237+0000","last_clean":"2024-08-17T17:38:09.530237+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:38:09.530237+0000","last_undegraded":"2024-08-17T17:38:09.530237+0000","last_fullsized":"2024-08-17T17:38:09.530237+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":370,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.387838+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:37:56.387838+0000","last_peered":"2024-08-17T17:37:56.387838+0000","last_clean":"2024-08-17T17:37:56.387838+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:37:56.387838+0000","last_undegraded":"2024-08-17T17:37:56.387838+0000","last_fullsized":"2024-08-17T17:37:56.387838+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":320,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370140+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:37:56.370140+0000","last_peered":"2024-08-17T17:37:56.370140+0000","last_clean":"2024-08-17T17:37:56.370140+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:37:56.370140+0000","last_undegraded":"2024-08-17T17:37:56.370140+0000","last_fullsized":"2024-08-17T17:37:56.370140+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":320,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370400+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:37:56.370400+0000","last_peered":"2024-08-17T17:37:56.370400+0000","last_clean":"2024-08-17T17:37:56.370400+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:37:56.370400+0000","last_undegraded":"2024-08-17T17:37:56.370400+0000","last_fullsized":"2024-08-17T17:37:56.370400+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":363,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370431+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:37:56.370431+0000","last_peered":"2024-08-17T17:37:56.370431+0000","last_clean":"2024-08-17T17:37:56.370431+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:37:56.370431+0000","last_undegraded":"2024-08-17T17:37:56.370431+0000","last_fullsized":"2024-08-17T17:37:56.370431+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":573,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381899+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:38:09.381899+0000","last_peered":"2024-08-17T17:38:09.381899+0000","last_clean":"2024-08-17T17:38:09.381899+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:38:09.381899+0000","last_undegraded":"2024-08-17T17:38:09.381899+0000","last_fullsized":"2024-08-17T17:38:09.381899+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":112,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.526438+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:38:09.526438+0000","last_peered":"2024-08-17T17:38:09.526438+0000","last_clean":"2024-08-17T17:38:09.526438+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:38:09.526438+0000","last_undegraded":"2024-08-17T17:38:09.526438+0000","last_fullsized":"2024-08-17T17:38:09.526438+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":270,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384237+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:37:56.384237+0000","last_peered":"2024-08-17T17:37:56.384237+0000","last_clean":"2024-08-17T17:37:56.384237+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:37:56.384237+0000","last_undegraded":"2024-08-17T17:37:56.384237+0000","last_fullsized":"2024-08-17T17:37:56.384237+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":530,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.527622+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:38:09.527622+0000","last_peered":"2024-08-17T17:38:09.527622+0000","last_clean":"2024-08-17T17:38:09.527622+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:38:09.527622+0000","last_undegraded":"2024-08-17T17:38:09.527622+0000","last_fullsized":"2024-08-17T17:38:09.527622+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":414,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.384898+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:37:56.384898+0000","last_peered":"2024-08-17T17:37:56.384898+0000","last_clean":"2024-08-17T17:37:56.384898+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:37:56.384898+0000","last_undegraded":"2024-08-17T17:37:56.384898+0000","last_fullsized":"2024-08-17T17:37:56.384898+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":524,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525264+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:38:09.525264+0000","last_peered":"2024-08-17T17:38:09.525264+0000","last_clean":"2024-08-17T17:38:09.525264+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:38:09.525264+0000","last_undegraded":"2024-08-17T17:38:09.525264+0000","last_fullsized":"2024-08-17T17:38:09.525264+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":337,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.387766+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:37:56.387766+0000","last_peered":"2024-08-17T17:37:56.387766+0000","last_clean":"2024-08-17T17:37:56.387766+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:37:56.387766+0000","last_undegraded":"2024-08-17T17:37:56.387766+0000","last_fullsized":"2024-08-17T17:37:56.387766+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":556,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525436+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:38:09.525436+0000","last_peered":"2024-08-17T17:38:09.525436+0000","last_clean":"2024-08-17T17:38:09.525436+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:38:09.525436+0000","last_undegraded":"2024-08-17T17:38:09.525436+0000","last_fullsized":"2024-08-17T17:38:09.525436+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":272,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.526073+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:38:09.526073+0000","last_peered":"2024-08-17T17:38:09.526073+0000","last_clean":"2024-08-17T17:38:09.526073+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:38:09.526073+0000","last_undegraded":"2024-08-17T17:38:09.526073+0000","last_fullsized":"2024-08-17T17:38:09.526073+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":564,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381702+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:38:09.381702+0000","last_peered":"2024-08-17T17:38:09.381702+0000","last_clean":"2024-08-17T17:38:09.381702+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:38:09.381702+0000","last_undegraded":"2024-08-17T17:38:09.381702+0000","last_fullsized":"2024-08-17T17:38:09.381702+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":524,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.526353+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:38:09.526353+0000","last_peered":"2024-08-17T17:38:09.526353+0000","last_clean":"2024-08-17T17:38:09.526353+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:38:09.526353+0000","last_undegraded":"2024-08-17T17:38:09.526353+0000","last_fullsized":"2024-08-17T17:38:09.526353+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":349,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370359+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:37:56.370359+0000","last_peered":"2024-08-17T17:37:56.370359+0000","last_clean":"2024-08-17T17:37:56.370359+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:37:56.370359+0000","last_undegraded":"2024-08-17T17:37:56.370359+0000","last_fullsized":"2024-08-17T17:37:56.370359+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":212,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525838+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:38:09.525838+0000","last_peered":"2024-08-17T17:38:09.525838+0000","last_clean":"2024-08-17T17:38:09.525838+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:38:09.525838+0000","last_undegraded":"2024-08-17T17:38:09.525838+0000","last_fullsized":"2024-08-17T17:38:09.525838+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":269,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.530339+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:38:09.530339+0000","last_peered":"2024-08-17T17:38:09.530339+0000","last_clean":"2024-08-17T17:38:09.530339+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:38:09.530339+0000","last_undegraded":"2024-08-17T17:38:09.530339+0000","last_fullsized":"2024-08-17T17:38:09.530339+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":357,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370340+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:37:56.370340+0000","last_peered":"2024-08-17T17:37:56.370340+0000","last_clean":"2024-08-17T17:37:56.370340+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:37:56.370340+0000","last_undegraded":"2024-08-17T17:37:56.370340+0000","last_fullsized":"2024-08-17T17:37:56.370340+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":220,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370335+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:37:56.370335+0000","last_peered":"2024-08-17T17:37:56.370335+0000","last_clean":"2024-08-17T17:37:56.370335+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:37:56.370335+0000","last_undegraded":"2024-08-17T17:37:56.370335+0000","last_fullsized":"2024-08-17T17:37:56.370335+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":357,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370232+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:37:56.370232+0000","last_peered":"2024-08-17T17:37:56.370232+0000","last_clean":"2024-08-17T17:37:56.370232+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:37:56.370232+0000","last_undegraded":"2024-08-17T17:37:56.370232+0000","last_fullsized":"2024-08-17T17:37:56.370232+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":220,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.370494+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:37:56.370494+0000","last_peered":"2024-08-17T17:37:56.370494+0000","last_clean":"2024-08-17T17:37:56.370494+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:37:56.370494+0000","last_undegraded":"2024-08-17T17:37:56.370494+0000","last_fullsized":"2024-08-17T17:37:56.370494+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":576,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.527798+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:38:09.527798+0000","last_peered":"2024-08-17T17:38:09.527798+0000","last_clean":"2024-08-17T17:38:09.527798+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:38:09.527798+0000","last_undegraded":"2024-08-17T17:38:09.527798+0000","last_fullsized":"2024-08-17T17:38:09.527798+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":549,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.525919+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:38:09.525919+0000","last_peered":"2024-08-17T17:38:09.525919+0000","last_clean":"2024-08-17T17:38:09.525919+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:38:09.525919+0000","last_undegraded":"2024-08-17T17:38:09.525919+0000","last_fullsized":"2024-08-17T17:38:09.525919+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":320,"reported_epoch":365,"state":"active+clean","last_fresh":"2024-08-17T17:37:56.369944+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:37:56.369944+0000","last_peered":"2024-08-17T17:37:56.369944+0000","last_clean":"2024-08-17T17:37:56.369944+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:37:56.369944+0000","last_undegraded":"2024-08-17T17:37:56.369944+0000","last_fullsized":"2024-08-17T17:37:56.369944+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":271,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.530223+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:38:09.530223+0000","last_peered":"2024-08-17T17:38:09.530223+0000","last_clean":"2024-08-17T17:38:09.530223+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:38:09.530223+0000","last_undegraded":"2024-08-17T17:38:09.530223+0000","last_fullsized":"2024-08-17T17:38:09.530223+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5818744856,"num_objects":88788,"num_object_clones":0,"num_object_copies":266364,"num_objects_missing_on_primary":6186,"num_objects_missing":6186,"num_objects_degraded":55485,"num_objects_misplaced":34823,"num_objects_unfound":0,"num_objects_dirty":88788,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147602,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25682,"num_bytes_recovered":1560543232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18858848256,"data_stored":18858836040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104521,"ondisk_log_size":104521,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170061,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463431,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3377912,"kb_used_data":2488368,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90367240,"statfs":{"total":95995035648,"available":92536053760,"internally_reserved":0,"allocated":2548088832,"data_stored":2546136355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492105,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1115872,"kb_used_data":716372,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92629280,"statfs":{"total":95995035648,"available":94852382720,"internally_reserved":0,"allocated":733564928,"data_stored":731634751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986697,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301552,"kb_used_data":1413092,"kb_used_omap":1,"kb_used_meta":888446,"kb_avail":91443600,"statfs":{"total":95995035648,"available":93638246400,"internally_reserved":0,"allocated":1447006208,"data_stored":1445064671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909769048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225524,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3768160,"kb_used_data":2844116,"kb_used_omap":4,"kb_used_meta":924027,"kb_avail":89976992,"statfs":{"total":95995035648,"available":92136439808,"internally_reserved":0,"allocated":2912374784,"data_stored":2910444905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":946204096},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075849,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533743,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3523424,"kb_used_data":2485592,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90221728,"statfs":{"total":95995035648,"available":92387049472,"internally_reserved":0,"allocated":2545246208,"data_stored":2543311007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089719,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3132840,"kb_used_data":2145824,"kb_used_omap":5,"kb_used_meta":987002,"kb_avail":90612312,"statfs":{"total":95995035648,"available":92787007488,"internally_reserved":0,"allocated":2197323776,"data_stored":2195379151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1010690107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2191523840,"data_stored":2191523840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2539327488,"data_stored":2539323416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3800043520,"data_stored":3800039448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2905735168,"data_stored":2905735168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1442054144,"data_stored":1442054144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":728825856,"data_stored":728825856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2539782144,"data_stored":2539782144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2711556096,"data_stored":2711552024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:17.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:38:17.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:17.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:17.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:17.699 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:17 smithi142 ceph-mon[106729]: pgmap v1337: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55488/265806 objects degraded (20.875%); 34823/265806 objects misplaced (13.101%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:38:17.699 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:17 smithi142 ceph-mon[103614]: pgmap v1337: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55488/265806 objects degraded (20.875%); 34823/265806 objects misplaced (13.101%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:38:17.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:17.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:17 smithi152 ceph-mon[93456]: pgmap v1337: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55488/265806 objects degraded (20.875%); 34823/265806 objects misplaced (13.101%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:38:17.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:17.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:17.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:18.015 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:18.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:18.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:18.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:18.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:18.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:18.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:18.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:18.465 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:18.465 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:18.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:18.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:18.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:18.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:18.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:18.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:18.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:18.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:18.880 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:18.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:18.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:19.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:19.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:19.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:19.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:19.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:19.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:38:19.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:19.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:19.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:19.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:19.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:19 smithi142 ceph-mon[103614]: pgmap v1338: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55459/264939 objects degraded (20.933%); 34696/264939 objects misplaced (13.096%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:19 smithi142 ceph-mon[106729]: pgmap v1338: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55459/264939 objects degraded (20.933%); 34696/264939 objects misplaced (13.096%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:19.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:19.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:19.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:19.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:19 smithi152 ceph-mon[93456]: pgmap v1338: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 55459/264939 objects degraded (20.933%); 34696/264939 objects misplaced (13.096%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:19.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:19.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:19.919 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:20.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:20.032 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:20.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:20.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:20.156 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:20.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:20.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:20.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.405 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:20.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:20.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:20.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:20.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:38:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:38:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:20.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:20.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:20.778 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:20.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:20.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:20.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:21.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:21.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:21.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:21.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:21.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:21.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:21.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:21.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:21.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:21.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:21.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:21.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:21.533 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:21 smithi142 ceph-mon[106729]: pgmap v1339: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55418/264432 objects degraded (20.957%); 34613/264432 objects misplaced (13.090%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:21.533 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55418/264432 objects degraded (20.957%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:21.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:21 smithi142 ceph-mon[103614]: pgmap v1339: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55418/264432 objects degraded (20.957%); 34613/264432 objects misplaced (13.090%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:21.533 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55418/264432 objects degraded (20.957%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:21.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:21.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:21.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:21.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:21.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:21 smithi152 ceph-mon[93456]: pgmap v1339: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 55418/264432 objects degraded (20.957%); 34613/264432 objects misplaced (13.090%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55418/264432 objects degraded (20.957%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:38:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:38:21] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:38:21.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:21.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:21.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:22.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:22.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:22.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:22.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:22.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:22.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:22.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:22.423 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:38:22.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:22.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:22.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:22.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3920697140' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:22.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3920697140' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:22.625 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:22.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:22.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:22.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:22.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3920697140' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:22.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:22.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:23.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:23.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:23.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:23.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:23.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:23.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:23.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:23.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:23.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:23.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[106729]: pgmap v1340: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 55386/263418 objects degraded (21.026%); 34490/263418 objects misplaced (13.093%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:38:23.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:23.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3920697140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:23.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[106729]: osdmap e367: 8 total, 8 up, 6 in 2024-08-17T17:38:23.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[103614]: pgmap v1340: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 55386/263418 objects degraded (21.026%); 34490/263418 objects misplaced (13.093%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:38:23.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:23.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3920697140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:23.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:23 smithi142 ceph-mon[103614]: osdmap e367: 8 total, 8 up, 6 in 2024-08-17T17:38:23.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:23.520 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:23.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:23.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:23.707 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:38:23.707 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 55386/263418 objects degraded (21.026%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1984,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":367,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":87806,"data_bytes":5753280485,"bytes_used":17498624000,"bytes_avail":558471589888,"bytes_total":575970213888,"degraded_objects":55386,"degraded_total":263418,"degraded_ratio":0.21025898002414414,"misplaced_objects":34490,"misplaced_total":263418,"misplaced_ratio":0.13093258623176851,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1463237,"recovering_keys_per_sec":0,"num_objects_recovered":268,"num_bytes_recovered":17563648,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":116},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":33,"modified":"2024-08-17T17:38:06.286948+0000","services":{"osd":{"daemons":{"summary":"","1":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:38:23.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:23.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:23 smithi152 ceph-mon[93456]: pgmap v1340: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 55386/263418 objects degraded (21.026%); 34490/263418 objects misplaced (13.093%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:38:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:23 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3920697140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:23 smithi152 ceph-mon[93456]: osdmap e367: 8 total, 8 up, 6 in 2024-08-17T17:38:23.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:23.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:23.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:24.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:24.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:24.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:24.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:24.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:24.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:24.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.514 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:24.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2579733749' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1092604312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2579733749' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1092604312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:24.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:24.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2579733749' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1092604312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:24.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:24.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:24.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:24.980 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:38:24.980 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:38:24.980 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:24.981 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:24.982 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:24.982 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:38:24.982 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:38:24.983 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:24.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:25.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:25.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:25.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:25.187 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:25.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:25.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:25.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:25.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:25.481 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:25.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:25.585 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:25 smithi142 ceph-mon[106729]: pgmap v1342: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 55363/263286 objects degraded (21.028%); 34446/263286 objects misplaced (13.083%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:25.585 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:25 smithi142 ceph-mon[103614]: pgmap v1342: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 55363/263286 objects degraded (21.028%); 34446/263286 objects misplaced (13.083%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:25.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:25.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:25.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:25.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.752 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:25.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:25 smithi152 ceph-mon[93456]: pgmap v1342: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 55363/263286 objects degraded (21.028%); 34446/263286 objects misplaced (13.083%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:25.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:25.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:25.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:25.998 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:26.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:26.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:26.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:26.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:26.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:26.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:26.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:26.464 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:38:26.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:26.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:26.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:26.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:26.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:26.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:26.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:26.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:26.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:38:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55363/263286 objects degraded (21.028%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:26.840 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55363/263286 objects degraded (21.028%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:26.841 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55363/263286 objects degraded (21.028%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:26.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:26.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:27.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:27.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:27.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:27.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:27.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:27.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:27.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:27.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:27.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:27.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:27.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:27.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:27.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:27.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:27.726 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:27 smithi142 ceph-mon[106729]: pgmap v1343: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 55363/262809 objects degraded (21.066%); 34446/262809 objects misplaced (13.107%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:27.726 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:27.726 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:27 smithi142 ceph-mon[103614]: pgmap v1343: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 55363/262809 objects degraded (21.066%); 34446/262809 objects misplaced (13.107%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:27.726 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:27.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:27.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:27 smithi152 ceph-mon[93456]: pgmap v1343: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 55363/262809 objects degraded (21.066%); 34446/262809 objects misplaced (13.107%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:27.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:27.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:28.001 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:38:28.002 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:38:28.002 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:38:28.002 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:38:28.002 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:38:28.009 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:28.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:28.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:28.155 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:28.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:28.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:28.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:28.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:28.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:28.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:28.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:28.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:28.644 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:28.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:28.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:38:28.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:28.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:28.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:28.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:28.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:28.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:28.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:29.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:29.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:29.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:29.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:29.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:29.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:29.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:29.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:29.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:29.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:29 smithi142 ceph-mon[106729]: pgmap v1344: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 55332/262017 objects degraded (21.118%); 34312/262017 objects misplaced (13.095%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:29.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:29 smithi142 ceph-mon[103614]: pgmap v1344: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 55332/262017 objects degraded (21.118%); 34312/262017 objects misplaced (13.095%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:29.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:29.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:29.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:29.809 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:29 smithi152 ceph-mon[93456]: pgmap v1344: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 55332/262017 objects degraded (21.118%); 34312/262017 objects misplaced (13.095%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:29.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:29.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:30.038 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:30.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:30.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:30.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:30.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:30.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:30.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:30.440 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:30.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:30.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:30.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:38:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:38:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:30.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:30.687 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:30.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:30.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:30.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:30.863 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:30.881 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:30.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:30.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:31.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:31.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:31.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:31.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:31.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:31.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:31.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:31.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:31.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:31.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:31.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:31.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:31.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:31.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-mon[106729]: pgmap v1345: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 55301/261564 objects degraded (21.142%); 34250/261564 objects misplaced (13.094%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:31.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55301/261564 objects degraded (21.142%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:31.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:31.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-mon[103614]: pgmap v1345: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 55301/261564 objects degraded (21.142%); 34250/261564 objects misplaced (13.094%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:31.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55301/261564 objects degraded (21.142%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:31.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:31.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:31.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:31.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:31 smithi152 ceph-mon[93456]: pgmap v1345: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 55301/261564 objects degraded (21.142%); 34250/261564 objects misplaced (13.094%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55301/261564 objects degraded (21.142%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:31.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:31.924 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:32.023 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:38:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:38:31] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-08-17T17:38:32.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:32.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:32.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:32.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:32.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:32.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:32.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:32.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:32.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:32.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:32.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:32.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:32.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:32.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:32.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:33.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:33.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:33.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:33.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:33.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:33.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:33.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:33.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:33.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:33.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:33.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:33 smithi142 ceph-mon[106729]: pgmap v1346: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55301/260691 objects degraded (21.213%); 34213/260691 objects misplaced (13.124%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:38:33.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:33 smithi142 ceph-mon[103614]: pgmap v1346: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55301/260691 objects degraded (21.213%); 34213/260691 objects misplaced (13.124%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:38:33.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:33.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:33.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:33.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:33 smithi152 ceph-mon[93456]: pgmap v1346: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55301/260691 objects degraded (21.213%); 34213/260691 objects misplaced (13.124%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:38:33.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:33.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:33.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:34.039 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:34.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:34.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:34.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:34.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:34.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:34.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:34.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:34.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:34.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:34.450 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:38:34.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:34.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:34.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:34.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:34.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3664528522' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:34 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3664528522' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:34 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3664528522' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.773 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:34 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:34.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:34.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:34.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:34.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:35.038 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:35.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:35.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:35.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.186 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:35.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:35.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:35.292 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:38:35.293 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:38:35.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:35.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:35.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:35.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:35.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[106729]: pgmap v1347: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55242/260211 objects degraded (21.230%); 34053/260211 objects misplaced (13.087%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[106729]: osdmap e368: 8 total, 8 up, 6 in 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[103614]: pgmap v1347: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55242/260211 objects degraded (21.230%); 34053/260211 objects misplaced (13.087%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:35.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:35 smithi142 ceph-mon[103614]: osdmap e368: 8 total, 8 up, 6 in 2024-08-17T17:38:35.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:35.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:35.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:35.801 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:35 smithi152 ceph-mon[93456]: pgmap v1347: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 55242/260211 objects degraded (21.230%); 34053/260211 objects misplaced (13.087%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:38:35.802 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:35 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:35.802 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:35 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:35.802 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:35 smithi152 ceph-mon[93456]: osdmap e368: 8 total, 8 up, 6 in 2024-08-17T17:38:35.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:35.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:35.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:35.974 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:38:35.974 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:35.991 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:38:35.991 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:36.010 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:38:36.010 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:36.027 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:38:36.027 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:36.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:36.043 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:38:36.043 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:36.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:36.059 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:38:36.059 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:36.074 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:38:36.074 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:36.090 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:38:36.090 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:36.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:36.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:36.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:36.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:36.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:36.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:36.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:36.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:36.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:36.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:36.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:36.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:36.774 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[106729]: from='client.47189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:36.774 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55242/260211 objects degraded (21.230%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:36.774 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3045194831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[103614]: from='client.47189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55242/260211 objects degraded (21.230%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3045194831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:36.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:36.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:36.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:36.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:36 smithi152 ceph-mon[93456]: from='client.47189 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55242/260211 objects degraded (21.230%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3045194831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:36.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:36.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:36.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:37.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:37.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:37.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:37.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:37.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:37.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:37.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:37.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:37.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:37.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:37 smithi142 ceph-mon[106729]: pgmap v1349: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 55242/259605 objects degraded (21.279%); 34053/259605 objects misplaced (13.117%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:37.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:37 smithi142 ceph-mon[103614]: pgmap v1349: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 55242/259605 objects degraded (21.279%); 34053/259605 objects misplaced (13.117%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:37.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:37.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:37.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:37.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:38:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:37.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:37 smithi152 ceph-mon[93456]: pgmap v1349: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 55242/259605 objects degraded (21.279%); 34053/259605 objects misplaced (13.117%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:38:37.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:37.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:37.928 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:37.929 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:38.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:38.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:38.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.162 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.287 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:38.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:38.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:38.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:38.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.759 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1348,"stamp":"2024-08-17T17:38:34.446563+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5683222501,"num_objects":86737,"num_object_clones":0,"num_object_copies":260211,"num_objects_missing_on_primary":6065,"num_objects_missing":6065,"num_objects_degraded":55242,"num_objects_misplaced":34053,"num_objects_unfound":0,"num_objects_dirty":86737,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":149957,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26126,"num_bytes_recovered":1589322382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104905,"ondisk_log_size":104905,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17236820,"kb_used_data":11809572,"kb_used_omap":27,"kb_used_meta":5427172,"kb_avail":545234092,"statfs":{"total":575970213888,"available":558319710208,"internally_reserved":0,"allocated":12093001728,"data_stored":12081313281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5557424872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,0,0,0,3,3,8,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-67174400,"num_objects":-1025,"num_object_clones":0,"num_object_copies":-3075,"num_objects_missing_on_primary":-58,"num_objects_missing":-58,"num_objects_degraded":-121,"num_objects_misplaced":-393,"num_objects_unfound":0,"num_objects_dirty":-1025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1025,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":207,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.154800"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":576,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427047+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:38:22.427047+0000","last_peered":"2024-08-17T17:38:22.427047+0000","last_clean":"2024-08-17T17:38:22.427047+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:38:22.427047+0000","last_undegraded":"2024-08-17T17:38:22.427047+0000","last_fullsized":"2024-08-17T17:38:22.427047+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":76,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415940+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:38:09.415940+0000","last_peered":"2024-08-17T17:38:09.415940+0000","last_clean":"2024-08-17T17:38:09.415940+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:38:09.415940+0000","last_undegraded":"2024-08-17T17:38:09.415940+0000","last_fullsized":"2024-08-17T17:38:09.415940+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":270,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.382200+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:38:09.382200+0000","last_peered":"2024-08-17T17:38:09.382200+0000","last_clean":"2024-08-17T17:38:09.382200+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:38:09.382200+0000","last_undegraded":"2024-08-17T17:38:09.382200+0000","last_fullsized":"2024-08-17T17:38:09.382200+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":274,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427021+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:38:22.427021+0000","last_peered":"2024-08-17T17:38:22.427021+0000","last_clean":"2024-08-17T17:38:22.427021+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:38:22.427021+0000","last_undegraded":"2024-08-17T17:38:22.427021+0000","last_fullsized":"2024-08-17T17:38:22.427021+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":451,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427638+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:38:22.427638+0000","last_peered":"2024-08-17T17:38:22.427638+0000","last_clean":"2024-08-17T17:38:22.427638+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:38:22.427638+0000","last_undegraded":"2024-08-17T17:38:22.427638+0000","last_fullsized":"2024-08-17T17:38:22.427638+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":219,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415843+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:38:09.415843+0000","last_peered":"2024-08-17T17:38:09.415843+0000","last_clean":"2024-08-17T17:38:09.415843+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:38:09.415843+0000","last_undegraded":"2024-08-17T17:38:09.415843+0000","last_fullsized":"2024-08-17T17:38:09.415843+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":544,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427104+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:38:22.427104+0000","last_peered":"2024-08-17T17:38:22.427104+0000","last_clean":"2024-08-17T17:38:22.427104+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:38:22.427104+0000","last_undegraded":"2024-08-17T17:38:22.427104+0000","last_fullsized":"2024-08-17T17:38:22.427104+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":387,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.494470+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:38:09.494470+0000","last_peered":"2024-08-17T17:38:09.494470+0000","last_clean":"2024-08-17T17:38:09.494470+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:38:09.494470+0000","last_undegraded":"2024-08-17T17:38:09.494470+0000","last_fullsized":"2024-08-17T17:38:09.494470+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"367'9362","reported_seq":12003,"reported_epoch":367,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:38:32.284646+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:38:32.284646+0000","last_peered":"2024-08-17T17:38:32.284646+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:38:32.284646+0000","last_undegraded":"2024-08-17T17:38:32.284646+0000","last_fullsized":"2024-08-17T17:38:32.284646+0000","mapping_epoch":337,"log_start":"324'2463","ondisk_log_start":"324'2463","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350093312,"num_objects":5342,"num_object_clones":0,"num_object_copies":16026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5342,"num_objects_unfound":0,"num_objects_dirty":5342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9362,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1281,"num_bytes_recovered":83951616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5342}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":72,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.485583+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:38:09.485583+0000","last_peered":"2024-08-17T17:38:09.485583+0000","last_clean":"2024-08-17T17:38:09.485583+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:38:09.485583+0000","last_undegraded":"2024-08-17T17:38:09.485583+0000","last_fullsized":"2024-08-17T17:38:09.485583+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381698+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:38:09.381698+0000","last_peered":"2024-08-17T17:38:09.381698+0000","last_clean":"2024-08-17T17:38:09.381698+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:38:09.381698+0000","last_undegraded":"2024-08-17T17:38:09.381698+0000","last_fullsized":"2024-08-17T17:38:09.381698+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"367'9544","reported_seq":9609,"reported_epoch":367,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:32.133259+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:38:32.133259+0000","last_peered":"2024-08-17T17:38:32.133259+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:38:32.133259+0000","last_undegraded":"2024-08-17T17:38:32.133259+0000","last_fullsized":"2024-08-17T17:38:32.133259+0000","mapping_epoch":337,"log_start":"325'2982","ondisk_log_start":"325'2982","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362414080,"num_objects":5530,"num_object_clones":0,"num_object_copies":16590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5530,"num_objects_unfound":0,"num_objects_dirty":5530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9544,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5530}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":74,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416703+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:38:09.416703+0000","last_peered":"2024-08-17T17:38:09.416703+0000","last_clean":"2024-08-17T17:38:09.416703+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:38:09.416703+0000","last_undegraded":"2024-08-17T17:38:09.416703+0000","last_fullsized":"2024-08-17T17:38:09.416703+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":338,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415639+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:38:09.415639+0000","last_peered":"2024-08-17T17:38:09.415639+0000","last_clean":"2024-08-17T17:38:09.415639+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:38:09.415639+0000","last_undegraded":"2024-08-17T17:38:09.415639+0000","last_fullsized":"2024-08-17T17:38:09.415639+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"367'9291","reported_seq":9359,"reported_epoch":367,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:31.188669+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:38:31.188669+0000","last_peered":"2024-08-17T17:38:31.188669+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:38:31.188669+0000","last_undegraded":"2024-08-17T17:38:31.188669+0000","last_fullsized":"2024-08-17T17:38:31.188669+0000","mapping_epoch":337,"log_start":"324'2453","ondisk_log_start":"324'2453","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6838,"log_dups_size":0,"ondisk_log_size":6838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356712448,"num_objects":5443,"num_object_clones":0,"num_object_copies":16329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5443,"num_objects_unfound":0,"num_objects_dirty":5443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9291,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5443}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":70,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.382129+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:38:09.382129+0000","last_peered":"2024-08-17T17:38:09.382129+0000","last_clean":"2024-08-17T17:38:09.382129+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:38:09.382129+0000","last_undegraded":"2024-08-17T17:38:09.382129+0000","last_fullsized":"2024-08-17T17:38:09.382129+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":348,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.494647+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:38:09.494647+0000","last_peered":"2024-08-17T17:38:09.494647+0000","last_clean":"2024-08-17T17:38:09.494647+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:38:09.494647+0000","last_undegraded":"2024-08-17T17:38:09.494647+0000","last_fullsized":"2024-08-17T17:38:09.494647+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"367'9425","reported_seq":9538,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:29.402818+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:38:29.402818+0000","last_peered":"2024-08-17T17:38:29.402818+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:38:29.402818+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2413","ondisk_log_start":"324'2413","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362610688,"num_objects":5533,"num_object_clones":0,"num_object_copies":16599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9425,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1236},{"shards":"0,3,7","objects":5533}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":487,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427302+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:38:22.427302+0000","last_peered":"2024-08-17T17:38:22.427302+0000","last_clean":"2024-08-17T17:38:22.427302+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:38:22.427302+0000","last_undegraded":"2024-08-17T17:38:22.427302+0000","last_fullsized":"2024-08-17T17:38:22.427302+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":254,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.495790+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:38:22.495790+0000","last_peered":"2024-08-17T17:38:22.495790+0000","last_clean":"2024-08-17T17:38:22.495790+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:38:22.495790+0000","last_undegraded":"2024-08-17T17:38:22.495790+0000","last_fullsized":"2024-08-17T17:38:22.495790+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"367'9235","reported_seq":9291,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:32.098640+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:38:32.098640+0000","last_peered":"2024-08-17T17:38:32.098640+0000","last_clean":"2024-08-17T17:38:32.098640+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:38:32.098640+0000","last_undegraded":"2024-08-17T17:38:32.098640+0000","last_fullsized":"2024-08-17T17:38:32.098640+0000","mapping_epoch":319,"log_start":"325'2674","ondisk_log_start":"325'2674","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351207424,"num_objects":5359,"num_object_clones":0,"num_object_copies":16077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9235,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":577,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427308+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:38:22.427308+0000","last_peered":"2024-08-17T17:38:22.427308+0000","last_clean":"2024-08-17T17:38:22.427308+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:38:22.427308+0000","last_undegraded":"2024-08-17T17:38:22.427308+0000","last_fullsized":"2024-08-17T17:38:22.427308+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":321,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416143+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:38:09.416143+0000","last_peered":"2024-08-17T17:38:09.416143+0000","last_clean":"2024-08-17T17:38:09.416143+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:38:09.416143+0000","last_undegraded":"2024-08-17T17:38:09.416143+0000","last_fullsized":"2024-08-17T17:38:09.416143+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"367'9386","reported_seq":9452,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:32.103047+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:38:32.103047+0000","last_peered":"2024-08-17T17:38:32.103047+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:38:32.103047+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'2835","ondisk_log_start":"325'2835","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363462656,"num_objects":5546,"num_object_clones":0,"num_object_copies":16638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9386,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5546},{"shards":"1,6","objects":-1820}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":322,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.494722+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:38:09.494722+0000","last_peered":"2024-08-17T17:38:09.494722+0000","last_clean":"2024-08-17T17:38:09.494722+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:38:09.494722+0000","last_undegraded":"2024-08-17T17:38:09.494722+0000","last_fullsized":"2024-08-17T17:38:09.494722+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":348,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416543+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:38:09.416543+0000","last_peered":"2024-08-17T17:38:09.416543+0000","last_clean":"2024-08-17T17:38:09.416543+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:38:09.416543+0000","last_undegraded":"2024-08-17T17:38:09.416543+0000","last_fullsized":"2024-08-17T17:38:09.416543+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"367'9537","reported_seq":9651,"reported_epoch":367,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:33.829213+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:38:33.829213+0000","last_peered":"2024-08-17T17:38:33.829213+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:38:33.829213+0000","last_undegraded":"2024-08-17T17:38:33.829213+0000","last_fullsized":"2024-08-17T17:38:33.829213+0000","mapping_epoch":337,"log_start":"326'3961","ondisk_log_start":"326'3961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358678528,"num_objects":5473,"num_object_clones":0,"num_object_copies":16419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5473,"num_objects_unfound":0,"num_objects_dirty":5473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5473}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":359,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415782+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:38:09.415782+0000","last_peered":"2024-08-17T17:38:09.415782+0000","last_clean":"2024-08-17T17:38:09.415782+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:38:09.415782+0000","last_undegraded":"2024-08-17T17:38:09.415782+0000","last_fullsized":"2024-08-17T17:38:09.415782+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":321,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416768+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:38:09.416768+0000","last_peered":"2024-08-17T17:38:09.416768+0000","last_clean":"2024-08-17T17:38:09.416768+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:38:09.416768+0000","last_undegraded":"2024-08-17T17:38:09.416768+0000","last_fullsized":"2024-08-17T17:38:09.416768+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"367'9334","reported_seq":11971,"reported_epoch":367,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:32.326739+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:38:32.326739+0000","last_peered":"2024-08-17T17:38:32.326739+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:38:32.326739+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:38:32.326739+0000","mapping_epoch":337,"log_start":"324'2441","ondisk_log_start":"324'2441","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6893,"log_dups_size":0,"ondisk_log_size":6893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351928320,"num_objects":5370,"num_object_clones":0,"num_object_copies":16110,"num_objects_missing_on_primary":6065,"num_objects_missing":6065,"num_objects_degraded":5370,"num_objects_misplaced":6065,"num_objects_unfound":0,"num_objects_dirty":5370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9334,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1277,"num_bytes_recovered":83623936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":10},{"shards":"1,2,7","objects":5359},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":546,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426864+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:38:22.426864+0000","last_peered":"2024-08-17T17:38:22.426864+0000","last_clean":"2024-08-17T17:38:22.426864+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:38:22.426864+0000","last_undegraded":"2024-08-17T17:38:22.426864+0000","last_fullsized":"2024-08-17T17:38:22.426864+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":483,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427132+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:38:22.427132+0000","last_peered":"2024-08-17T17:38:22.427132+0000","last_clean":"2024-08-17T17:38:22.427132+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:38:22.427132+0000","last_undegraded":"2024-08-17T17:38:22.427132+0000","last_fullsized":"2024-08-17T17:38:22.427132+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"367'9446","reported_seq":12045,"reported_epoch":367,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:33.166049+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:38:33.166049+0000","last_peered":"2024-08-17T17:38:33.166049+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:38:33.166049+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:38:33.166049+0000","mapping_epoch":337,"log_start":"326'3844","ondisk_log_start":"326'3844","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357105688,"num_objects":5450,"num_object_clones":0,"num_object_copies":16350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5450,"num_objects_misplaced":6200,"num_objects_unfound":0,"num_objects_dirty":5450,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9446,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1248,"num_bytes_recovered":81723392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5450}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":677,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:22.495242+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:38:22.495242+0000","last_peered":"2024-08-17T17:38:22.495242+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:38:22.495242+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":296,"reported_epoch":366,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:38:09.381802+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:38:09.381802+0000","last_peered":"2024-08-17T17:38:09.381802+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:38:09.381802+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:38:09.381802+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":526,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426919+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:38:22.426919+0000","last_peered":"2024-08-17T17:38:22.426919+0000","last_clean":"2024-08-17T17:38:22.426919+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:38:22.426919+0000","last_undegraded":"2024-08-17T17:38:22.426919+0000","last_fullsized":"2024-08-17T17:38:22.426919+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.494915+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:38:09.494915+0000","last_peered":"2024-08-17T17:38:09.494915+0000","last_clean":"2024-08-17T17:38:09.494915+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:38:09.494915+0000","last_undegraded":"2024-08-17T17:38:09.494915+0000","last_fullsized":"2024-08-17T17:38:09.494915+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"367'9330","reported_seq":9446,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:28.982859+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:38:28.982859+0000","last_peered":"2024-08-17T17:38:28.982859+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:38:28.982859+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2321","ondisk_log_start":"324'2321","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7009,"log_dups_size":0,"ondisk_log_size":7009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354811904,"num_objects":5414,"num_object_clones":0,"num_object_copies":16242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9330,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1233},{"shards":"0,4,7","objects":5414}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":342,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427992+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:38:22.427992+0000","last_peered":"2024-08-17T17:38:22.427992+0000","last_clean":"2024-08-17T17:38:22.427992+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:38:22.427992+0000","last_undegraded":"2024-08-17T17:38:22.427992+0000","last_fullsized":"2024-08-17T17:38:22.427992+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":303,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415624+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:38:09.415624+0000","last_peered":"2024-08-17T17:38:09.415624+0000","last_clean":"2024-08-17T17:38:09.415624+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:38:09.415624+0000","last_undegraded":"2024-08-17T17:38:09.415624+0000","last_fullsized":"2024-08-17T17:38:09.415624+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":268,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.495884+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:38:22.495884+0000","last_peered":"2024-08-17T17:38:22.495884+0000","last_clean":"2024-08-17T17:38:22.495884+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:38:22.495884+0000","last_undegraded":"2024-08-17T17:38:22.495884+0000","last_fullsized":"2024-08-17T17:38:22.495884+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"367'9337","reported_seq":9403,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:31.353780+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:38:31.353780+0000","last_peered":"2024-08-17T17:38:31.353780+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:38:31.353780+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'2717","ondisk_log_start":"325'2717","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357761024,"num_objects":5459,"num_object_clones":0,"num_object_copies":16377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9337,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5459},{"shards":"5,6","objects":-1837}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":434,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427062+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:38:22.427062+0000","last_peered":"2024-08-17T17:38:22.427062+0000","last_clean":"2024-08-17T17:38:22.427062+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:38:22.427062+0000","last_undegraded":"2024-08-17T17:38:22.427062+0000","last_fullsized":"2024-08-17T17:38:22.427062+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":103,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.494259+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:38:09.494259+0000","last_peered":"2024-08-17T17:38:09.494259+0000","last_clean":"2024-08-17T17:38:09.494259+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:38:09.494259+0000","last_undegraded":"2024-08-17T17:38:09.494259+0000","last_fullsized":"2024-08-17T17:38:09.494259+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":619,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.428784+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:38:22.428784+0000","last_peered":"2024-08-17T17:38:22.428784+0000","last_clean":"2024-08-17T17:38:22.428784+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:38:22.428784+0000","last_undegraded":"2024-08-17T17:38:22.428784+0000","last_fullsized":"2024-08-17T17:38:22.428784+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"367'9242","reported_seq":10082,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:29.394804+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:38:29.394804+0000","last_peered":"2024-08-17T17:38:29.394804+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:38:29.394804+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2216","ondisk_log_start":"324'2216","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354156544,"num_objects":5404,"num_object_clones":0,"num_object_copies":16212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9242,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5404},{"shards":"0,5","objects":-1250}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":410,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.428395+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:38:22.428395+0000","last_peered":"2024-08-17T17:38:22.428395+0000","last_clean":"2024-08-17T17:38:22.428395+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:38:22.428395+0000","last_undegraded":"2024-08-17T17:38:22.428395+0000","last_fullsized":"2024-08-17T17:38:22.428395+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":342,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416259+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:38:09.416259+0000","last_peered":"2024-08-17T17:38:09.416259+0000","last_clean":"2024-08-17T17:38:09.416259+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:38:09.416259+0000","last_undegraded":"2024-08-17T17:38:09.416259+0000","last_fullsized":"2024-08-17T17:38:09.416259+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":205,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426964+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:38:22.426964+0000","last_peered":"2024-08-17T17:38:22.426964+0000","last_clean":"2024-08-17T17:38:22.426964+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:38:22.426964+0000","last_undegraded":"2024-08-17T17:38:22.426964+0000","last_fullsized":"2024-08-17T17:38:22.426964+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"367'9408","reported_seq":24367,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:30.232639+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:38:30.232639+0000","last_peered":"2024-08-17T17:38:30.232639+0000","last_clean":"2024-08-17T17:38:30.232639+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:38:30.232639+0000","last_undegraded":"2024-08-17T17:38:30.232639+0000","last_fullsized":"2024-08-17T17:38:30.232639+0000","mapping_epoch":355,"log_start":"325'3130","ondisk_log_start":"325'3130","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355729408,"num_objects":5428,"num_object_clones":0,"num_object_copies":16284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9408,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":281,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.382047+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:38:09.382047+0000","last_peered":"2024-08-17T17:38:09.382047+0000","last_clean":"2024-08-17T17:38:09.382047+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:38:09.382047+0000","last_undegraded":"2024-08-17T17:38:09.382047+0000","last_fullsized":"2024-08-17T17:38:09.382047+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":74,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426752+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:38:22.426752+0000","last_peered":"2024-08-17T17:38:22.426752+0000","last_clean":"2024-08-17T17:38:22.426752+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:38:22.426752+0000","last_undegraded":"2024-08-17T17:38:22.426752+0000","last_fullsized":"2024-08-17T17:38:22.426752+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":569,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.428535+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:38:22.428535+0000","last_peered":"2024-08-17T17:38:22.428535+0000","last_clean":"2024-08-17T17:38:22.428535+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:38:22.428535+0000","last_undegraded":"2024-08-17T17:38:22.428535+0000","last_fullsized":"2024-08-17T17:38:22.428535+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"367'9116","reported_seq":23643,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:30.213216+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:38:30.213216+0000","last_peered":"2024-08-17T17:38:30.213216+0000","last_clean":"2024-08-17T17:38:30.213216+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:38:30.213216+0000","last_undegraded":"2024-08-17T17:38:30.213216+0000","last_fullsized":"2024-08-17T17:38:30.213216+0000","mapping_epoch":345,"log_start":"325'2772","ondisk_log_start":"325'2772","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346554368,"num_objects":5288,"num_object_clones":0,"num_object_copies":15864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9116,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":277,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381979+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:38:09.381979+0000","last_peered":"2024-08-17T17:38:09.381979+0000","last_clean":"2024-08-17T17:38:09.381979+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:38:09.381979+0000","last_undegraded":"2024-08-17T17:38:09.381979+0000","last_fullsized":"2024-08-17T17:38:09.381979+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":325,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.495183+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:38:22.495183+0000","last_peered":"2024-08-17T17:38:22.495183+0000","last_clean":"2024-08-17T17:38:22.495183+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:38:22.495183+0000","last_undegraded":"2024-08-17T17:38:22.495183+0000","last_fullsized":"2024-08-17T17:38:22.495183+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":419,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381687+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:38:09.381687+0000","last_peered":"2024-08-17T17:38:09.381687+0000","last_clean":"2024-08-17T17:38:09.381687+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:38:09.381687+0000","last_undegraded":"2024-08-17T17:38:09.381687+0000","last_fullsized":"2024-08-17T17:38:09.381687+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"367'9393","reported_seq":9528,"reported_epoch":367,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:30.398817+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:38:30.398817+0000","last_peered":"2024-08-17T17:38:30.398817+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:38:30.398817+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'3087","ondisk_log_start":"325'3087","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6306,"log_dups_size":0,"ondisk_log_size":6306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353304576,"num_objects":5391,"num_object_clones":0,"num_object_copies":16173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9393,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5391},{"shards":"0,3","objects":-1857}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":599,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427190+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:38:22.427190+0000","last_peered":"2024-08-17T17:38:22.427190+0000","last_clean":"2024-08-17T17:38:22.427190+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:38:22.427190+0000","last_undegraded":"2024-08-17T17:38:22.427190+0000","last_fullsized":"2024-08-17T17:38:22.427190+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":568,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.428784+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:38:22.428784+0000","last_peered":"2024-08-17T17:38:22.428784+0000","last_clean":"2024-08-17T17:38:22.428784+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:38:22.428784+0000","last_undegraded":"2024-08-17T17:38:22.428784+0000","last_fullsized":"2024-08-17T17:38:22.428784+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":585,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426685+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:38:22.426685+0000","last_peered":"2024-08-17T17:38:22.426685+0000","last_clean":"2024-08-17T17:38:22.426685+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:38:22.426685+0000","last_undegraded":"2024-08-17T17:38:22.426685+0000","last_fullsized":"2024-08-17T17:38:22.426685+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"367'9293","reported_seq":23957,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:32.056820+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:38:32.056820+0000","last_peered":"2024-08-17T17:38:32.056820+0000","last_clean":"2024-08-17T17:38:32.056820+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:38:32.056820+0000","last_undegraded":"2024-08-17T17:38:32.056820+0000","last_fullsized":"2024-08-17T17:38:32.056820+0000","mapping_epoch":347,"log_start":"325'2672","ondisk_log_start":"325'2672","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346095616,"num_objects":5281,"num_object_clones":0,"num_object_copies":15843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9293,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":258,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.495511+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:38:22.495511+0000","last_peered":"2024-08-17T17:38:22.495511+0000","last_clean":"2024-08-17T17:38:22.495511+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:38:22.495511+0000","last_undegraded":"2024-08-17T17:38:22.495511+0000","last_fullsized":"2024-08-17T17:38:22.495511+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":371,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.495035+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:38:09.495035+0000","last_peered":"2024-08-17T17:38:09.495035+0000","last_clean":"2024-08-17T17:38:09.495035+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:38:09.495035+0000","last_undegraded":"2024-08-17T17:38:09.495035+0000","last_fullsized":"2024-08-17T17:38:09.495035+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":321,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415711+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:38:09.415711+0000","last_peered":"2024-08-17T17:38:09.415711+0000","last_clean":"2024-08-17T17:38:09.415711+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:38:09.415711+0000","last_undegraded":"2024-08-17T17:38:09.415711+0000","last_fullsized":"2024-08-17T17:38:09.415711+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":321,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416138+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:38:09.416138+0000","last_peered":"2024-08-17T17:38:09.416138+0000","last_clean":"2024-08-17T17:38:09.416138+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:38:09.416138+0000","last_undegraded":"2024-08-17T17:38:09.416138+0000","last_fullsized":"2024-08-17T17:38:09.416138+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":364,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416292+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:38:09.416292+0000","last_peered":"2024-08-17T17:38:09.416292+0000","last_clean":"2024-08-17T17:38:09.416292+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:38:09.416292+0000","last_undegraded":"2024-08-17T17:38:09.416292+0000","last_fullsized":"2024-08-17T17:38:09.416292+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":575,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.428436+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:38:22.428436+0000","last_peered":"2024-08-17T17:38:22.428436+0000","last_clean":"2024-08-17T17:38:22.428436+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:38:22.428436+0000","last_undegraded":"2024-08-17T17:38:22.428436+0000","last_fullsized":"2024-08-17T17:38:22.428436+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":115,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427439+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:38:22.427439+0000","last_peered":"2024-08-17T17:38:22.427439+0000","last_clean":"2024-08-17T17:38:22.427439+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:38:22.427439+0000","last_undegraded":"2024-08-17T17:38:22.427439+0000","last_fullsized":"2024-08-17T17:38:22.427439+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":271,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.381862+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:38:09.381862+0000","last_peered":"2024-08-17T17:38:09.381862+0000","last_clean":"2024-08-17T17:38:09.381862+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:38:09.381862+0000","last_undegraded":"2024-08-17T17:38:09.381862+0000","last_fullsized":"2024-08-17T17:38:09.381862+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":533,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426703+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:38:22.426703+0000","last_peered":"2024-08-17T17:38:22.426703+0000","last_clean":"2024-08-17T17:38:22.426703+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:38:22.426703+0000","last_undegraded":"2024-08-17T17:38:22.426703+0000","last_fullsized":"2024-08-17T17:38:22.426703+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":415,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.382307+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:38:09.382307+0000","last_peered":"2024-08-17T17:38:09.382307+0000","last_clean":"2024-08-17T17:38:09.382307+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:38:09.382307+0000","last_undegraded":"2024-08-17T17:38:09.382307+0000","last_fullsized":"2024-08-17T17:38:09.382307+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":527,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426575+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:38:22.426575+0000","last_peered":"2024-08-17T17:38:22.426575+0000","last_clean":"2024-08-17T17:38:22.426575+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:38:22.426575+0000","last_undegraded":"2024-08-17T17:38:22.426575+0000","last_fullsized":"2024-08-17T17:38:22.426575+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":338,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.494995+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:38:09.494995+0000","last_peered":"2024-08-17T17:38:09.494995+0000","last_clean":"2024-08-17T17:38:09.494995+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:38:09.494995+0000","last_undegraded":"2024-08-17T17:38:09.494995+0000","last_fullsized":"2024-08-17T17:38:09.494995+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":558,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.426770+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:38:22.426770+0000","last_peered":"2024-08-17T17:38:22.426770+0000","last_clean":"2024-08-17T17:38:22.426770+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:38:22.426770+0000","last_undegraded":"2024-08-17T17:38:22.426770+0000","last_fullsized":"2024-08-17T17:38:22.426770+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":274,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427597+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:38:22.427597+0000","last_peered":"2024-08-17T17:38:22.427597+0000","last_clean":"2024-08-17T17:38:22.427597+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:38:22.427597+0000","last_undegraded":"2024-08-17T17:38:22.427597+0000","last_fullsized":"2024-08-17T17:38:22.427597+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":566,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.428359+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:38:22.428359+0000","last_peered":"2024-08-17T17:38:22.428359+0000","last_clean":"2024-08-17T17:38:22.428359+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:38:22.428359+0000","last_undegraded":"2024-08-17T17:38:22.428359+0000","last_fullsized":"2024-08-17T17:38:22.428359+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":526,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427184+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:38:22.427184+0000","last_peered":"2024-08-17T17:38:22.427184+0000","last_clean":"2024-08-17T17:38:22.427184+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:38:22.427184+0000","last_undegraded":"2024-08-17T17:38:22.427184+0000","last_fullsized":"2024-08-17T17:38:22.427184+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":350,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415982+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:38:09.415982+0000","last_peered":"2024-08-17T17:38:09.415982+0000","last_clean":"2024-08-17T17:38:09.415982+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:38:09.415982+0000","last_undegraded":"2024-08-17T17:38:09.415982+0000","last_fullsized":"2024-08-17T17:38:09.415982+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":215,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427239+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:38:22.427239+0000","last_peered":"2024-08-17T17:38:22.427239+0000","last_clean":"2024-08-17T17:38:22.427239+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:38:22.427239+0000","last_undegraded":"2024-08-17T17:38:22.427239+0000","last_fullsized":"2024-08-17T17:38:22.427239+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":271,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.495514+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:38:22.495514+0000","last_peered":"2024-08-17T17:38:22.495514+0000","last_clean":"2024-08-17T17:38:22.495514+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:38:22.495514+0000","last_undegraded":"2024-08-17T17:38:22.495514+0000","last_fullsized":"2024-08-17T17:38:22.495514+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":358,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416070+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:38:09.416070+0000","last_peered":"2024-08-17T17:38:09.416070+0000","last_clean":"2024-08-17T17:38:09.416070+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:38:09.416070+0000","last_undegraded":"2024-08-17T17:38:09.416070+0000","last_fullsized":"2024-08-17T17:38:09.416070+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":221,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416214+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:38:09.416214+0000","last_peered":"2024-08-17T17:38:09.416214+0000","last_clean":"2024-08-17T17:38:09.416214+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:38:09.416214+0000","last_undegraded":"2024-08-17T17:38:09.416214+0000","last_fullsized":"2024-08-17T17:38:09.416214+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":358,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415851+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:38:09.415851+0000","last_peered":"2024-08-17T17:38:09.415851+0000","last_clean":"2024-08-17T17:38:09.415851+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:38:09.415851+0000","last_undegraded":"2024-08-17T17:38:09.415851+0000","last_fullsized":"2024-08-17T17:38:09.415851+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":221,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.416459+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:38:09.416459+0000","last_peered":"2024-08-17T17:38:09.416459+0000","last_clean":"2024-08-17T17:38:09.416459+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:38:09.416459+0000","last_undegraded":"2024-08-17T17:38:09.416459+0000","last_fullsized":"2024-08-17T17:38:09.416459+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":578,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427457+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:38:22.427457+0000","last_peered":"2024-08-17T17:38:22.427457+0000","last_clean":"2024-08-17T17:38:22.427457+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:38:22.427457+0000","last_undegraded":"2024-08-17T17:38:22.427457+0000","last_fullsized":"2024-08-17T17:38:22.427457+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":551,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.427682+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:38:22.427682+0000","last_peered":"2024-08-17T17:38:22.427682+0000","last_clean":"2024-08-17T17:38:22.427682+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:38:22.427682+0000","last_undegraded":"2024-08-17T17:38:22.427682+0000","last_fullsized":"2024-08-17T17:38:22.427682+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":321,"reported_epoch":366,"state":"active+clean","last_fresh":"2024-08-17T17:38:09.415716+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:38:09.415716+0000","last_peered":"2024-08-17T17:38:09.415716+0000","last_clean":"2024-08-17T17:38:09.415716+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:38:09.415716+0000","last_undegraded":"2024-08-17T17:38:09.415716+0000","last_fullsized":"2024-08-17T17:38:09.415716+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":273,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.495603+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:38:22.495603+0000","last_peered":"2024-08-17T17:38:22.495603+0000","last_clean":"2024-08-17T17:38:22.495603+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:38:22.495603+0000","last_undegraded":"2024-08-17T17:38:22.495603+0000","last_fullsized":"2024-08-17T17:38:22.495603+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5682626584,"num_objects":86711,"num_object_clones":0,"num_object_copies":260133,"num_objects_missing_on_primary":6065,"num_objects_missing":6065,"num_objects_degraded":55239,"num_objects_misplaced":34053,"num_objects_unfound":0,"num_objects_dirty":86711,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149679,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26085,"num_bytes_recovered":1586954240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18410123264,"data_stored":18410111048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104698,"ondisk_log_size":104698,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170065,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463435,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3322824,"kb_used_data":2424960,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":90422328,"statfs":{"total":95995035648,"available":92592463872,"internally_reserved":0,"allocated":2483159040,"data_stored":2481198770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492109,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1100332,"kb_used_data":700836,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92644820,"statfs":{"total":95995035648,"available":94868295680,"internally_reserved":0,"allocated":717656064,"data_stored":715718094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986701,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2564604,"kb_used_data":1379444,"kb_used_omap":1,"kb_used_meta":1185150,"kb_avail":91180548,"statfs":{"total":95995035648,"available":93368881152,"internally_reserved":0,"allocated":1412550656,"data_stored":1410601326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213593944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,3,3,8,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225528,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3690284,"kb_used_data":2778724,"kb_used_omap":4,"kb_used_meta":911547,"kb_avail":90054868,"statfs":{"total":95995035648,"available":92216184832,"internally_reserved":0,"allocated":2845413376,"data_stored":2843475704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":933424576},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075853,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533747,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3469680,"kb_used_data":2427688,"kb_used_omap":5,"kb_used_meta":1041978,"kb_avail":90275472,"statfs":{"total":95995035648,"available":92442083328,"internally_reserved":0,"allocated":2485952512,"data_stored":2484009518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1066985712},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089723,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3089096,"kb_used_data":2097920,"kb_used_omap":5,"kb_used_meta":991162,"kb_avail":90656056,"statfs":{"total":95995035648,"available":92831801344,"internally_reserved":0,"allocated":2148270080,"data_stored":2146309869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1014949947},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2142437376,"data_stored":2142437376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2479886336,"data_stored":2479882264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3682078720,"data_stored":3682074648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2836463616,"data_stored":2836463616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1407320064,"data_stored":1407320064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":712900608,"data_stored":712900608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2476998656,"data_stored":2476998656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2672037888,"data_stored":2672033816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:38.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:39.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:39.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:39.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:39.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:39.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:39.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:38:39.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:39.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:39.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:39.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:38:39.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:39.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:39.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:39.524 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:39 smithi142 ceph-mon[106729]: pgmap v1350: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 55213/258936 objects degraded (21.323%); 33944/258936 objects misplaced (13.109%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:39.524 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:39 smithi142 ceph-mon[103614]: pgmap v1350: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 55213/258936 objects degraded (21.323%); 33944/258936 objects misplaced (13.109%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:39.562 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:39.570 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:39.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:39 smithi152 ceph-mon[93456]: pgmap v1350: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 55213/258936 objects degraded (21.323%); 33944/258936 objects misplaced (13.109%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:38:39.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:39.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:39.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:39.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:39.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:39.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:39.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:39.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:40.009 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:40.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:40.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:40.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:40.280 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:40.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:40.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:38:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:38:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:40.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:40.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:40.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:40.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:40.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:40.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:40.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:40.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:40.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:40.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:40.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:41.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:41.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:41.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:41.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:41.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:41.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:41.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:41.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:41.450 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[103614]: pgmap v1351: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 55196/258570 objects degraded (21.347%); 33901/258570 objects misplaced (13.111%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:38:41.450 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55196/258570 objects degraded (21.347%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:41.450 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:41.450 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:41.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:41.483 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:41.484 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:41.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:41 smithi152 ceph-mon[93456]: pgmap v1351: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 55196/258570 objects degraded (21.347%); 33901/258570 objects misplaced (13.111%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:38:41.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55196/258570 objects degraded (21.347%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:41.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:41.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:41.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:41.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:41.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:41.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:41.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[106729]: pgmap v1351: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 55196/258570 objects degraded (21.347%); 33901/258570 objects misplaced (13.111%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:38:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55196/258570 objects degraded (21.347%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:41.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:41.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:41.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:41.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:41.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:41.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:41.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:42.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:42.116 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:38:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:38:41] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:38:42.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:42.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:42.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:42.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:42.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:38:42.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:42.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:42.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:42.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:42.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:42.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:42.703 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:42.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:42.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:42.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:42.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:42.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:43.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:43.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:43.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:43.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:43.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:43.372 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:43.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.469 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:38:43.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:43 smithi142 ceph-mon[103614]: pgmap v1352: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55196/257895 objects degraded (21.403%); 33874/257895 objects misplaced (13.135%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:38:43.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1247224117' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:43.470 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:43 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:43.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:43 smithi142 ceph-mon[106729]: pgmap v1352: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55196/257895 objects degraded (21.403%); 33874/257895 objects misplaced (13.135%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:38:43.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1247224117' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:43.470 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:43 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:43.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:43.594 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:43.696 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:43 smithi152 ceph-mon[93456]: pgmap v1352: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 55196/257895 objects degraded (21.403%); 33874/257895 objects misplaced (13.135%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:38:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1247224117' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:43 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:43.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:43.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:43.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:44.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:44.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:44.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:44.257 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:38:44.257 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:44.258 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:44.259 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:38:44.259 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:38:44.259 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:38:44.259 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:44.259 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:44.259 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:44.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:44.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:44.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:44.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:44.475 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:38:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:38:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:38:44.475 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:38:44.476 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:38:44.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:44 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:44.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:44 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:44.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:44 smithi152 ceph-mon[93456]: osdmap e369: 8 total, 8 up, 6 in 2024-08-17T17:38:44.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2199680846' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:44.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:44.595 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:44.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:44.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:44.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:44.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[103614]: osdmap e369: 8 total, 8 up, 6 in 2024-08-17T17:38:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2199680846' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[106729]: osdmap e369: 8 total, 8 up, 6 in 2024-08-17T17:38:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2199680846' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:44.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:44.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:45.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:38:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:45.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:45.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:45.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:45.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:45.539 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:45.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:45.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:45.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:45.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:45.729 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:45 smithi152 ceph-mon[93456]: pgmap v1354: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 55142/257565 objects degraded (21.409%); 33763/257565 objects misplaced (13.109%); 955 KiB/s, 14 objects/s recovering 2024-08-17T17:38:45.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:45.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:45 smithi142 ceph-mon[106729]: pgmap v1354: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 55142/257565 objects degraded (21.409%); 33763/257565 objects misplaced (13.109%); 955 KiB/s, 14 objects/s recovering 2024-08-17T17:38:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:45 smithi142 ceph-mon[103614]: pgmap v1354: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 55142/257565 objects degraded (21.409%); 33763/257565 objects misplaced (13.109%); 955 KiB/s, 14 objects/s recovering 2024-08-17T17:38:45.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.964 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:45.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:46.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:46.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:46.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:38:46.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:46.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:46.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:46.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:46.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:46.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:46.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:46.587 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55142/257565 objects degraded (21.409%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:46.587 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55142/257565 objects degraded (21.409%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:46.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:46.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:46.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:46.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55142/257565 objects degraded (21.409%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:46.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:46.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:46.923 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:46.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:46.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:46.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:46.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:47.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:47.090 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:38:47.091 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:38:47.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:47.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:47.234 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:47.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:47.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:47.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:47.408 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:38:47.408 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:38:47.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:38:47.409 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:38:47.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.409 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129835 12.7 12:e447ee12:::benchmark_data_smithi142_261410_object11638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.409 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.539141+0000", 2024-08-17T17:38:47.409 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 262.866365395, 2024-08-17T17:38:47.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9776896339999999, 2024-08-17T17:38:47.410 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.410 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.410 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.410 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129835 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.539141+0000", 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.539141+0000", 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.539139+0000", 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.539149+0000", 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0186999999999999e-05 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.539171+0000", 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.207e-05 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.539176+0000", 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7520000000000004e-06 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.559651+0000", 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0204746659999999 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.576422+0000", 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016770956 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.824419+0000", 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24799732199999999 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.840044+0000", 2024-08-17T17:38:47.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015624702000000001 2024-08-17T17:38:47.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.324868+0000", 2024-08-17T17:38:47.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.484824063 2024-08-17T17:38:47.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.345641+0000", 2024-08-17T17:38:47.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020773301000000001 2024-08-17T17:38:47.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.374976+0000", 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.029334981999999999 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.455511+0000", 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.080535161999999993 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.461865+0000", 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063540699999999999 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.465556+0000", 2024-08-17T17:38:47.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00369053 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.470346+0000", 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047902559999999997 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.478842+0000", 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084957939999999992 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.516831+0000", 2024-08-17T17:38:47.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037988839000000003 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129838 12.7 12:e3bc9699:::benchmark_data_smithi142_261410_object11641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.423 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.545127+0000", 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 262.86037985399997, 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9085866139999998, 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.424 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129838 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.545127+0000", 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.545127+0000", 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.545126+0000", 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.545130+0000", 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9709999999999998e-06 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.545136+0000", 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.939e-06 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.545140+0000", 2024-08-17T17:38:47.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8809999999999998e-06 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.578572+0000", 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.03343122 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.593636+0000", 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015063976999999999 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.740905+0000", 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14726994199999999 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.763672+0000", 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022766946 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.209405+0000", 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4457326849999999 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.224629+0000", 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015223873000000001 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.262814+0000", 2024-08-17T17:38:47.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038185087999999999 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.398499+0000", 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.135684798 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.400235+0000", 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001736015 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.404929+0000", 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046945779999999996 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.409466+0000", 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004536165 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.416574+0000", 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071082200000000002 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.453713+0000", 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037139450999999997 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129858 12.7 12:ecd1f66f:::benchmark_data_smithi142_261410_object11661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.677268+0000", 2024-08-17T17:38:47.433 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 262.72823837700003, 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2253638670000004, 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129858 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.434 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.677268+0000", 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.677268+0000", 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.677266+0000", 2024-08-17T17:38:47.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.677274+0000", 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.126e-06 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.677288+0000", 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4151999999999999e-05 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.677294+0000", 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.981e-06 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.846108+0000", 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1688139870000001 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.860520+0000", 2024-08-17T17:38:47.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014412428999999999 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.006633+0000", 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14611306299999999 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.022783+0000", 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016150129999999999 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.590332+0000", 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.567548699 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.602369+0000", 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01203684 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.628044+0000", 2024-08-17T17:38:47.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025675058000000001 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.723829+0000", 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.095785281 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.729362+0000", 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005532866 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.735677+0000", 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063154049999999996 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.841729+0000", 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10605155400000001 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.853169+0000", 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011440014 2024-08-17T17:38:47.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.902632+0000", 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.049463009000000002 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129865 12.7 12:e0b9a064:::benchmark_data_smithi142_261410_object11668:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.993298+0000", 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 262.41220826, 2024-08-17T17:38:47.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3358309940000002, 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129865 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993298+0000", 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.443 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993298+0000", 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993297+0000", 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993303+0000", 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0820000000000004e-06 2024-08-17T17:38:47.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993315+0000", 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1936999999999999e-05 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993321+0000", 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5729999999999998e-06 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.861914+0000", 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86859384500000003 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.876600+0000", 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014685807 2024-08-17T17:38:47.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.989060+0000", 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.1124595 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.004576+0000", 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015516162 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.985942+0000", 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.981366102 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.011692+0000", 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025749648 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.052098+0000", 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040406769000000002 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.151725+0000", 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.099627131999999993 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.157571+0000", 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058450790000000004 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.159422+0000", 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018513749999999999 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.266755+0000", 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.107333265 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.276951+0000", 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010195549 2024-08-17T17:38:47.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.329129+0000", 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.052178537999999997 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.452 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129869 12.7 12:e4c5b74a:::benchmark_data_smithi142_261410_object11672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:25.016723+0000", 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 262.38878367199999, 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5390109890000003, 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.453 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129869 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.016723+0000", 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.016723+0000", 2024-08-17T17:38:47.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.016721+0000", 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.016730+0000", 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4440000000000008e-06 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.016744+0000", 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4504e-05 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:25.016752+0000", 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2679999999999994e-06 2024-08-17T17:38:47.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.031827+0000", 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.015074265 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.046536+0000", 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01470893 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.207041+0000", 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16050548100000001 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.224432+0000", 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017390994 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.382381+0000", 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1579488250000001 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.400688+0000", 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018307535999999999 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.436229+0000", 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.035540729 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.482555+0000", 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046325929000000002 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.494369+0000", 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01181369 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.499913+0000", 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055438900000000001 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.502108+0000", 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021949230000000001 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.509288+0000", 2024-08-17T17:38:47.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071805059999999997 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.555734+0000", 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046445699 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129911 12.7 12:ebb38d4a:::benchmark_data_smithi142_261410_object11714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:26.699485+0000", 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 260.70602165600002, 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.657954438, 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129911 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.463 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.699485+0000", 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.699485+0000", 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.699483+0000", 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.699491+0000", 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7549999999999994e-06 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.699506+0000", 2024-08-17T17:38:47.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4630000000000001e-05 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.699513+0000", 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4850000000000003e-06 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.958088+0000", 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25857479500000002 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:26.973429+0000", 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015341319000000001 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.309155+0000", 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33572561499999998 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.327126+0000", 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017970634999999999 2024-08-17T17:38:47.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.126501+0000", 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7993752609999998 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.144739+0000", 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018238351 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.178027+0000", 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.033287803999999997 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.279093+0000", 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.101065564 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.289827+0000", 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010734168000000001 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.297963+0000", 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081366000000000008 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.306953+0000", 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089898579999999999 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.318133+0000", 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011179863999999999 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.357439+0000", 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039306214999999999 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129948 12.7 12:e54ccaa3:::benchmark_data_smithi142_261410_object11751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.471 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:27.797411+0000", 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 259.60809514200002, 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4502519400000002, 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129948 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.797411+0000", 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.797411+0000", 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.797410+0000", 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.797417+0000", 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4390000000000003e-06 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.797429+0000", 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1678999999999999e-05 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.797434+0000", 2024-08-17T17:38:47.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4880000000000001e-06 2024-08-17T17:38:47.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.913581+0000", 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11614748599999999 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:27.932537+0000", 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018956082999999999 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.134368+0000", 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20183035399999999 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.150992+0000", 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016624346000000002 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.042417+0000", 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8914246299999999 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.056753+0000", 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014336972 2024-08-17T17:38:47.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.087369+0000", 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030615034999999999 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.179442+0000", 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.092073817000000002 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.183097+0000", 2024-08-17T17:38:47.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036545100000000001 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.187307+0000", 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042105529999999997 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.202922+0000", 2024-08-17T17:38:47.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015614697 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.209084+0000", 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061619489999999999 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.247663+0000", 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038579302000000003 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.482 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129964 12.7 12:ea20d991:::benchmark_data_smithi142_261410_object11767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:28.008017+0000", 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 259.39748908500002, 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5036716859999997, 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129964 2024-08-17T17:38:47.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.008017+0000", 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.008017+0000", 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.008016+0000", 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.008023+0000", 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7290000000000003e-06 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.008036+0000", 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3704e-05 2024-08-17T17:38:47.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.008043+0000", 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1510000000000001e-06 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.213377+0000", 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.205334608 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.232725+0000", 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01934789 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.414833+0000", 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18210780300000001 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:28.430569+0000", 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015736515999999999 2024-08-17T17:38:47.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.295484+0000", 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8649143810000002 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.308055+0000", 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012570967000000001 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.332991+0000", 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024936234000000002 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.360893+0000", 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027901657 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.456857+0000", 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.095964241000000006 2024-08-17T17:38:47.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.461710+0000", 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048535820000000004 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.465996+0000", 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042858319999999998 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.472305+0000", 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063090860000000002 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.511689+0000", 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039383728999999999 2024-08-17T17:38:47.491 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130045 12.7 12:ea981e1d:::benchmark_data_smithi142_261410_object11848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:29.657694+0000", 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 257.74781215600001, 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4545502319999999, 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130045 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.657694+0000", 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.657694+0000", 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.657693+0000", 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.657700+0000", 2024-08-17T17:38:47.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.657713+0000", 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2957e-05 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.657718+0000", 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4990000000000002e-06 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.904705+0000", 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.246987022 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.919670+0000", 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014965002 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.125562+0000", 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20589143200000001 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.147427+0000", 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021865595000000002 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.983260+0000", 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8358329869999999 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.994303+0000", 2024-08-17T17:38:47.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011042573999999999 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.017055+0000", 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022751674999999999 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.059453+0000", 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042397899000000003 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.062873+0000", 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003420845 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.067957+0000", 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050835250000000002 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.069176+0000", 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012187859999999999 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.076270+0000", 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070946029999999997 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.112245+0000", 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.035974320999999997 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.500 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130051 12.7 12:e9c41fa1:::benchmark_data_smithi142_261410_object11854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:29.894610+0000", 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 257.51089633200002, 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5993367660000004, 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130051 2024-08-17T17:38:47.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.894610+0000", 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.894610+0000", 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.894608+0000", 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.894616+0000", 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4680000000000003e-06 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.894630+0000", 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.396e-05 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.894637+0000", 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1550000000000001e-06 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.922058+0000", 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027421529 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:29.940469+0000", 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018410255 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.231697+0000", 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29122809900000002 2024-08-17T17:38:47.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:30.257662+0000", 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025964779 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.219125+0000", 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9614631550000001 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.232178+0000", 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013053011 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.257196+0000", 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025017894999999998 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.386940+0000", 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.129744797 2024-08-17T17:38:47.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.440367+0000", 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.053426986000000003 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.444884+0000", 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045160590000000002 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.449220+0000", 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043364750000000002 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.457128+0000", 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079079400000000005 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.493947+0000", 2024-08-17T17:38:47.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036819052999999997 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130105 12.7 12:ecc7c3f5:::benchmark_data_smithi142_261410_object11908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:31.401052+0000", 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.00445434099998, 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4837566820000001, 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.510 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130105 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.401052+0000", 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.401052+0000", 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.401050+0000", 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.401061+0000", 2024-08-17T17:38:47.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0531e-05 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.401079+0000", 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7748999999999999e-05 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.401087+0000", 2024-08-17T17:38:47.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6170000000000007e-06 2024-08-17T17:38:47.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.701679+0000", 2024-08-17T17:38:47.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30059180200000002 2024-08-17T17:38:47.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.716085+0000", 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014406524 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.893232+0000", 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.17714611399999999 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.909237+0000", 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016005156999999999 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.749385+0000", 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8401478959999999 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.762009+0000", 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01262424 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.786750+0000", 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024741295999999999 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.828561+0000", 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.041811214999999999 2024-08-17T17:38:47.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.829812+0000", 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012511320000000001 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.835045+0000", 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052322080000000003 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.840136+0000", 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005091567 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.846547+0000", 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064110859999999999 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:35.884809+0000", 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038261529000000002 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130117 12.7 12:e06552e6:::benchmark_data_smithi142_261410_object11920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:31.749808+0000", 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 255.65569823600001, 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5067379709999997, 2024-08-17T17:38:47.521 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130117 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.749808+0000", 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.749808+0000", 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.749807+0000", 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.749812+0000", 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1120000000000004e-06 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.749822+0000", 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.55e-06 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.749826+0000", 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9519999999999996e-06 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:31.978754+0000", 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.228928417 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.001646+0000", 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022891326 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.162206+0000", 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16056034599999999 2024-08-17T17:38:47.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.175232+0000", 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013025993 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.016230+0000", 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8409981050000002 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.029030+0000", 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012799642999999999 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.057530+0000", 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028500343000000001 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.165142+0000", 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10761180100000001 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.169589+0000", 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044471290000000002 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.174712+0000", 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051231970000000003 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.216705+0000", 2024-08-17T17:38:47.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.041992663999999999 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.223920+0000", 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072156210000000002 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.256546+0000", 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.032625933000000003 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.529 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130126 12.7 12:e6678d02:::benchmark_data_smithi142_261410_object11929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:32.000377+0000", 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 255.40512917800001, 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4984574500000001, 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.530 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130126 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.000377+0000", 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.000377+0000", 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.000376+0000", 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.000383+0000", 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2219999999999996e-06 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.000396+0000", 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3570999999999999e-05 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.000403+0000", 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.421e-06 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.188970+0000", 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18856746999999999 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.201598+0000", 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012628210000000001 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.423469+0000", 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.221870174 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:32.438130+0000", 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014661805999999999 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.262756+0000", 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.824625229 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.283706+0000", 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020950643000000001 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.321211+0000", 2024-08-17T17:38:47.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037505157999999997 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.444674+0000", 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.123462979 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.446387+0000", 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017122229999999999 2024-08-17T17:38:47.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.450935+0000", 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045482730000000002 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.454710+0000", 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003774754 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.462580+0000", 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078702330000000008 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:36.498835+0000", 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036254911000000001 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.539 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130178 12.7 12:e52d4755:::benchmark_data_smithi142_261410_object11981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.539 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:33.063903+0000", 2024-08-17T17:38:47.539 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 254.34160394899999, 2024-08-17T17:38:47.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4599355489999999, 2024-08-17T17:38:47.539 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.540 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.540 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130178 2024-08-17T17:38:47.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.063903+0000", 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.063903+0000", 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.063901+0000", 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.063908+0000", 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0539999999999997e-06 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.063922+0000", 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3959e-05 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.063930+0000", 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2099999999999993e-06 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.347925+0000", 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28399432200000002 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.361449+0000", 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013524259 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.432710+0000", 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.071261492999999995 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.448169+0000", 2024-08-17T17:38:47.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015458614000000001 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.322246+0000", 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8740774890000003 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.334885+0000", 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012638646 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.362385+0000", 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027499487 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.453683+0000", 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.091298282999999994 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.459351+0000", 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056685720000000002 2024-08-17T17:38:47.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.462986+0000", 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036345029999999999 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.468212+0000", 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052258670000000004 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.477491+0000", 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092793029999999992 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.523838+0000", 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046347070999999997 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.548 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130190 12.7 12:ecbdfe19:::benchmark_data_smithi142_261410_object11993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:33.276068+0000", 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 254.12943891399999, 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5216707060000001, 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130190 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.276068+0000", 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.276068+0000", 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.276066+0000", 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.276074+0000", 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9209999999999992e-06 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.276086+0000", 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2172e-05 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.276092+0000", 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8579999999999996e-06 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.366581+0000", 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.090489599000000004 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.380019+0000", 2024-08-17T17:38:47.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013437976000000001 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.512749+0000", 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13272935 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.526565+0000", 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013815768000000001 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.674283+0000", 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1477180100000002 2024-08-17T17:38:47.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.687171+0000", 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012887991 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.711033+0000", 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023862429000000001 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.750415+0000", 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039382031999999997 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.751727+0000", 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013122050000000001 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.755672+0000", 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039443020000000002 2024-08-17T17:38:47.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.760775+0000", 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051032819999999998 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.767020+0000", 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062450070000000003 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:37.797738+0000", 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030718523000000001 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:130205 12.7 12:ef86d0d3:::benchmark_data_smithi142_261410_object12008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:33.972156+0000", 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 253.43335013000001, 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.477586015, 2024-08-17T17:38:47.558 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 130205 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.972156+0000", 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.972156+0000", 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.972155+0000", 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.972162+0000", 2024-08-17T17:38:47.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.674e-06 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.972174+0000", 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2244e-05 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:33.972180+0000", 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6999999999999996e-06 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.042403+0000", 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.070223457000000003 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.055348+0000", 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012945190000000001 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.166782+0000", 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.111434123 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:34.182609+0000", 2024-08-17T17:38:47.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015826946000000001 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.187166+0000", 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0045567970000002 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.207980+0000", 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020813518999999999 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.248170+0000", 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040190858000000003 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.393937+0000", 2024-08-17T17:38:47.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.145766855 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.398594+0000", 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004656426 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.400009+0000", 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001415562 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.405296+0000", 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052865220000000001 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.411338+0000", 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060417099999999996 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:38.449742+0000", 2024-08-17T17:38:47.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038404922000000001 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131066 12.7 12:e40cd3d0:::benchmark_data_smithi142_261410_object12869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:52.712386+0000", 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 234.693120588, 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6989075009999999, 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.569 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131066 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:52.712386+0000", 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:52.712386+0000", 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:52.712384+0000", 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:52.712392+0000", 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0679999999999997e-06 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:52.712403+0000", 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0902e-05 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:52.712409+0000", 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0920000000000003e-06 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.065630+0000", 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35322105399999998 2024-08-17T17:38:47.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.084546+0000", 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018915392 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.246221+0000", 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16167545699999999 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.262125+0000", 2024-08-17T17:38:47.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015903805999999999 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.265468+0000", 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0033427379999997 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.277657+0000", 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012188717999999999 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.304154+0000", 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026497447 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.343603+0000", 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039449074000000001 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.348258+0000", 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046544550000000001 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.349904+0000", 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00164638 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.355566+0000", 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005662238 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.363836+0000", 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082696090000000007 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.411293+0000", 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.047457669000000001 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131095 12.7 12:eb6341ba:::benchmark_data_smithi142_261410_object12898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:53.164734+0000", 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 234.24077214799999, 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5422613759999999, 2024-08-17T17:38:47.578 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131095 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.164734+0000", 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.164734+0000", 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.164733+0000", 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.164740+0000", 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1330000000000001e-06 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.164755+0000", 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4371e-05 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.164760+0000", 2024-08-17T17:38:47.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6500000000000001e-06 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.336274+0000", 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.17151429000000001 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.357891+0000", 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021616878999999999 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.562922+0000", 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.205030988 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.579879+0000", 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016956513999999999 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.563125+0000", 2024-08-17T17:38:47.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9832461860000001 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.581812+0000", 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018687259000000001 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.610602+0000", 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028789581000000002 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.637501+0000", 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026898668000000001 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.644379+0000", 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068781620000000002 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.652159+0000", 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077807350000000004 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.654423+0000", 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022633089999999998 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.665956+0000", 2024-08-17T17:38:47.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011533121 2024-08-17T17:38:47.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.706996+0000", 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.041039900999999997 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131128 12.7 12:e2857ef5:::benchmark_data_smithi142_261410_object12931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:53.561163+0000", 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 233.844343818, 2024-08-17T17:38:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4466719049999996, 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131128 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.561163+0000", 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.561163+0000", 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.561162+0000", 2024-08-17T17:38:47.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.561166+0000", 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4129999999999999e-06 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.561177+0000", 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1257000000000001e-05 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.561184+0000", 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2879999999999998e-06 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.651848+0000", 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.090664143000000003 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.671616+0000", 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01976847 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.849629+0000", 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.178013052 2024-08-17T17:38:47.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:53.868488+0000", 2024-08-17T17:38:47.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018858460000000001 2024-08-17T17:38:47.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.795934+0000", 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9274459180000001 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.810927+0000", 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014993263999999999 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.843228+0000", 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.032300707999999997 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.938537+0000", 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.095309238000000004 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.945468+0000", 2024-08-17T17:38:47.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069311169999999997 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.952082+0000", 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066144339999999998 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.962283+0000", 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010200298 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:57.969346+0000", 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070630750000000003 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:58.007835+0000", 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038488873999999999 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131189 12.7 12:ed699f07:::benchmark_data_smithi142_261410_object12992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:54.483373+0000", 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.92213308699999, 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6878283039999999, 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131189 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.483373+0000", 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.483373+0000", 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.483372+0000", 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:47.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.483381+0000", 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2019999999999993e-06 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.483395+0000", 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4334e-05 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.483402+0000", 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4309999999999999e-06 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.621748+0000", 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13834675299999999 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.635298+0000", 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013549555 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.873106+0000", 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.237808465 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.894336+0000", 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021229380999999999 2024-08-17T17:38:47.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.011365+0000", 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1170288150000003 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.022387+0000", 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011022769999999999 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:38:47.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.044202+0000", 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021814164 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.113814+0000", 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.069612307999999998 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.119820+0000", 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060065279999999997 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.123432+0000", 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003611773 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.126796+0000", 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033634149999999998 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.132228+0000", 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054321559999999996 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:59.171202+0000", 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038973986000000002 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:47.609 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:47.610 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:38:47.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:47.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:47 smithi142 ceph-mon[106729]: pgmap v1355: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 55142/257175 objects degraded (21.441%); 33763/257175 objects misplaced (13.128%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:38:47.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:47 smithi142 ceph-mon[103614]: pgmap v1355: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 55142/257175 objects degraded (21.441%); 33763/257175 objects misplaced (13.128%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:38:47.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:47.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:47 smithi152 ceph-mon[93456]: pgmap v1355: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 55142/257175 objects degraded (21.441%); 33763/257175 objects misplaced (13.128%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:38:47.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:47.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:47.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:47.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:48.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:48.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:48.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:48.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:48.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:48.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:48.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:48.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:48.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:48.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:48.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:48.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:48.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:48 smithi152 ceph-mon[93456]: from='client.37350 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:48.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:48 smithi152 ceph-mon[93456]: pgmap v1356: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 55113/256419 objects degraded (21.493%); 33636/256419 objects misplaced (13.118%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:38:48.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:48.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:48.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:48 smithi142 ceph-mon[106729]: from='client.37350 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:48.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:48 smithi142 ceph-mon[106729]: pgmap v1356: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 55113/256419 objects degraded (21.493%); 33636/256419 objects misplaced (13.118%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:38:48.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:48 smithi142 ceph-mon[103614]: from='client.37350 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:48.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:48 smithi142 ceph-mon[103614]: pgmap v1356: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 55113/256419 objects degraded (21.493%); 33636/256419 objects misplaced (13.118%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:38:48.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:48.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:48.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:38:48.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:48.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:49.006 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:49.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:49.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:49.122 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:49.129 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:49.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:49.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:49.276 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:49.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:49.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:49.402 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:49.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:49.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:49.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:49.610 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1355,"stamp":"2024-08-17T17:38:46.298137+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5616900069,"num_objects":85725,"num_object_clones":0,"num_object_copies":257175,"num_objects_missing_on_primary":6004,"num_objects_missing":6004,"num_objects_degraded":55142,"num_objects_misplaced":33763,"num_objects_unfound":0,"num_objects_dirty":85725,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":150969,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26273,"num_bytes_recovered":1598956174,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104917,"ondisk_log_size":104917,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17106048,"kb_used_data":11662148,"kb_used_omap":27,"kb_used_meta":5443812,"kb_avail":545364864,"statfs":{"total":575970213888,"available":558453620736,"internally_reserved":0,"allocated":11942039552,"data_stored":11930273175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5574464232},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-53084160,"num_objects":-810,"num_object_clones":0,"num_object_copies":-2430,"num_objects_missing_on_primary":-61,"num_objects_missing":-61,"num_objects_degraded":-100,"num_objects_misplaced":-290,"num_objects_unfound":0,"num_objects_dirty":-810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":810,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":9633792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002821"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":580,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479975+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:38:43.479975+0000","last_peered":"2024-08-17T17:38:43.479975+0000","last_clean":"2024-08-17T17:38:43.479975+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:38:43.479975+0000","last_undegraded":"2024-08-17T17:38:43.479975+0000","last_fullsized":"2024-08-17T17:38:43.479975+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":77,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442425+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:38:22.442425+0000","last_peered":"2024-08-17T17:38:22.442425+0000","last_clean":"2024-08-17T17:38:22.442425+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:38:22.442425+0000","last_undegraded":"2024-08-17T17:38:22.442425+0000","last_fullsized":"2024-08-17T17:38:22.442425+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":272,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489541+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:38:34.489541+0000","last_peered":"2024-08-17T17:38:34.489541+0000","last_clean":"2024-08-17T17:38:34.489541+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:38:34.489541+0000","last_undegraded":"2024-08-17T17:38:34.489541+0000","last_fullsized":"2024-08-17T17:38:34.489541+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":278,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.480019+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:38:43.480019+0000","last_peered":"2024-08-17T17:38:43.480019+0000","last_clean":"2024-08-17T17:38:43.480019+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:38:43.480019+0000","last_undegraded":"2024-08-17T17:38:43.480019+0000","last_fullsized":"2024-08-17T17:38:43.480019+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":455,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482354+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:38:43.482354+0000","last_peered":"2024-08-17T17:38:43.482354+0000","last_clean":"2024-08-17T17:38:43.482354+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:38:43.482354+0000","last_undegraded":"2024-08-17T17:38:43.482354+0000","last_fullsized":"2024-08-17T17:38:43.482354+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":220,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442091+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:38:22.442091+0000","last_peered":"2024-08-17T17:38:22.442091+0000","last_clean":"2024-08-17T17:38:22.442091+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:38:22.442091+0000","last_undegraded":"2024-08-17T17:38:22.442091+0000","last_fullsized":"2024-08-17T17:38:22.442091+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":548,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482225+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:38:43.482225+0000","last_peered":"2024-08-17T17:38:43.482225+0000","last_clean":"2024-08-17T17:38:43.482225+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:38:43.482225+0000","last_undegraded":"2024-08-17T17:38:43.482225+0000","last_fullsized":"2024-08-17T17:38:43.482225+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":388,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.499550+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:38:22.499550+0000","last_peered":"2024-08-17T17:38:22.499550+0000","last_clean":"2024-08-17T17:38:22.499550+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:38:22.499550+0000","last_undegraded":"2024-08-17T17:38:22.499550+0000","last_fullsized":"2024-08-17T17:38:22.499550+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"368'9409","reported_seq":12145,"reported_epoch":368,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:38:42.533362+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:38:42.533362+0000","last_peered":"2024-08-17T17:38:42.533362+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:38:42.533362+0000","last_undegraded":"2024-08-17T17:38:42.533362+0000","last_fullsized":"2024-08-17T17:38:42.533362+0000","mapping_epoch":337,"log_start":"324'2563","ondisk_log_start":"324'2563","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347013120,"num_objects":5295,"num_object_clones":0,"num_object_copies":15885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5295,"num_objects_unfound":0,"num_objects_dirty":5295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9409,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1328,"num_bytes_recovered":87031808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5295}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":73,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.483193+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:38:22.483193+0000","last_peered":"2024-08-17T17:38:22.483193+0000","last_clean":"2024-08-17T17:38:22.483193+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:38:22.483193+0000","last_undegraded":"2024-08-17T17:38:22.483193+0000","last_fullsized":"2024-08-17T17:38:22.483193+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489116+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:38:34.489116+0000","last_peered":"2024-08-17T17:38:34.489116+0000","last_clean":"2024-08-17T17:38:34.489116+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:38:34.489116+0000","last_undegraded":"2024-08-17T17:38:34.489116+0000","last_fullsized":"2024-08-17T17:38:34.489116+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"368'9596","reported_seq":9662,"reported_epoch":368,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:41.645207+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:38:41.645207+0000","last_peered":"2024-08-17T17:38:41.645207+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:38:41.645207+0000","last_undegraded":"2024-08-17T17:38:41.645207+0000","last_fullsized":"2024-08-17T17:38:41.645207+0000","mapping_epoch":337,"log_start":"325'2982","ondisk_log_start":"325'2982","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359006208,"num_objects":5478,"num_object_clones":0,"num_object_copies":16434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5478,"num_objects_unfound":0,"num_objects_dirty":5478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9596,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5478}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442841+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:38:22.442841+0000","last_peered":"2024-08-17T17:38:22.442841+0000","last_clean":"2024-08-17T17:38:22.442841+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:38:22.442841+0000","last_undegraded":"2024-08-17T17:38:22.442841+0000","last_fullsized":"2024-08-17T17:38:22.442841+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":339,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442046+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:38:22.442046+0000","last_peered":"2024-08-17T17:38:22.442046+0000","last_clean":"2024-08-17T17:38:22.442046+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:38:22.442046+0000","last_undegraded":"2024-08-17T17:38:22.442046+0000","last_fullsized":"2024-08-17T17:38:22.442046+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"368'9336","reported_seq":9405,"reported_epoch":368,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:42.484594+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:38:42.484594+0000","last_peered":"2024-08-17T17:38:42.484594+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:38:42.484594+0000","last_undegraded":"2024-08-17T17:38:42.484594+0000","last_fullsized":"2024-08-17T17:38:42.484594+0000","mapping_epoch":337,"log_start":"324'2453","ondisk_log_start":"324'2453","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353763328,"num_objects":5398,"num_object_clones":0,"num_object_copies":16194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5398,"num_objects_unfound":0,"num_objects_dirty":5398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9336,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5398}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":72,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489498+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:38:34.489498+0000","last_peered":"2024-08-17T17:38:34.489498+0000","last_clean":"2024-08-17T17:38:34.489498+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:38:34.489498+0000","last_undegraded":"2024-08-17T17:38:34.489498+0000","last_fullsized":"2024-08-17T17:38:34.489498+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":349,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.499567+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:38:22.499567+0000","last_peered":"2024-08-17T17:38:22.499567+0000","last_clean":"2024-08-17T17:38:22.499567+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:38:22.499567+0000","last_undegraded":"2024-08-17T17:38:22.499567+0000","last_fullsized":"2024-08-17T17:38:22.499567+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"369'9523","reported_seq":9640,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:44.684781+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:38:44.684781+0000","last_peered":"2024-08-17T17:38:44.684781+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:38:44.684781+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2513","ondisk_log_start":"324'2513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356188160,"num_objects":5435,"num_object_clones":0,"num_object_copies":16305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9523,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1334},{"shards":"0,3,7","objects":5435}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":492,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482062+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:38:43.482062+0000","last_peered":"2024-08-17T17:38:43.482062+0000","last_clean":"2024-08-17T17:38:43.482062+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:38:43.482062+0000","last_undegraded":"2024-08-17T17:38:43.482062+0000","last_fullsized":"2024-08-17T17:38:43.482062+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":258,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479330+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:38:43.479330+0000","last_peered":"2024-08-17T17:38:43.479330+0000","last_clean":"2024-08-17T17:38:43.479330+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:38:43.479330+0000","last_undegraded":"2024-08-17T17:38:43.479330+0000","last_fullsized":"2024-08-17T17:38:43.479330+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"368'9291","reported_seq":9348,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:41.819963+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:38:41.819963+0000","last_peered":"2024-08-17T17:38:41.819963+0000","last_clean":"2024-08-17T17:38:41.819963+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:38:41.819963+0000","last_undegraded":"2024-08-17T17:38:41.819963+0000","last_fullsized":"2024-08-17T17:38:41.819963+0000","mapping_epoch":319,"log_start":"325'2674","ondisk_log_start":"325'2674","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347537408,"num_objects":5303,"num_object_clones":0,"num_object_copies":15909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9291,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":581,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.480375+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:38:43.480375+0000","last_peered":"2024-08-17T17:38:43.480375+0000","last_clean":"2024-08-17T17:38:43.480375+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:38:43.480375+0000","last_undegraded":"2024-08-17T17:38:43.480375+0000","last_fullsized":"2024-08-17T17:38:43.480375+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":322,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442649+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:38:22.442649+0000","last_peered":"2024-08-17T17:38:22.442649+0000","last_clean":"2024-08-17T17:38:22.442649+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:38:22.442649+0000","last_undegraded":"2024-08-17T17:38:22.442649+0000","last_fullsized":"2024-08-17T17:38:22.442649+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"368'9434","reported_seq":9501,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:40.684491+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:38:40.684491+0000","last_peered":"2024-08-17T17:38:40.684491+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:38:40.684491+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'2835","ondisk_log_start":"325'2835","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360316928,"num_objects":5498,"num_object_clones":0,"num_object_copies":16494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9434,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5498},{"shards":"1,6","objects":-1868}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":323,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.499682+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:38:22.499682+0000","last_peered":"2024-08-17T17:38:22.499682+0000","last_clean":"2024-08-17T17:38:22.499682+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:38:22.499682+0000","last_undegraded":"2024-08-17T17:38:22.499682+0000","last_fullsized":"2024-08-17T17:38:22.499682+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":349,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442984+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:38:22.442984+0000","last_peered":"2024-08-17T17:38:22.442984+0000","last_clean":"2024-08-17T17:38:22.442984+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:38:22.442984+0000","last_undegraded":"2024-08-17T17:38:22.442984+0000","last_fullsized":"2024-08-17T17:38:22.442984+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"369'9582","reported_seq":9700,"reported_epoch":369,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:43.497668+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:38:43.497668+0000","last_peered":"2024-08-17T17:38:43.497668+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:38:43.497668+0000","last_undegraded":"2024-08-17T17:38:43.497668+0000","last_fullsized":"2024-08-17T17:38:43.497668+0000","mapping_epoch":337,"log_start":"326'3961","ondisk_log_start":"326'3961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5621,"log_dups_size":0,"ondisk_log_size":5621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355729408,"num_objects":5428,"num_object_clones":0,"num_object_copies":16284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5428,"num_objects_unfound":0,"num_objects_dirty":5428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9582,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5428}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":360,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442168+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:38:22.442168+0000","last_peered":"2024-08-17T17:38:22.442168+0000","last_clean":"2024-08-17T17:38:22.442168+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:38:22.442168+0000","last_undegraded":"2024-08-17T17:38:22.442168+0000","last_fullsized":"2024-08-17T17:38:22.442168+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":322,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442969+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:38:22.442969+0000","last_peered":"2024-08-17T17:38:22.442969+0000","last_clean":"2024-08-17T17:38:22.442969+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:38:22.442969+0000","last_undegraded":"2024-08-17T17:38:22.442969+0000","last_fullsized":"2024-08-17T17:38:22.442969+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"368'9394","reported_seq":12153,"reported_epoch":368,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:42.444748+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:38:42.444748+0000","last_peered":"2024-08-17T17:38:42.444748+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:38:42.444748+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:38:42.444748+0000","mapping_epoch":337,"log_start":"324'2541","ondisk_log_start":"324'2541","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6853,"log_dups_size":0,"ondisk_log_size":6853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347996160,"num_objects":5310,"num_object_clones":0,"num_object_copies":15930,"num_objects_missing_on_primary":6004,"num_objects_missing":6004,"num_objects_degraded":5310,"num_objects_misplaced":6004,"num_objects_unfound":0,"num_objects_dirty":5310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9394,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1337,"num_bytes_recovered":87556096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":11},{"shards":"1,2,7","objects":5298},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":551,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479789+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:38:43.479789+0000","last_peered":"2024-08-17T17:38:43.479789+0000","last_clean":"2024-08-17T17:38:43.479789+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:38:43.479789+0000","last_undegraded":"2024-08-17T17:38:43.479789+0000","last_fullsized":"2024-08-17T17:38:43.479789+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":488,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482267+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:38:43.482267+0000","last_peered":"2024-08-17T17:38:43.482267+0000","last_clean":"2024-08-17T17:38:43.482267+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:38:43.482267+0000","last_undegraded":"2024-08-17T17:38:43.482267+0000","last_fullsized":"2024-08-17T17:38:43.482267+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"369'9486","reported_seq":12169,"reported_epoch":369,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:43.604337+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:38:43.604337+0000","last_peered":"2024-08-17T17:38:43.604337+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:38:43.604337+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:38:43.604337+0000","mapping_epoch":337,"log_start":"326'3844","ondisk_log_start":"326'3844","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354484248,"num_objects":5410,"num_object_clones":0,"num_object_copies":16230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5410,"num_objects_misplaced":6160,"num_objects_unfound":0,"num_objects_dirty":5410,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9486,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1288,"num_bytes_recovered":84344832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5410}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":681,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:43.477983+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:38:43.477983+0000","last_peered":"2024-08-17T17:38:43.477983+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:38:43.477983+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":298,"reported_epoch":368,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:38:34.489408+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:38:34.489408+0000","last_peered":"2024-08-17T17:38:34.489408+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:38:34.489408+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:38:34.489408+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":530,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479942+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:38:43.479942+0000","last_peered":"2024-08-17T17:38:43.479942+0000","last_clean":"2024-08-17T17:38:43.479942+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:38:43.479942+0000","last_undegraded":"2024-08-17T17:38:43.479942+0000","last_fullsized":"2024-08-17T17:38:43.479942+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":332,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.500365+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:38:22.500365+0000","last_peered":"2024-08-17T17:38:22.500365+0000","last_clean":"2024-08-17T17:38:22.500365+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:38:22.500365+0000","last_undegraded":"2024-08-17T17:38:22.500365+0000","last_fullsized":"2024-08-17T17:38:22.500365+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"369'9408","reported_seq":9528,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:44.411839+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:38:44.411839+0000","last_peered":"2024-08-17T17:38:44.411839+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:38:44.411839+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2421","ondisk_log_start":"324'2421","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349700096,"num_objects":5336,"num_object_clones":0,"num_object_copies":16008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9408,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1311},{"shards":"0,4,7","objects":5336}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":346,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.466889+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:38:43.466889+0000","last_peered":"2024-08-17T17:38:43.466889+0000","last_clean":"2024-08-17T17:38:43.466889+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:38:43.466889+0000","last_undegraded":"2024-08-17T17:38:43.466889+0000","last_fullsized":"2024-08-17T17:38:43.466889+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":304,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.441921+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:38:22.441921+0000","last_peered":"2024-08-17T17:38:22.441921+0000","last_clean":"2024-08-17T17:38:22.441921+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:38:22.441921+0000","last_undegraded":"2024-08-17T17:38:22.441921+0000","last_fullsized":"2024-08-17T17:38:22.441921+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":272,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479379+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:38:43.479379+0000","last_peered":"2024-08-17T17:38:43.479379+0000","last_clean":"2024-08-17T17:38:43.479379+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:38:43.479379+0000","last_undegraded":"2024-08-17T17:38:43.479379+0000","last_fullsized":"2024-08-17T17:38:43.479379+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"368'9397","reported_seq":9464,"reported_epoch":368,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:41.586214+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:38:41.586214+0000","last_peered":"2024-08-17T17:38:41.586214+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:38:41.586214+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'2817","ondisk_log_start":"325'2817","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353828864,"num_objects":5399,"num_object_clones":0,"num_object_copies":16197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9397,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5399},{"shards":"5,6","objects":-1897}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":438,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.480576+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:38:43.480576+0000","last_peered":"2024-08-17T17:38:43.480576+0000","last_clean":"2024-08-17T17:38:43.480576+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:38:43.480576+0000","last_undegraded":"2024-08-17T17:38:43.480576+0000","last_fullsized":"2024-08-17T17:38:43.480576+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":104,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.499436+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:38:22.499436+0000","last_peered":"2024-08-17T17:38:22.499436+0000","last_clean":"2024-08-17T17:38:22.499436+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:38:22.499436+0000","last_undegraded":"2024-08-17T17:38:22.499436+0000","last_fullsized":"2024-08-17T17:38:22.499436+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":623,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.468735+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:38:43.468735+0000","last_peered":"2024-08-17T17:38:43.468735+0000","last_clean":"2024-08-17T17:38:43.468735+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:38:43.468735+0000","last_undegraded":"2024-08-17T17:38:43.468735+0000","last_fullsized":"2024-08-17T17:38:43.468735+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"369'9323","reported_seq":10167,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:44.661948+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:38:44.661948+0000","last_peered":"2024-08-17T17:38:44.661948+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:38:44.661948+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2316","ondisk_log_start":"324'2316","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348848128,"num_objects":5323,"num_object_clones":0,"num_object_copies":15969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9323,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5323},{"shards":"0,5","objects":-1331}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":414,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467716+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:38:43.467716+0000","last_peered":"2024-08-17T17:38:43.467716+0000","last_clean":"2024-08-17T17:38:43.467716+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:38:43.467716+0000","last_undegraded":"2024-08-17T17:38:43.467716+0000","last_fullsized":"2024-08-17T17:38:43.467716+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":343,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442604+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:38:22.442604+0000","last_peered":"2024-08-17T17:38:22.442604+0000","last_clean":"2024-08-17T17:38:22.442604+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:38:22.442604+0000","last_undegraded":"2024-08-17T17:38:22.442604+0000","last_fullsized":"2024-08-17T17:38:22.442604+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":209,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479875+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:38:43.479875+0000","last_peered":"2024-08-17T17:38:43.479875+0000","last_clean":"2024-08-17T17:38:43.479875+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:38:43.479875+0000","last_undegraded":"2024-08-17T17:38:43.479875+0000","last_fullsized":"2024-08-17T17:38:43.479875+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"369'9494","reported_seq":24457,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:45.436144+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:38:45.436144+0000","last_peered":"2024-08-17T17:38:45.436144+0000","last_clean":"2024-08-17T17:38:45.436144+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:38:45.436144+0000","last_undegraded":"2024-08-17T17:38:45.436144+0000","last_fullsized":"2024-08-17T17:38:45.436144+0000","mapping_epoch":355,"log_start":"325'3230","ondisk_log_start":"325'3230","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350093312,"num_objects":5342,"num_object_clones":0,"num_object_copies":16026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9494,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":283,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489415+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:38:34.489415+0000","last_peered":"2024-08-17T17:38:34.489415+0000","last_clean":"2024-08-17T17:38:34.489415+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:38:34.489415+0000","last_undegraded":"2024-08-17T17:38:34.489415+0000","last_fullsized":"2024-08-17T17:38:34.489415+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479789+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:38:43.479789+0000","last_peered":"2024-08-17T17:38:43.479789+0000","last_clean":"2024-08-17T17:38:43.479789+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:38:43.479789+0000","last_undegraded":"2024-08-17T17:38:43.479789+0000","last_fullsized":"2024-08-17T17:38:43.479789+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":573,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467821+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:38:43.467821+0000","last_peered":"2024-08-17T17:38:43.467821+0000","last_clean":"2024-08-17T17:38:43.467821+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:38:43.467821+0000","last_undegraded":"2024-08-17T17:38:43.467821+0000","last_fullsized":"2024-08-17T17:38:43.467821+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"369'9200","reported_seq":23731,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:44.432930+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:38:44.432930+0000","last_peered":"2024-08-17T17:38:44.432930+0000","last_clean":"2024-08-17T17:38:44.432930+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:38:44.432930+0000","last_undegraded":"2024-08-17T17:38:44.432930+0000","last_fullsized":"2024-08-17T17:38:44.432930+0000","mapping_epoch":345,"log_start":"325'2872","ondisk_log_start":"325'2872","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341049344,"num_objects":5204,"num_object_clones":0,"num_object_copies":15612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9200,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":279,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489816+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:38:34.489816+0000","last_peered":"2024-08-17T17:38:34.489816+0000","last_clean":"2024-08-17T17:38:34.489816+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:38:34.489816+0000","last_undegraded":"2024-08-17T17:38:34.489816+0000","last_fullsized":"2024-08-17T17:38:34.489816+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":329,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.475630+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:38:43.475630+0000","last_peered":"2024-08-17T17:38:43.475630+0000","last_clean":"2024-08-17T17:38:43.475630+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:38:43.475630+0000","last_undegraded":"2024-08-17T17:38:43.475630+0000","last_fullsized":"2024-08-17T17:38:43.475630+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":421,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489297+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:38:34.489297+0000","last_peered":"2024-08-17T17:38:34.489297+0000","last_clean":"2024-08-17T17:38:34.489297+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:38:34.489297+0000","last_undegraded":"2024-08-17T17:38:34.489297+0000","last_fullsized":"2024-08-17T17:38:34.489297+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"369'9467","reported_seq":9606,"reported_epoch":369,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:44.869540+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:38:44.869540+0000","last_peered":"2024-08-17T17:38:44.869540+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:38:44.869540+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'3187","ondisk_log_start":"325'3187","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6280,"log_dups_size":0,"ondisk_log_size":6280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348454912,"num_objects":5317,"num_object_clones":0,"num_object_copies":15951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9467,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5317},{"shards":"0,3","objects":-1931}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":604,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482308+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:38:43.482308+0000","last_peered":"2024-08-17T17:38:43.482308+0000","last_clean":"2024-08-17T17:38:43.482308+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:38:43.482308+0000","last_undegraded":"2024-08-17T17:38:43.482308+0000","last_fullsized":"2024-08-17T17:38:43.482308+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":572,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.468847+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:38:43.468847+0000","last_peered":"2024-08-17T17:38:43.468847+0000","last_clean":"2024-08-17T17:38:43.468847+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:38:43.468847+0000","last_undegraded":"2024-08-17T17:38:43.468847+0000","last_fullsized":"2024-08-17T17:38:43.468847+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":590,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479559+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:38:43.479559+0000","last_peered":"2024-08-17T17:38:43.479559+0000","last_clean":"2024-08-17T17:38:43.479559+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:38:43.479559+0000","last_undegraded":"2024-08-17T17:38:43.479559+0000","last_fullsized":"2024-08-17T17:38:43.479559+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"368'9351","reported_seq":24016,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:41.534361+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:38:41.534361+0000","last_peered":"2024-08-17T17:38:41.534361+0000","last_clean":"2024-08-17T17:38:41.534361+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:38:41.534361+0000","last_undegraded":"2024-08-17T17:38:41.534361+0000","last_fullsized":"2024-08-17T17:38:41.534361+0000","mapping_epoch":347,"log_start":"325'2772","ondisk_log_start":"325'2772","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342294528,"num_objects":5223,"num_object_clones":0,"num_object_copies":15669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9351,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":262,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.478383+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:38:43.478383+0000","last_peered":"2024-08-17T17:38:43.478383+0000","last_clean":"2024-08-17T17:38:43.478383+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:38:43.478383+0000","last_undegraded":"2024-08-17T17:38:43.478383+0000","last_fullsized":"2024-08-17T17:38:43.478383+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":372,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.500152+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:38:22.500152+0000","last_peered":"2024-08-17T17:38:22.500152+0000","last_clean":"2024-08-17T17:38:22.500152+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:38:22.500152+0000","last_undegraded":"2024-08-17T17:38:22.500152+0000","last_fullsized":"2024-08-17T17:38:22.500152+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":322,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442113+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:38:22.442113+0000","last_peered":"2024-08-17T17:38:22.442113+0000","last_clean":"2024-08-17T17:38:22.442113+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:38:22.442113+0000","last_undegraded":"2024-08-17T17:38:22.442113+0000","last_fullsized":"2024-08-17T17:38:22.442113+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":322,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442536+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:38:22.442536+0000","last_peered":"2024-08-17T17:38:22.442536+0000","last_clean":"2024-08-17T17:38:22.442536+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:38:22.442536+0000","last_undegraded":"2024-08-17T17:38:22.442536+0000","last_fullsized":"2024-08-17T17:38:22.442536+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":365,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442808+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:38:22.442808+0000","last_peered":"2024-08-17T17:38:22.442808+0000","last_clean":"2024-08-17T17:38:22.442808+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:38:22.442808+0000","last_undegraded":"2024-08-17T17:38:22.442808+0000","last_fullsized":"2024-08-17T17:38:22.442808+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":579,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467708+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:38:43.467708+0000","last_peered":"2024-08-17T17:38:43.467708+0000","last_clean":"2024-08-17T17:38:43.467708+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:38:43.467708+0000","last_undegraded":"2024-08-17T17:38:43.467708+0000","last_fullsized":"2024-08-17T17:38:43.467708+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":119,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482731+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:38:43.482731+0000","last_peered":"2024-08-17T17:38:43.482731+0000","last_clean":"2024-08-17T17:38:43.482731+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:38:43.482731+0000","last_undegraded":"2024-08-17T17:38:43.482731+0000","last_fullsized":"2024-08-17T17:38:43.482731+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":273,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.489651+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:38:34.489651+0000","last_peered":"2024-08-17T17:38:34.489651+0000","last_clean":"2024-08-17T17:38:34.489651+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:38:34.489651+0000","last_undegraded":"2024-08-17T17:38:34.489651+0000","last_fullsized":"2024-08-17T17:38:34.489651+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":537,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479527+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:38:43.479527+0000","last_peered":"2024-08-17T17:38:43.479527+0000","last_clean":"2024-08-17T17:38:43.479527+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:38:43.479527+0000","last_undegraded":"2024-08-17T17:38:43.479527+0000","last_fullsized":"2024-08-17T17:38:43.479527+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":417,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-08-17T17:38:34.491652+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:38:34.491652+0000","last_peered":"2024-08-17T17:38:34.491652+0000","last_clean":"2024-08-17T17:38:34.491652+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:38:34.491652+0000","last_undegraded":"2024-08-17T17:38:34.491652+0000","last_fullsized":"2024-08-17T17:38:34.491652+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":531,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479450+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:38:43.479450+0000","last_peered":"2024-08-17T17:38:43.479450+0000","last_clean":"2024-08-17T17:38:43.479450+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:38:43.479450+0000","last_undegraded":"2024-08-17T17:38:43.479450+0000","last_fullsized":"2024-08-17T17:38:43.479450+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":339,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.500088+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:38:22.500088+0000","last_peered":"2024-08-17T17:38:22.500088+0000","last_clean":"2024-08-17T17:38:22.500088+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:38:22.500088+0000","last_undegraded":"2024-08-17T17:38:22.500088+0000","last_fullsized":"2024-08-17T17:38:22.500088+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":562,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.479706+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:38:43.479706+0000","last_peered":"2024-08-17T17:38:43.479706+0000","last_clean":"2024-08-17T17:38:43.479706+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:38:43.479706+0000","last_undegraded":"2024-08-17T17:38:43.479706+0000","last_fullsized":"2024-08-17T17:38:43.479706+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":278,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482306+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:38:43.482306+0000","last_peered":"2024-08-17T17:38:43.482306+0000","last_clean":"2024-08-17T17:38:43.482306+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:38:43.482306+0000","last_undegraded":"2024-08-17T17:38:43.482306+0000","last_fullsized":"2024-08-17T17:38:43.482306+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":570,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467502+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:38:43.467502+0000","last_peered":"2024-08-17T17:38:43.467502+0000","last_clean":"2024-08-17T17:38:43.467502+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:38:43.467502+0000","last_undegraded":"2024-08-17T17:38:43.467502+0000","last_fullsized":"2024-08-17T17:38:43.467502+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":530,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482582+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:38:43.482582+0000","last_peered":"2024-08-17T17:38:43.482582+0000","last_clean":"2024-08-17T17:38:43.482582+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:38:43.482582+0000","last_undegraded":"2024-08-17T17:38:43.482582+0000","last_fullsized":"2024-08-17T17:38:43.482582+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":351,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442328+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:38:22.442328+0000","last_peered":"2024-08-17T17:38:22.442328+0000","last_clean":"2024-08-17T17:38:22.442328+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:38:22.442328+0000","last_undegraded":"2024-08-17T17:38:22.442328+0000","last_fullsized":"2024-08-17T17:38:22.442328+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":219,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.480060+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:38:43.480060+0000","last_peered":"2024-08-17T17:38:43.480060+0000","last_clean":"2024-08-17T17:38:43.480060+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:38:43.480060+0000","last_undegraded":"2024-08-17T17:38:43.480060+0000","last_fullsized":"2024-08-17T17:38:43.480060+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":275,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.476293+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:38:43.476293+0000","last_peered":"2024-08-17T17:38:43.476293+0000","last_clean":"2024-08-17T17:38:43.476293+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:38:43.476293+0000","last_undegraded":"2024-08-17T17:38:43.476293+0000","last_fullsized":"2024-08-17T17:38:43.476293+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":359,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442508+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:38:22.442508+0000","last_peered":"2024-08-17T17:38:22.442508+0000","last_clean":"2024-08-17T17:38:22.442508+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:38:22.442508+0000","last_undegraded":"2024-08-17T17:38:22.442508+0000","last_fullsized":"2024-08-17T17:38:22.442508+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":222,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442725+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:38:22.442725+0000","last_peered":"2024-08-17T17:38:22.442725+0000","last_clean":"2024-08-17T17:38:22.442725+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:38:22.442725+0000","last_undegraded":"2024-08-17T17:38:22.442725+0000","last_fullsized":"2024-08-17T17:38:22.442725+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":359,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442244+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:38:22.442244+0000","last_peered":"2024-08-17T17:38:22.442244+0000","last_clean":"2024-08-17T17:38:22.442244+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:38:22.442244+0000","last_undegraded":"2024-08-17T17:38:22.442244+0000","last_fullsized":"2024-08-17T17:38:22.442244+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":222,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.442866+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:38:22.442866+0000","last_peered":"2024-08-17T17:38:22.442866+0000","last_clean":"2024-08-17T17:38:22.442866+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:38:22.442866+0000","last_undegraded":"2024-08-17T17:38:22.442866+0000","last_fullsized":"2024-08-17T17:38:22.442866+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":582,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.480643+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:38:43.480643+0000","last_peered":"2024-08-17T17:38:43.480643+0000","last_clean":"2024-08-17T17:38:43.480643+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:38:43.480643+0000","last_undegraded":"2024-08-17T17:38:43.480643+0000","last_fullsized":"2024-08-17T17:38:43.480643+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":556,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482398+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:38:43.482398+0000","last_peered":"2024-08-17T17:38:43.482398+0000","last_clean":"2024-08-17T17:38:43.482398+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:38:43.482398+0000","last_undegraded":"2024-08-17T17:38:43.482398+0000","last_fullsized":"2024-08-17T17:38:43.482398+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":322,"reported_epoch":367,"state":"active+clean","last_fresh":"2024-08-17T17:38:22.441964+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:38:22.441964+0000","last_peered":"2024-08-17T17:38:22.441964+0000","last_clean":"2024-08-17T17:38:22.441964+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:38:22.441964+0000","last_undegraded":"2024-08-17T17:38:22.441964+0000","last_fullsized":"2024-08-17T17:38:22.441964+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":277,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.475902+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:38:43.475902+0000","last_peered":"2024-08-17T17:38:43.475902+0000","last_clean":"2024-08-17T17:38:43.475902+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:38:43.475902+0000","last_undegraded":"2024-08-17T17:38:43.475902+0000","last_fullsized":"2024-08-17T17:38:43.475902+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5616304152,"num_objects":85699,"num_object_clones":0,"num_object_copies":257097,"num_objects_missing_on_primary":6004,"num_objects_missing":6004,"num_objects_degraded":55139,"num_objects_misplaced":33763,"num_objects_unfound":0,"num_objects_dirty":85699,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150691,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26232,"num_bytes_recovered":1596588032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18147127296,"data_stored":18147115080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104710,"ondisk_log_size":104710,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170068,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463437,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3297560,"kb_used_data":2399696,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":90447592,"statfs":{"total":95995035648,"available":92618334208,"internally_reserved":0,"allocated":2457288704,"data_stored":2455320641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492112,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1094092,"kb_used_data":690436,"kb_used_omap":7,"kb_used_meta":403640,"kb_avail":92651060,"statfs":{"total":95995035648,"available":94874685440,"internally_reserved":0,"allocated":707006464,"data_stored":705052908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":413327693},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986703,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2551500,"kb_used_data":1366340,"kb_used_omap":1,"kb_used_meta":1185150,"kb_avail":91193652,"statfs":{"total":95995035648,"available":93382299648,"internally_reserved":0,"allocated":1399132160,"data_stored":1397175037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213593944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225531,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3650128,"kb_used_data":2734404,"kb_used_omap":4,"kb_used_meta":915707,"kb_avail":90095024,"statfs":{"total":95995035648,"available":92257304576,"internally_reserved":0,"allocated":2800029696,"data_stored":2798076438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":937684416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075856,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533749,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3450900,"kb_used_data":2404744,"kb_used_omap":5,"kb_used_meta":1046138,"kb_avail":90294252,"statfs":{"total":95995035648,"available":92461314048,"internally_reserved":0,"allocated":2462457856,"data_stored":2460499276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1071245552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089726,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3061868,"kb_used_data":2066528,"kb_used_omap":5,"kb_used_meta":995322,"kb_avail":90683284,"statfs":{"total":95995035648,"available":92859682816,"internally_reserved":0,"allocated":2116124672,"data_stored":2114148875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1019209787},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2109014016,"data_stored":2109014016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2456424448,"data_stored":2456420376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3593605120,"data_stored":3593601048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2794061824,"data_stored":2794061824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1393557504,"data_stored":1393557504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":702021632,"data_stored":702021632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2451898368,"data_stored":2451898368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2646544384,"data_stored":2646540312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:49.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:38:49.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:49.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:49.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:49.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:49.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:49.927 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:50.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:50.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:50.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:50.154 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:50.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:50.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:50.290 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:38:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:38:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:50.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:50.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:50.402 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:38:50.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:50.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:50.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/291327326' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:50.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:50.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:50.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:50.764 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:50.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:50.784 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/291327326' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:50.784 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:50.784 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/291327326' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:50.784 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:50.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:151443 12.f 12:fc99f405:::benchmark_data_smithi142_261410_object33246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e370)", 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:50.855189+0000", 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.013771887, 2024-08-17T17:38:50.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013797307, 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 151443 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855189+0000", 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855189+0000", 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855188+0000", 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855195+0000", 2024-08-17T17:38:50.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3599999999999998e-06 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855207+0000", 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1620000000000001e-05 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855212+0000", 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1950000000000002e-06 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:38:50.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855254+0000", 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2070000000000002e-05 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855376+0000", 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122132 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.855484+0000", 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108096 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.858514+0000", 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030295119999999998 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:50.859142+0000", 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062871100000000001 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:38:50.875 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 1 2024-08-17T17:38:50.876 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:38:50.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:50.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:50.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:51.025 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:51.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:51.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:51.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:51.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:51.274 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:51.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:51.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:51.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:51.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:51.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[103614]: pgmap v1357: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 55073/255924 objects degraded (21.519%); 33563/255924 objects misplaced (13.114%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:38:51.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[103614]: osdmap e370: 8 total, 8 up, 6 in 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 55073/255924 objects degraded (21.519%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[106729]: pgmap v1357: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 55073/255924 objects degraded (21.519%); 33563/255924 objects misplaced (13.114%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[106729]: osdmap e370: 8 total, 8 up, 6 in 2024-08-17T17:38:51.495 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 55073/255924 objects degraded (21.519%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:51.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:51.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.546 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:51.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:51.569 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:51 smithi152 ceph-mon[93456]: pgmap v1357: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 55073/255924 objects degraded (21.519%); 33563/255924 objects misplaced (13.114%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:38:51.569 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:51 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:51.569 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:51 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:51.569 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:51 smithi152 ceph-mon[93456]: osdmap e370: 8 total, 8 up, 6 in 2024-08-17T17:38:51.569 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 55073/255924 objects degraded (21.519%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:51.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:38:51.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:51.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:51.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:51.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:51.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:51.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:51.909 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:51.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:52.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:52.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:52.094 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:38:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:38:51] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:38:52.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:52.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:52.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:52.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:52.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:52.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:52.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:52.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:52.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:52.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:52.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:52.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:52.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:52.796 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:52.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:52.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:52.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:52.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:53.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:53.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:53.172 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:53.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:53.274 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:53.296 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:53.398 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:53.420 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.522 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:53.545 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:53.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:53 smithi142 ceph-mon[106729]: pgmap v1359: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 55073/255138 objects degraded (21.586%); 33520/255138 objects misplaced (13.138%); 898 KiB/s, 14 objects/s recovering 2024-08-17T17:38:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:53 smithi142 ceph-mon[103614]: pgmap v1359: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 55073/255138 objects degraded (21.586%); 33520/255138 objects misplaced (13.138%); 898 KiB/s, 14 objects/s recovering 2024-08-17T17:38:53.647 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.670 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:53.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:53.737 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:53 smithi152 ceph-mon[93456]: pgmap v1359: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 55073/255138 objects degraded (21.586%); 33520/255138 objects misplaced (13.138%); 898 KiB/s, 14 objects/s recovering 2024-08-17T17:38:53.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:53.793 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:53.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:38:53.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:53.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:53.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:53.959 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:38:53.959 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:38:53.959 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:38:53.960 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:38:53.960 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:38:54.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:54.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:54.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:54.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:54.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:54.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:54.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:54.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:54.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:54.568 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:54.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:54.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:54.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:54.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:54.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:54.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:38:54.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:54.884 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:38:54.884 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 54997/254598 objects degraded (21.602%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2015,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":370,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":84866,"data_bytes":5560604645,"bytes_used":17419145216,"bytes_avail":558551068672,"bytes_total":575970213888,"degraded_objects":54997,"degraded_total":254598,"degraded_ratio":0.21601505117872097,"misplaced_objects":33341,"misplaced_total":254598,"misplaced_ratio":0.13095546705001609,"recovering_objects_per_sec":21,"recovering_bytes_per_sec":1388993,"recovering_keys_per_sec":0,"num_objects_recovered":212,"num_bytes_recovered":13893632,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":98},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:38:54.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:54.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:55.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:55.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:55.122 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:55.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:55.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:55.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:55.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:55.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:55.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:55.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:55.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:55 smithi152 ceph-mon[93456]: pgmap v1360: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 54997/254598 objects degraded (21.602%); 33341/254598 objects misplaced (13.096%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:55.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3007194643' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:55.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:55.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:55.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:55.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[106729]: pgmap v1360: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 54997/254598 objects degraded (21.602%); 33341/254598 objects misplaced (13.096%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3007194643' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[103614]: pgmap v1360: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 54997/254598 objects degraded (21.602%); 33341/254598 objects misplaced (13.096%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3007194643' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:55.667 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:55.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:55.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:55.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:55.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:55.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:56.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:56.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:56.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:56.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:56.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:56.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:56.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:56.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:56.493 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:38:56.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:56.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:56.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54997/254598 objects degraded (21.602%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:56.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:56.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:56.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54997/254598 objects degraded (21.602%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:56.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54997/254598 objects degraded (21.602%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:56.751 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:56.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:56.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:56.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:56.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:56.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:57.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:57.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:57.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:57.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:57.217 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:57.217 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:38:57.265 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:57.266 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:57.267 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:38:57.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:57.312 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:57.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:57.437 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:57.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:57.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.563 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:57.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:57.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:57.688 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:57.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:57.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: pgmap v1361: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54997/253971 objects degraded (21.655%); 33341/253971 objects misplaced (13.128%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/789309650' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: pgmap v1361: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54997/253971 objects degraded (21.655%); 33341/253971 objects misplaced (13.128%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/789309650' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: pgmap v1361: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54997/253971 objects degraded (21.655%); 33341/253971 objects misplaced (13.128%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:57.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:57.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:38:57.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/789309650' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:57.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:57.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:57.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:57.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:38:58.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:58.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:58.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:58.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:58.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:58.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:58.323 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:38:58.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:58.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:58.451 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:58.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:58.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:58.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:58.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:58.721 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:58.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:58.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:58.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:58.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:58.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:59.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:59.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:59.202 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:59.304 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:59.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:59.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:59.573 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:59.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:59.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:59.759 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:38:59 smithi152 ceph-mon[93456]: pgmap v1362: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54966/253230 objects degraded (21.706%); 33206/253230 objects misplaced (13.113%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:38:59.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:38:59 smithi142 ceph-mon[106729]: pgmap v1362: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54966/253230 objects degraded (21.706%); 33206/253230 objects misplaced (13.113%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:38:59.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:38:59 smithi142 ceph-mon[103614]: pgmap v1362: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54966/253230 objects degraded (21.706%); 33206/253230 objects misplaced (13.113%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:38:59.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:59.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:38:59.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:59.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:38:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:00.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:00.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:00.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:00.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:00.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:00.197 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:00.197 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:00.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:00.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:00.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:00.322 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:39:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:39:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:00.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:00.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:00.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:00.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:00.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:00.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:00.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:00.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:00.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:00.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:00.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:00.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:00.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:01.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:01.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:01.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:39:01.207 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:01.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:01.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:01.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:01.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:01.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[106729]: pgmap v1363: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 54933/252813 objects degraded (21.729%); 33146/252813 objects misplaced (13.111%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:01.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54933/252813 objects degraded (21.729%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:01.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1788887130' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:01.488 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:01.489 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[103614]: pgmap v1363: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 54933/252813 objects degraded (21.729%); 33146/252813 objects misplaced (13.111%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:01.489 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54933/252813 objects degraded (21.729%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:01.489 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1788887130' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:01.489 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:01.494 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:01.522 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:01.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:01.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:01.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:01.682 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:39:01.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:01.710 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:01.711 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:39:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:01.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:01.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:01 smithi152 ceph-mon[93456]: pgmap v1363: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 54933/252813 objects degraded (21.729%); 33146/252813 objects misplaced (13.111%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:01.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54933/252813 objects degraded (21.729%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:01.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1788887130' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:01.810 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:01 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:01.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:01.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:01.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:01.966 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:02.029 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:39:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:39:01] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-17T17:39:02.067 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:02.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:02.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:02.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:02.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:02.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:02.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:02.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:02.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:02.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:02.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:02.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:02 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:02 smithi142 ceph-mon[106729]: osdmap e371: 8 total, 8 up, 6 in 2024-08-17T17:39:02.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:02 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:02 smithi142 ceph-mon[103614]: osdmap e371: 8 total, 8 up, 6 in 2024-08-17T17:39:02.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:02.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:02.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:02.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:02 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:02.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:02.780 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:02 smithi152 ceph-mon[93456]: osdmap e371: 8 total, 8 up, 6 in 2024-08-17T17:39:02.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:02.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:02.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:02.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:02.959 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:02.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:03.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:03.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:03.182 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1364,"stamp":"2024-08-17T17:39:01.676643+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5515450341,"num_objects":84177,"num_object_clones":0,"num_object_copies":252531,"num_objects_missing_on_primary":5915,"num_objects_missing":5915,"num_objects_degraded":54933,"num_objects_misplaced":33146,"num_objects_unfound":0,"num_objects_dirty":84177,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":152517,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26586,"num_bytes_recovered":1619468942,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104765,"ondisk_log_size":104765,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16898804,"kb_used_data":11454916,"kb_used_omap":27,"kb_used_meta":5443812,"kb_avail":545572108,"statfs":{"total":575970213888,"available":558665838592,"internally_reserved":0,"allocated":11729833984,"data_stored":11718005263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5574464232},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,1,0,1,3,1,1,2,6,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-74121216,"num_objects":-1131,"num_object_clones":0,"num_object_copies":-3393,"num_objects_missing_on_primary":-57,"num_objects_missing":-57,"num_objects_degraded":-140,"num_objects_misplaced":-417,"num_objects_unfound":0,"num_objects_dirty":-1131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1131,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":13893632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.279971"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":582,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403355+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:38:50.403355+0000","last_peered":"2024-08-17T17:38:50.403355+0000","last_clean":"2024-08-17T17:38:50.403355+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:38:50.403355+0000","last_undegraded":"2024-08-17T17:38:50.403355+0000","last_fullsized":"2024-08-17T17:38:50.403355+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":79,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467542+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:38:43.467542+0000","last_peered":"2024-08-17T17:38:43.467542+0000","last_clean":"2024-08-17T17:38:43.467542+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:38:43.467542+0000","last_undegraded":"2024-08-17T17:38:43.467542+0000","last_fullsized":"2024-08-17T17:38:43.467542+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":273,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.495120+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:38:43.495120+0000","last_peered":"2024-08-17T17:38:43.495120+0000","last_clean":"2024-08-17T17:38:43.495120+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:38:43.495120+0000","last_undegraded":"2024-08-17T17:38:43.495120+0000","last_fullsized":"2024-08-17T17:38:43.495120+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":280,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401301+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:38:50.401301+0000","last_peered":"2024-08-17T17:38:50.401301+0000","last_clean":"2024-08-17T17:38:50.401301+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:38:50.401301+0000","last_undegraded":"2024-08-17T17:38:50.401301+0000","last_fullsized":"2024-08-17T17:38:50.401301+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":457,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403927+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:38:50.403927+0000","last_peered":"2024-08-17T17:38:50.403927+0000","last_clean":"2024-08-17T17:38:50.403927+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:38:50.403927+0000","last_undegraded":"2024-08-17T17:38:50.403927+0000","last_fullsized":"2024-08-17T17:38:50.403927+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":222,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467241+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:38:43.467241+0000","last_peered":"2024-08-17T17:38:43.467241+0000","last_clean":"2024-08-17T17:38:43.467241+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:38:43.467241+0000","last_undegraded":"2024-08-17T17:38:43.467241+0000","last_fullsized":"2024-08-17T17:38:43.467241+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":550,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403467+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:38:50.403467+0000","last_peered":"2024-08-17T17:38:50.403467+0000","last_clean":"2024-08-17T17:38:50.403467+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:38:50.403467+0000","last_undegraded":"2024-08-17T17:38:50.403467+0000","last_fullsized":"2024-08-17T17:38:50.403467+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":390,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482975+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:38:43.482975+0000","last_peered":"2024-08-17T17:38:43.482975+0000","last_clean":"2024-08-17T17:38:43.482975+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:38:43.482975+0000","last_undegraded":"2024-08-17T17:38:43.482975+0000","last_fullsized":"2024-08-17T17:38:43.482975+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"370'9514","reported_seq":12462,"reported_epoch":370,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:38:57.750267+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:38:57.750267+0000","last_peered":"2024-08-17T17:38:57.750267+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:38:57.750267+0000","last_undegraded":"2024-08-17T17:38:57.750267+0000","last_fullsized":"2024-08-17T17:38:57.750267+0000","mapping_epoch":337,"log_start":"325'2663","ondisk_log_start":"325'2663","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6851,"log_dups_size":0,"ondisk_log_size":6851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5190,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9514,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1433,"num_bytes_recovered":93913088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5190}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":75,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.531538+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:38:43.531538+0000","last_peered":"2024-08-17T17:38:43.531538+0000","last_clean":"2024-08-17T17:38:43.531538+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:38:43.531538+0000","last_undegraded":"2024-08-17T17:38:43.531538+0000","last_fullsized":"2024-08-17T17:38:43.531538+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":256,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.494588+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:38:43.494588+0000","last_peered":"2024-08-17T17:38:43.494588+0000","last_clean":"2024-08-17T17:38:43.494588+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:38:43.494588+0000","last_undegraded":"2024-08-17T17:38:43.494588+0000","last_fullsized":"2024-08-17T17:38:43.494588+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"370'9709","reported_seq":9777,"reported_epoch":370,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:56.704768+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:38:56.704768+0000","last_peered":"2024-08-17T17:38:56.704768+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:38:56.704768+0000","last_undegraded":"2024-08-17T17:38:56.704768+0000","last_fullsized":"2024-08-17T17:38:56.704768+0000","mapping_epoch":337,"log_start":"325'3182","ondisk_log_start":"325'3182","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351600640,"num_objects":5365,"num_object_clones":0,"num_object_copies":16095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5365,"num_objects_unfound":0,"num_objects_dirty":5365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9709,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5365}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":77,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467632+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:38:43.467632+0000","last_peered":"2024-08-17T17:38:43.467632+0000","last_clean":"2024-08-17T17:38:43.467632+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:38:43.467632+0000","last_undegraded":"2024-08-17T17:38:43.467632+0000","last_fullsized":"2024-08-17T17:38:43.467632+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":341,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467246+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:38:43.467246+0000","last_peered":"2024-08-17T17:38:43.467246+0000","last_clean":"2024-08-17T17:38:43.467246+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:38:43.467246+0000","last_undegraded":"2024-08-17T17:38:43.467246+0000","last_fullsized":"2024-08-17T17:38:43.467246+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"370'9432","reported_seq":9503,"reported_epoch":370,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:57.660743+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:38:57.660743+0000","last_peered":"2024-08-17T17:38:57.660743+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:38:57.660743+0000","last_undegraded":"2024-08-17T17:38:57.660743+0000","last_fullsized":"2024-08-17T17:38:57.660743+0000","mapping_epoch":337,"log_start":"324'2553","ondisk_log_start":"324'2553","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347471872,"num_objects":5302,"num_object_clones":0,"num_object_copies":15906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5302,"num_objects_unfound":0,"num_objects_dirty":5302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9432,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5302}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":73,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.495076+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:38:43.495076+0000","last_peered":"2024-08-17T17:38:43.495076+0000","last_clean":"2024-08-17T17:38:43.495076+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:38:43.495076+0000","last_undegraded":"2024-08-17T17:38:43.495076+0000","last_fullsized":"2024-08-17T17:38:43.495076+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":351,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.474603+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:38:43.474603+0000","last_peered":"2024-08-17T17:38:43.474603+0000","last_clean":"2024-08-17T17:38:43.474603+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:38:43.474603+0000","last_undegraded":"2024-08-17T17:38:43.474603+0000","last_fullsized":"2024-08-17T17:38:43.474603+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"370'9614","reported_seq":9733,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:59.116409+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:38:59.116409+0000","last_peered":"2024-08-17T17:38:59.116409+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:38:59.116409+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2613","ondisk_log_start":"324'2613","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350224384,"num_objects":5344,"num_object_clones":0,"num_object_copies":16032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9614,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1425},{"shards":"0,3,7","objects":5344}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":494,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.404076+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:38:50.404076+0000","last_peered":"2024-08-17T17:38:50.404076+0000","last_clean":"2024-08-17T17:38:50.404076+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:38:50.404076+0000","last_undegraded":"2024-08-17T17:38:50.404076+0000","last_fullsized":"2024-08-17T17:38:50.404076+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":260,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.435300+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:38:50.435300+0000","last_peered":"2024-08-17T17:38:50.435300+0000","last_clean":"2024-08-17T17:38:50.435300+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:38:50.435300+0000","last_undegraded":"2024-08-17T17:38:50.435300+0000","last_fullsized":"2024-08-17T17:38:50.435300+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"370'9385","reported_seq":9444,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.011638+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:38:57.011638+0000","last_peered":"2024-08-17T17:38:57.011638+0000","last_clean":"2024-08-17T17:38:57.011638+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:38:57.011638+0000","last_undegraded":"2024-08-17T17:38:57.011638+0000","last_fullsized":"2024-08-17T17:38:57.011638+0000","mapping_epoch":319,"log_start":"325'2774","ondisk_log_start":"325'2774","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341377024,"num_objects":5209,"num_object_clones":0,"num_object_copies":15627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9385,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":583,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401351+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:38:50.401351+0000","last_peered":"2024-08-17T17:38:50.401351+0000","last_clean":"2024-08-17T17:38:50.401351+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:38:50.401351+0000","last_undegraded":"2024-08-17T17:38:50.401351+0000","last_fullsized":"2024-08-17T17:38:50.401351+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467462+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:38:43.467462+0000","last_peered":"2024-08-17T17:38:43.467462+0000","last_clean":"2024-08-17T17:38:43.467462+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:38:43.467462+0000","last_undegraded":"2024-08-17T17:38:43.467462+0000","last_fullsized":"2024-08-17T17:38:43.467462+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"370'9522","reported_seq":9591,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:56.616518+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:38:56.616518+0000","last_peered":"2024-08-17T17:38:56.616518+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:38:56.616518+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'2935","ondisk_log_start":"325'2935","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354549760,"num_objects":5410,"num_object_clones":0,"num_object_copies":16230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9522,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5410},{"shards":"1,6","objects":-1956}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":325,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.474849+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:38:43.474849+0000","last_peered":"2024-08-17T17:38:43.474849+0000","last_clean":"2024-08-17T17:38:43.474849+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:38:43.474849+0000","last_undegraded":"2024-08-17T17:38:43.474849+0000","last_fullsized":"2024-08-17T17:38:43.474849+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":351,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467725+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:38:43.467725+0000","last_peered":"2024-08-17T17:38:43.467725+0000","last_clean":"2024-08-17T17:38:43.467725+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:38:43.467725+0000","last_undegraded":"2024-08-17T17:38:43.467725+0000","last_fullsized":"2024-08-17T17:38:43.467725+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"370'9676","reported_seq":9796,"reported_epoch":370,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:38:58.757952+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:38:58.757952+0000","last_peered":"2024-08-17T17:38:58.757952+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:38:58.757952+0000","last_undegraded":"2024-08-17T17:38:58.757952+0000","last_fullsized":"2024-08-17T17:38:58.757952+0000","mapping_epoch":337,"log_start":"326'4061","ondisk_log_start":"326'4061","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349569024,"num_objects":5334,"num_object_clones":0,"num_object_copies":16002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5334,"num_objects_unfound":0,"num_objects_dirty":5334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9676,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5334}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":362,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467329+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:38:43.467329+0000","last_peered":"2024-08-17T17:38:43.467329+0000","last_clean":"2024-08-17T17:38:43.467329+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:38:43.467329+0000","last_undegraded":"2024-08-17T17:38:43.467329+0000","last_fullsized":"2024-08-17T17:38:43.467329+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467738+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:38:43.467738+0000","last_peered":"2024-08-17T17:38:43.467738+0000","last_clean":"2024-08-17T17:38:43.467738+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:38:43.467738+0000","last_undegraded":"2024-08-17T17:38:43.467738+0000","last_fullsized":"2024-08-17T17:38:43.467738+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"370'9483","reported_seq":12420,"reported_epoch":370,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:57.639486+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:38:57.639486+0000","last_peered":"2024-08-17T17:38:57.639486+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:38:57.639486+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:38:57.639486+0000","mapping_epoch":337,"log_start":"325'2641","ondisk_log_start":"325'2641","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6842,"log_dups_size":0,"ondisk_log_size":6842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342163456,"num_objects":5221,"num_object_clones":0,"num_object_copies":15663,"num_objects_missing_on_primary":5915,"num_objects_missing":5915,"num_objects_degraded":5221,"num_objects_misplaced":5915,"num_objects_unfound":0,"num_objects_dirty":5221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9483,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93323264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":5209}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":553,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401589+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:38:50.401589+0000","last_peered":"2024-08-17T17:38:50.401589+0000","last_clean":"2024-08-17T17:38:50.401589+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:38:50.401589+0000","last_undegraded":"2024-08-17T17:38:50.401589+0000","last_fullsized":"2024-08-17T17:38:50.401589+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":490,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403556+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:38:50.403556+0000","last_peered":"2024-08-17T17:38:50.403556+0000","last_clean":"2024-08-17T17:38:50.403556+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:38:50.403556+0000","last_undegraded":"2024-08-17T17:38:50.403556+0000","last_fullsized":"2024-08-17T17:38:50.403556+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"370'9606","reported_seq":12531,"reported_epoch":370,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:38:58.661176+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:38:58.661176+0000","last_peered":"2024-08-17T17:38:58.661176+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:38:58.661176+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:38:58.661176+0000","mapping_epoch":337,"log_start":"326'4044","ondisk_log_start":"326'4044","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346619928,"num_objects":5290,"num_object_clones":0,"num_object_copies":15870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5290,"num_objects_misplaced":6040,"num_objects_unfound":0,"num_objects_dirty":5290,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9606,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1408,"num_bytes_recovered":92209152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5290}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":683,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:50.437142+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:38:50.437142+0000","last_peered":"2024-08-17T17:38:50.437142+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:38:50.437142+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":299,"reported_epoch":369,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:38:43.494803+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:38:43.494803+0000","last_peered":"2024-08-17T17:38:43.494803+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:38:43.494803+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:38:43.494803+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":532,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401179+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:38:50.401179+0000","last_peered":"2024-08-17T17:38:50.401179+0000","last_clean":"2024-08-17T17:38:50.401179+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:38:50.401179+0000","last_undegraded":"2024-08-17T17:38:50.401179+0000","last_fullsized":"2024-08-17T17:38:50.401179+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":334,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.475279+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:38:43.475279+0000","last_peered":"2024-08-17T17:38:43.475279+0000","last_clean":"2024-08-17T17:38:43.475279+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:38:43.475279+0000","last_undegraded":"2024-08-17T17:38:43.475279+0000","last_fullsized":"2024-08-17T17:38:43.475279+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"370'9491","reported_seq":9613,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:59.682254+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:38:59.682254+0000","last_peered":"2024-08-17T17:38:59.682254+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:38:59.682254+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2421","ondisk_log_start":"324'2421","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344260608,"num_objects":5253,"num_object_clones":0,"num_object_copies":15759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9491,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1394},{"shards":"0,4,7","objects":5253}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":348,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.398131+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:38:50.398131+0000","last_peered":"2024-08-17T17:38:50.398131+0000","last_clean":"2024-08-17T17:38:50.398131+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:38:50.398131+0000","last_undegraded":"2024-08-17T17:38:50.398131+0000","last_fullsized":"2024-08-17T17:38:50.398131+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":306,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467173+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:38:43.467173+0000","last_peered":"2024-08-17T17:38:43.467173+0000","last_clean":"2024-08-17T17:38:43.467173+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:38:43.467173+0000","last_undegraded":"2024-08-17T17:38:43.467173+0000","last_fullsized":"2024-08-17T17:38:43.467173+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":274,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.435998+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:38:50.435998+0000","last_peered":"2024-08-17T17:38:50.435998+0000","last_clean":"2024-08-17T17:38:50.435998+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:38:50.435998+0000","last_undegraded":"2024-08-17T17:38:50.435998+0000","last_fullsized":"2024-08-17T17:38:50.435998+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"370'9482","reported_seq":9551,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:57.013650+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:38:57.013650+0000","last_peered":"2024-08-17T17:38:57.013650+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:38:57.013650+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'2917","ondisk_log_start":"325'2917","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348258304,"num_objects":5314,"num_object_clones":0,"num_object_copies":15942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9482,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5314},{"shards":"5,6","objects":-1982}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":440,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400949+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:38:50.400949+0000","last_peered":"2024-08-17T17:38:50.400949+0000","last_clean":"2024-08-17T17:38:50.400949+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:38:50.400949+0000","last_undegraded":"2024-08-17T17:38:50.400949+0000","last_fullsized":"2024-08-17T17:38:50.400949+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":106,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.482838+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:38:43.482838+0000","last_peered":"2024-08-17T17:38:43.482838+0000","last_clean":"2024-08-17T17:38:43.482838+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:38:43.482838+0000","last_undegraded":"2024-08-17T17:38:43.482838+0000","last_fullsized":"2024-08-17T17:38:43.482838+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":625,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.398516+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:38:50.398516+0000","last_peered":"2024-08-17T17:38:50.398516+0000","last_clean":"2024-08-17T17:38:50.398516+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:38:50.398516+0000","last_undegraded":"2024-08-17T17:38:50.398516+0000","last_fullsized":"2024-08-17T17:38:50.398516+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"370'9424","reported_seq":10270,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:59.727577+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:38:59.727577+0000","last_peered":"2024-08-17T17:38:59.727577+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:38:59.727577+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2416","ondisk_log_start":"324'2416","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342228992,"num_objects":5222,"num_object_clones":0,"num_object_copies":15666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9424,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5222},{"shards":"0,5","objects":-1432}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":416,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.398898+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:38:50.398898+0000","last_peered":"2024-08-17T17:38:50.398898+0000","last_clean":"2024-08-17T17:38:50.398898+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:38:50.398898+0000","last_undegraded":"2024-08-17T17:38:50.398898+0000","last_fullsized":"2024-08-17T17:38:50.398898+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":345,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467775+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:38:43.467775+0000","last_peered":"2024-08-17T17:38:43.467775+0000","last_clean":"2024-08-17T17:38:43.467775+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:38:43.467775+0000","last_undegraded":"2024-08-17T17:38:43.467775+0000","last_fullsized":"2024-08-17T17:38:43.467775+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":211,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403274+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:38:50.403274+0000","last_peered":"2024-08-17T17:38:50.403274+0000","last_clean":"2024-08-17T17:38:50.403274+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:38:50.403274+0000","last_undegraded":"2024-08-17T17:38:50.403274+0000","last_fullsized":"2024-08-17T17:38:50.403274+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"370'9590","reported_seq":24555,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:39:00.486902+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:39:00.486902+0000","last_peered":"2024-08-17T17:39:00.486902+0000","last_clean":"2024-08-17T17:39:00.486902+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:39:00.486902+0000","last_undegraded":"2024-08-17T17:39:00.486902+0000","last_fullsized":"2024-08-17T17:39:00.486902+0000","mapping_epoch":355,"log_start":"325'3330","ondisk_log_start":"325'3330","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343801856,"num_objects":5246,"num_object_clones":0,"num_object_copies":15738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9590,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.494997+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:38:43.494997+0000","last_peered":"2024-08-17T17:38:43.494997+0000","last_clean":"2024-08-17T17:38:43.494997+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:38:43.494997+0000","last_undegraded":"2024-08-17T17:38:43.494997+0000","last_fullsized":"2024-08-17T17:38:43.494997+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":80,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403177+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:38:50.403177+0000","last_peered":"2024-08-17T17:38:50.403177+0000","last_clean":"2024-08-17T17:38:50.403177+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:38:50.403177+0000","last_undegraded":"2024-08-17T17:38:50.403177+0000","last_fullsized":"2024-08-17T17:38:50.403177+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":575,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.399185+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:38:50.399185+0000","last_peered":"2024-08-17T17:38:50.399185+0000","last_clean":"2024-08-17T17:38:50.399185+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:38:50.399185+0000","last_undegraded":"2024-08-17T17:38:50.399185+0000","last_fullsized":"2024-08-17T17:38:50.399185+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"370'9293","reported_seq":23826,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:39:00.045265+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:39:00.045265+0000","last_peered":"2024-08-17T17:39:00.045265+0000","last_clean":"2024-08-17T17:39:00.045265+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:39:00.045265+0000","last_undegraded":"2024-08-17T17:39:00.045265+0000","last_fullsized":"2024-08-17T17:39:00.045265+0000","mapping_epoch":345,"log_start":"325'2972","ondisk_log_start":"325'2972","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334954496,"num_objects":5111,"num_object_clones":0,"num_object_copies":15333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9293,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":280,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.494980+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:38:43.494980+0000","last_peered":"2024-08-17T17:38:43.494980+0000","last_clean":"2024-08-17T17:38:43.494980+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:38:43.494980+0000","last_undegraded":"2024-08-17T17:38:43.494980+0000","last_fullsized":"2024-08-17T17:38:43.494980+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":331,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.434559+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:38:50.434559+0000","last_peered":"2024-08-17T17:38:50.434559+0000","last_clean":"2024-08-17T17:38:50.434559+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:38:50.434559+0000","last_undegraded":"2024-08-17T17:38:50.434559+0000","last_fullsized":"2024-08-17T17:38:50.434559+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":422,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.494699+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:38:43.494699+0000","last_peered":"2024-08-17T17:38:43.494699+0000","last_clean":"2024-08-17T17:38:43.494699+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:38:43.494699+0000","last_undegraded":"2024-08-17T17:38:43.494699+0000","last_fullsized":"2024-08-17T17:38:43.494699+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"370'9563","reported_seq":9704,"reported_epoch":370,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:00.455481+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:39:00.455481+0000","last_peered":"2024-08-17T17:39:00.455481+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:39:00.455481+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'3287","ondisk_log_start":"325'3287","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6276,"log_dups_size":0,"ondisk_log_size":6276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342163456,"num_objects":5221,"num_object_clones":0,"num_object_copies":15663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9563,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5221},{"shards":"0,3","objects":-2027}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":606,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403640+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:38:50.403640+0000","last_peered":"2024-08-17T17:38:50.403640+0000","last_clean":"2024-08-17T17:38:50.403640+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:38:50.403640+0000","last_undegraded":"2024-08-17T17:38:50.403640+0000","last_fullsized":"2024-08-17T17:38:50.403640+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":574,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400060+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:38:50.400060+0000","last_peered":"2024-08-17T17:38:50.400060+0000","last_clean":"2024-08-17T17:38:50.400060+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:38:50.400060+0000","last_undegraded":"2024-08-17T17:38:50.400060+0000","last_fullsized":"2024-08-17T17:38:50.400060+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":592,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401449+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:38:50.401449+0000","last_peered":"2024-08-17T17:38:50.401449+0000","last_clean":"2024-08-17T17:38:50.401449+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:38:50.401449+0000","last_undegraded":"2024-08-17T17:38:50.401449+0000","last_fullsized":"2024-08-17T17:38:50.401449+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"370'9455","reported_seq":24122,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:56.758634+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:38:56.758634+0000","last_peered":"2024-08-17T17:38:56.758634+0000","last_clean":"2024-08-17T17:38:56.758634+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:38:56.758634+0000","last_undegraded":"2024-08-17T17:38:56.758634+0000","last_fullsized":"2024-08-17T17:38:56.758634+0000","mapping_epoch":347,"log_start":"325'2872","ondisk_log_start":"325'2872","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9455,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":264,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.435606+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:38:50.435606+0000","last_peered":"2024-08-17T17:38:50.435606+0000","last_clean":"2024-08-17T17:38:50.435606+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:38:50.435606+0000","last_undegraded":"2024-08-17T17:38:50.435606+0000","last_fullsized":"2024-08-17T17:38:50.435606+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":374,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.477984+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:38:43.477984+0000","last_peered":"2024-08-17T17:38:43.477984+0000","last_clean":"2024-08-17T17:38:43.477984+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:38:43.477984+0000","last_undegraded":"2024-08-17T17:38:43.477984+0000","last_fullsized":"2024-08-17T17:38:43.477984+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467330+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:38:43.467330+0000","last_peered":"2024-08-17T17:38:43.467330+0000","last_clean":"2024-08-17T17:38:43.467330+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:38:43.467330+0000","last_undegraded":"2024-08-17T17:38:43.467330+0000","last_fullsized":"2024-08-17T17:38:43.467330+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467715+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:38:43.467715+0000","last_peered":"2024-08-17T17:38:43.467715+0000","last_clean":"2024-08-17T17:38:43.467715+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:38:43.467715+0000","last_undegraded":"2024-08-17T17:38:43.467715+0000","last_fullsized":"2024-08-17T17:38:43.467715+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":367,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467574+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:38:43.467574+0000","last_peered":"2024-08-17T17:38:43.467574+0000","last_clean":"2024-08-17T17:38:43.467574+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:38:43.467574+0000","last_undegraded":"2024-08-17T17:38:43.467574+0000","last_fullsized":"2024-08-17T17:38:43.467574+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":581,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.398990+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:38:50.398990+0000","last_peered":"2024-08-17T17:38:50.398990+0000","last_clean":"2024-08-17T17:38:50.398990+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:38:50.398990+0000","last_undegraded":"2024-08-17T17:38:50.398990+0000","last_fullsized":"2024-08-17T17:38:50.398990+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":121,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.404223+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:38:50.404223+0000","last_peered":"2024-08-17T17:38:50.404223+0000","last_clean":"2024-08-17T17:38:50.404223+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:38:50.404223+0000","last_undegraded":"2024-08-17T17:38:50.404223+0000","last_fullsized":"2024-08-17T17:38:50.404223+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":274,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.495212+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:38:43.495212+0000","last_peered":"2024-08-17T17:38:43.495212+0000","last_clean":"2024-08-17T17:38:43.495212+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:38:43.495212+0000","last_undegraded":"2024-08-17T17:38:43.495212+0000","last_fullsized":"2024-08-17T17:38:43.495212+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":539,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400671+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:38:50.400671+0000","last_peered":"2024-08-17T17:38:50.400671+0000","last_clean":"2024-08-17T17:38:50.400671+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:38:50.400671+0000","last_undegraded":"2024-08-17T17:38:50.400671+0000","last_fullsized":"2024-08-17T17:38:50.400671+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":418,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.495454+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:38:43.495454+0000","last_peered":"2024-08-17T17:38:43.495454+0000","last_clean":"2024-08-17T17:38:43.495454+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:38:43.495454+0000","last_undegraded":"2024-08-17T17:38:43.495454+0000","last_fullsized":"2024-08-17T17:38:43.495454+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":533,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400623+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:38:50.400623+0000","last_peered":"2024-08-17T17:38:50.400623+0000","last_clean":"2024-08-17T17:38:50.400623+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:38:50.400623+0000","last_undegraded":"2024-08-17T17:38:50.400623+0000","last_fullsized":"2024-08-17T17:38:50.400623+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":341,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.477872+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:38:43.477872+0000","last_peered":"2024-08-17T17:38:43.477872+0000","last_clean":"2024-08-17T17:38:43.477872+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:38:43.477872+0000","last_undegraded":"2024-08-17T17:38:43.477872+0000","last_fullsized":"2024-08-17T17:38:43.477872+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":564,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401518+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:38:50.401518+0000","last_peered":"2024-08-17T17:38:50.401518+0000","last_clean":"2024-08-17T17:38:50.401518+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:38:50.401518+0000","last_undegraded":"2024-08-17T17:38:50.401518+0000","last_fullsized":"2024-08-17T17:38:50.401518+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":280,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.403826+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:38:50.403826+0000","last_peered":"2024-08-17T17:38:50.403826+0000","last_clean":"2024-08-17T17:38:50.403826+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:38:50.403826+0000","last_undegraded":"2024-08-17T17:38:50.403826+0000","last_fullsized":"2024-08-17T17:38:50.403826+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":572,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.399346+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:38:50.399346+0000","last_peered":"2024-08-17T17:38:50.399346+0000","last_clean":"2024-08-17T17:38:50.399346+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:38:50.399346+0000","last_undegraded":"2024-08-17T17:38:50.399346+0000","last_fullsized":"2024-08-17T17:38:50.399346+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":532,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.404018+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:38:50.404018+0000","last_peered":"2024-08-17T17:38:50.404018+0000","last_clean":"2024-08-17T17:38:50.404018+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:38:50.404018+0000","last_undegraded":"2024-08-17T17:38:50.404018+0000","last_fullsized":"2024-08-17T17:38:50.404018+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":353,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467444+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:38:43.467444+0000","last_peered":"2024-08-17T17:38:43.467444+0000","last_clean":"2024-08-17T17:38:43.467444+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:38:43.467444+0000","last_undegraded":"2024-08-17T17:38:43.467444+0000","last_fullsized":"2024-08-17T17:38:43.467444+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":221,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.404309+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:38:50.404309+0000","last_peered":"2024-08-17T17:38:50.404309+0000","last_clean":"2024-08-17T17:38:50.404309+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:38:50.404309+0000","last_undegraded":"2024-08-17T17:38:50.404309+0000","last_fullsized":"2024-08-17T17:38:50.404309+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":277,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.435208+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:38:50.435208+0000","last_peered":"2024-08-17T17:38:50.435208+0000","last_clean":"2024-08-17T17:38:50.435208+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:38:50.435208+0000","last_undegraded":"2024-08-17T17:38:50.435208+0000","last_fullsized":"2024-08-17T17:38:50.435208+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":361,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467646+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:38:43.467646+0000","last_peered":"2024-08-17T17:38:43.467646+0000","last_clean":"2024-08-17T17:38:43.467646+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:38:43.467646+0000","last_undegraded":"2024-08-17T17:38:43.467646+0000","last_fullsized":"2024-08-17T17:38:43.467646+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":224,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467508+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:38:43.467508+0000","last_peered":"2024-08-17T17:38:43.467508+0000","last_clean":"2024-08-17T17:38:43.467508+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:38:43.467508+0000","last_undegraded":"2024-08-17T17:38:43.467508+0000","last_fullsized":"2024-08-17T17:38:43.467508+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":361,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467320+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:38:43.467320+0000","last_peered":"2024-08-17T17:38:43.467320+0000","last_clean":"2024-08-17T17:38:43.467320+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:38:43.467320+0000","last_undegraded":"2024-08-17T17:38:43.467320+0000","last_fullsized":"2024-08-17T17:38:43.467320+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":224,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467637+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:38:43.467637+0000","last_peered":"2024-08-17T17:38:43.467637+0000","last_clean":"2024-08-17T17:38:43.467637+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:38:43.467637+0000","last_undegraded":"2024-08-17T17:38:43.467637+0000","last_fullsized":"2024-08-17T17:38:43.467637+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":584,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401025+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:38:50.401025+0000","last_peered":"2024-08-17T17:38:50.401025+0000","last_clean":"2024-08-17T17:38:50.401025+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:38:50.401025+0000","last_undegraded":"2024-08-17T17:38:50.401025+0000","last_fullsized":"2024-08-17T17:38:50.401025+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":558,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.404024+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:38:50.404024+0000","last_peered":"2024-08-17T17:38:50.404024+0000","last_clean":"2024-08-17T17:38:50.404024+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:38:50.404024+0000","last_undegraded":"2024-08-17T17:38:50.404024+0000","last_fullsized":"2024-08-17T17:38:50.404024+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":324,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.467195+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:38:43.467195+0000","last_peered":"2024-08-17T17:38:43.467195+0000","last_clean":"2024-08-17T17:38:43.467195+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:38:43.467195+0000","last_undegraded":"2024-08-17T17:38:43.467195+0000","last_fullsized":"2024-08-17T17:38:43.467195+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":279,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.434850+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:38:50.434850+0000","last_peered":"2024-08-17T17:38:50.434850+0000","last_clean":"2024-08-17T17:38:50.434850+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:38:50.434850+0000","last_undegraded":"2024-08-17T17:38:50.434850+0000","last_fullsized":"2024-08-17T17:38:50.434850+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5514854424,"num_objects":84151,"num_object_clones":0,"num_object_copies":252453,"num_objects_missing_on_primary":5915,"num_objects_missing":5915,"num_objects_degraded":54930,"num_objects_misplaced":33146,"num_objects_unfound":0,"num_objects_dirty":84151,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152239,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26545,"num_bytes_recovered":1617100800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17814532096,"data_stored":17814519880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104558,"ondisk_log_size":104558,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170071,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463440,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3258744,"kb_used_data":2356720,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":90486408,"statfs":{"total":95995035648,"available":92658081792,"internally_reserved":0,"allocated":2413281280,"data_stored":2411297631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492115,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065116,"kb_used_data":678100,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92680036,"statfs":{"total":95995035648,"available":94904356864,"internally_reserved":0,"allocated":694374400,"data_stored":692413051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986706,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2532460,"kb_used_data":1343140,"kb_used_omap":1,"kb_used_meta":1189310,"kb_avail":91212692,"statfs":{"total":95995035648,"available":93401796608,"internally_reserved":0,"allocated":1375375360,"data_stored":1373402651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1217853784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,0,1,3,1,1,2,6,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225534,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3608924,"kb_used_data":2684884,"kb_used_omap":4,"kb_used_meta":924027,"kb_avail":90136228,"statfs":{"total":95995035648,"available":92299497472,"internally_reserved":0,"allocated":2749321216,"data_stored":2747360165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":946204096},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075859,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533752,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3407456,"kb_used_data":2361304,"kb_used_omap":5,"kb_used_meta":1046138,"kb_avail":90337696,"statfs":{"total":95995035648,"available":92505800704,"internally_reserved":0,"allocated":2417975296,"data_stored":2416008923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1071245552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089729,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3026104,"kb_used_data":2030768,"kb_used_omap":5,"kb_used_meta":995322,"kb_avail":90719048,"statfs":{"total":95995035648,"available":92896305152,"internally_reserved":0,"allocated":2079506432,"data_stored":2077522842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1019209787},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2072444928,"data_stored":2072444928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2411859968,"data_stored":2411855896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3510112256,"data_stored":3510108184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2741960704,"data_stored":2741960704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1369047040,"data_stored":1369047040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":689438720,"data_stored":689438720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2407858176,"data_stored":2407858176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2611810304,"data_stored":2611806232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:03.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:03.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:03.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:03.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:03.341 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:03.342 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:03.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:03.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:03.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:03.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:03.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:03.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:03.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:03.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:03 smithi152 ceph-mon[93456]: from='client.37380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:03.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:03 smithi152 ceph-mon[93456]: pgmap v1365: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 54933/252075 objects degraded (21.792%); 33116/252075 objects misplaced (13.137%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:03.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:03 smithi142 ceph-mon[106729]: from='client.37380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:03.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:03 smithi142 ceph-mon[106729]: pgmap v1365: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 54933/252075 objects degraded (21.792%); 33116/252075 objects misplaced (13.137%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:03.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:03 smithi142 ceph-mon[103614]: from='client.37380 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:03.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:03 smithi142 ceph-mon[103614]: pgmap v1365: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 54933/252075 objects degraded (21.792%); 33116/252075 objects misplaced (13.137%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:03.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:03.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:03.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:03.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:04.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:04.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:39:04.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:04.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:04.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:04.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:04.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:04.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:04.393 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:04.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:04.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:04.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:04.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:04.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:04.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:04.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:04.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:04.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:04.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:05.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:05.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:05.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:05.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:05.192 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:39:05.192 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:05.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:05.209 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:39:05.210 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:05.229 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:39:05.229 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:05.246 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:39:05.247 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:05.263 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:39:05.263 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:05.278 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:39:05.279 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:05.294 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:39:05.294 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:05.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:05.310 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:39:05.310 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:05.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:05.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:05.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:05.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:05.626 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:05 smithi152 ceph-mon[93456]: pgmap v1366: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 54875/251652 objects degraded (21.806%); 32973/251652 objects misplaced (13.103%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:05.626 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4150998971' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:05.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:05.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:05.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:05.748 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:39:05.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:05 smithi142 ceph-mon[106729]: pgmap v1366: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 54875/251652 objects degraded (21.806%); 32973/251652 objects misplaced (13.103%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:05.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4150998971' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:05.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:05 smithi142 ceph-mon[103614]: pgmap v1366: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 54875/251652 objects degraded (21.806%); 32973/251652 objects misplaced (13.103%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:05.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4150998971' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:05.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:05.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:05.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:05.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:06.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:06.050 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:06.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:06.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:06.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:06.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:06.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:135101 12.c e353/336 12:31ea3325:::benchmark_data_smithi142_261410_object16904:head v 353'8351, mlcod=324'669)", 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:49.354929+0000", 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 196.91262723700001, 2024-08-17T17:39:06.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043811529999999996, 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.354929+0000", 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.354929+0000", 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.354929+0000", 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.354937+0000", 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8669999999999994e-06 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.354950+0000", 2024-08-17T17:39:06.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3219e-05 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.354959+0000", 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7790000000000001e-06 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.355045+0000", 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5984999999999998e-05 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.355072+0000", 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7046000000000002e-05 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.359268+0000", 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041965910000000004 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.359311+0000", 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2049999999999999e-05 2024-08-17T17:39:06.274 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:135102 12.f e353/332 12:f10a3458:::benchmark_data_smithi142_261410_object16905:head v 353'8460, mlcod=322'102)", 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:49.363006+0000", 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 196.90455030199999, 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064529970000000002, 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363006+0000", 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363006+0000", 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363006+0000", 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363013+0000", 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.939e-06 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363025+0000", 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2626000000000001e-05 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363033+0000", 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5800000000000003e-06 2024-08-17T17:39:06.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363086+0000", 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3189000000000002e-05 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.363116+0000", 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0045000000000001e-05 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.369419+0000", 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063027530000000003 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.369459+0000", 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0543000000000001e-05 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:134894 12.7 e352/337 12:e0ed92c0:::benchmark_data_smithi142_261410_object16697:head v 352'8432, mlcod=324'722)", 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:49.366743+0000", 2024-08-17T17:39:06.279 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 196.900813379, 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005073335, 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.366743+0000", 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.366743+0000", 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.366743+0000", 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.366747+0000", 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2209999999999999e-06 2024-08-17T17:39:06.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.366755+0000", 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1149999999999994e-06 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.366759+0000", 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0049999999999998e-06 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.369480+0000", 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027204479999999999 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.369495+0000", 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5296999999999999e-05 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.371787+0000", 2024-08-17T17:39:06.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022918830000000002 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:49.371817+0000", 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.972e-05 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:135769 12.c e353/336 12:363b060d:::benchmark_data_smithi142_261410_object17572:head v 353'8393, mlcod=324'669)", 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:58.768664+0000", 2024-08-17T17:39:06.284 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 187.49889290300001, 2024-08-17T17:39:06.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042137920000000001, 2024-08-17T17:39:06.285 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.285 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.285 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768664+0000", 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768664+0000", 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768663+0000", 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768672+0000", 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1929999999999999e-06 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768688+0000", 2024-08-17T17:39:06.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6011999999999998e-05 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768694+0000", 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0889999999999996e-06 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768741+0000", 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7271000000000002e-05 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.768768+0000", 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6746000000000001e-05 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.772832+0000", 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040638130000000003 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.772878+0000", 2024-08-17T17:39:06.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5878999999999999e-05 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:137769 12.f e356/332 12:fdbe3c73:::benchmark_data_smithi142_261410_object19572:head v 356'8625, mlcod=322'102)", 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:23.686291+0000", 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 162.581265793, 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042661180000000002, 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686291+0000", 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686291+0000", 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686289+0000", 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686298+0000", 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1749999999999994e-06 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686316+0000", 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7714999999999999e-05 2024-08-17T17:39:06.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686323+0000", 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.64e-06 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686377+0000", 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3820999999999999e-05 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.686405+0000", 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8683e-05 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.690517+0000", 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041117259999999996 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:23.690557+0000", 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9912e-05 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.294 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:137567 12.7 e356/337 12:e036a3a7:::benchmark_data_smithi142_261410_object19370:head v 356'8621, mlcod=324'722)", 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:24.430689+0000", 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 161.83686774899999, 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043590290000000004, 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430689+0000", 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430689+0000", 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430687+0000", 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430698+0000", 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0795000000000001e-05 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430720+0000", 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1546999999999999e-05 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430728+0000", 2024-08-17T17:39:06.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4929999999999998e-06 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430786+0000", 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7355000000000002e-05 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.430805+0000", 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9468999999999999e-05 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.435004+0000", 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004199024 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:24.435048+0000", 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3856000000000002e-05 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.299 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:138766 12.c e357/336 12:3c0d727e:::benchmark_data_smithi142_261410_object20569:head v 357'8565, mlcod=324'669)", 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:37.750125+0000", 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 148.51743157199999, 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062698420000000003, 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750125+0000", 2024-08-17T17:39:06.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750125+0000", 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750124+0000", 2024-08-17T17:39:06.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750133+0000", 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0389999999999993e-06 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750150+0000", 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6546999999999999e-05 2024-08-17T17:39:06.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750157+0000", 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1019999999999999e-06 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750205+0000", 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8791000000000001e-05 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.750238+0000", 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2097000000000001e-05 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.756330+0000", 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060925559999999998 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:37.756395+0000", 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4794000000000003e-05 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.304 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:140280 12.c e359/336 12:3fa94ca7:::benchmark_data_smithi142_261410_object22083:head v 359'8670, mlcod=324'669)", 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:54.251353+0000", 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 132.01620393600001, 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041101790000000003, 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251353+0000", 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251353+0000", 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251351+0000", 2024-08-17T17:39:06.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251363+0000", 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1531e-05 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251383+0000", 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0458e-05 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251392+0000", 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9919999999999996e-06 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251456+0000", 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3546000000000004e-05 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.251482+0000", 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6512999999999998e-05 2024-08-17T17:39:06.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.255381+0000", 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038988260000000002 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:54.255463+0000", 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1631999999999994e-05 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:140609 12.c e360/336 12:3ad91812:::benchmark_data_smithi142_261410_object22412:head v 360'8689, mlcod=324'669)", 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:57.968929+0000", 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 128.29862801900001, 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040911330000000003, 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.968929+0000", 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.968929+0000", 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.968927+0000", 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.968937+0000", 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6639999999999993e-06 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.968958+0000", 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1308999999999999e-05 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.968967+0000", 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5299999999999996e-06 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.969024+0000", 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6956000000000002e-05 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.969054+0000", 2024-08-17T17:39:06.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9887e-05 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.972962+0000", 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003908183 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:57.973020+0000", 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7877999999999997e-05 2024-08-17T17:39:06.314 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:144196 12.7 e363/337 12:e0383ecd:::benchmark_data_smithi142_261410_object25999:head v 363'9023, mlcod=324'722)", 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:39.770194+0000", 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 86.497363098999998, 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045901190000000001, 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770194+0000", 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770194+0000", 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770192+0000", 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770201+0000", 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1060000000000006e-06 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770218+0000", 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6115000000000001e-05 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770224+0000", 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9349999999999999e-06 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770283+0000", 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9596000000000003e-05 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.770304+0000", 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1375000000000001e-05 2024-08-17T17:39:06.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.774731+0000", 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004426485 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.774784+0000", 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2669e-05 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:146440 12.0 e365/337 12:0042ccba:::benchmark_data_smithi142_261410_object28243:head v 365'9121, mlcod=324'731)", 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:59.958590+0000", 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 66.308966670999993, 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042419689999999999, 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958590+0000", 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958590+0000", 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958589+0000", 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958599+0000", 2024-08-17T17:39:06.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0593e-05 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958622+0000", 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2493e-05 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958630+0000", 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9510000000000006e-06 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958689+0000", 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9530000000000001e-05 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.958712+0000", 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3085e-05 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.962783+0000", 2024-08-17T17:39:06.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004070344 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:59.962832+0000", 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9311000000000003e-05 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:146751 12.c e365/336 12:3397af55:::benchmark_data_smithi142_261410_object28554:head v 365'9091, mlcod=324'669)", 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:02.877359+0000", 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 63.390197196000003, 2024-08-17T17:39:06.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067784400000000002, 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877359+0000", 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877359+0000", 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877358+0000", 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877370+0000", 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2197e-05 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877388+0000", 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8019000000000001e-05 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877395+0000", 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0049999999999998e-06 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877443+0000", 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7224000000000001e-05 2024-08-17T17:39:06.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.877468+0000", 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5626000000000002e-05 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.884093+0000", 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066243869999999998 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.884138+0000", 2024-08-17T17:39:06.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5191999999999997e-05 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:147408 12.f e365/332 12:f06045c8:::benchmark_data_smithi142_261410_object29211:head v 365'9185, mlcod=322'102)", 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:09.376464+0000", 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 56.891092243000003, 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041367210000000003, 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376464+0000", 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376464+0000", 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376463+0000", 2024-08-17T17:39:06.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376474+0000", 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0744000000000001e-05 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376493+0000", 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9207000000000001e-05 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376501+0000", 2024-08-17T17:39:06.332 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:06.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7009999999999996e-06 2024-08-17T17:39:06.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376548+0000", 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7373000000000002e-05 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.376568+0000", 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9415e-05 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.380559+0000", 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039912239999999998 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:09.380601+0000", 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2231000000000003e-05 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:149337 12.f e367/332 12:f050e5ce:::benchmark_data_smithi142_261410_object31140:head v 367'9313, mlcod=322'102)", 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:28.271039+0000", 2024-08-17T17:39:06.335 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 37.996518010999999, 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041540800000000001, 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271039+0000", 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271039+0000", 2024-08-17T17:39:06.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271037+0000", 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271047+0000", 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0509000000000001e-05 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271066+0000", 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8700000000000001e-05 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271074+0000", 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4490000000000002e-06 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271124+0000", 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0176000000000002e-05 2024-08-17T17:39:06.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.271153+0000", 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8781000000000001e-05 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.275149+0000", 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039967309999999999 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:28.275193+0000", 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3381999999999999e-05 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:149980 12.0 e367/337 12:083a1815:::benchmark_data_smithi142_261410_object31783:head v 367'9370, mlcod=324'731)", 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:34.394044+0000", 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 31.873512315999999, 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055483190000000003, 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.340 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394044+0000", 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394044+0000", 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394043+0000", 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394053+0000", 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0265000000000001e-05 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394074+0000", 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1248999999999999e-05 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394082+0000", 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8730000000000007e-06 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394162+0000", 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9721999999999993e-05 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.394181+0000", 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9038e-05 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.399550+0000", 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053692710000000001 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.399593+0000", 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2253999999999999e-05 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.344 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:150042 12.c e368/336 12:33a98036:::benchmark_data_smithi142_261410_object31845:head v 368'9272, mlcod=324'669)", 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:34.935186+0000", 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 31.332370312999998, 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040815219999999998, 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.345 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935186+0000", 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935186+0000", 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935185+0000", 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935196+0000", 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1409999999999999e-05 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935218+0000", 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1435000000000001e-05 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935226+0000", 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9000000000000006e-06 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935300+0000", 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4609000000000003e-05 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.935326+0000", 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5505999999999999e-05 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.939222+0000", 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038959250000000002 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:34.939268+0000", 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6276999999999998e-05 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:150822 12.c e369/336 12:371888e0:::benchmark_data_smithi142_261410_object32625:head v 369'9323, mlcod=324'669)", 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:44.839663+0000", 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 21.427893207, 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052523480000000004, 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839663+0000", 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839663+0000", 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839662+0000", 2024-08-17T17:39:06.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839676+0000", 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3441999999999999e-05 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839694+0000", 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8887999999999999e-05 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839702+0000", 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9759999999999995e-06 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839751+0000", 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8544000000000001e-05 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.839775+0000", 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4491e-05 2024-08-17T17:39:06.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.844865+0000", 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050893639999999999 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:44.844916+0000", 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0933000000000002e-05 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.354 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:151572 12.0 e370/337 12:02615c7d:::benchmark_data_smithi142_261410_object33375:head v 370'9472, mlcod=324'731)", 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:51.819043+0000", 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 14.448513591999999, 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040698330000000001, 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819043+0000", 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819043+0000", 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819041+0000", 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819057+0000", 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5540999999999999e-05 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819081+0000", 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4763e-05 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819091+0000", 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3500000000000003e-06 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819163+0000", 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2680000000000002e-05 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.819187+0000", 2024-08-17T17:39:06.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3782999999999999e-05 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.822941+0000", 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037535699999999999 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:51.823113+0000", 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000172134 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:151748 12.f e370/332 12:f62db41d:::benchmark_data_smithi142_261410_object33551:head v 370'9460, mlcod=322'102)", 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:53.655621+0000", 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 12.61193608, 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048633089999999997, 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.655621+0000", 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.655621+0000", 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.655619+0000", 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.655630+0000", 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.119e-05 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.656795+0000", 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011644489999999999 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.656804+0000", 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7660000000000004e-06 2024-08-17T17:39:06.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.656886+0000", 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2369999999999999e-05 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.656923+0000", 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6653999999999998e-05 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.660432+0000", 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035095040000000001 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.660484+0000", 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1635999999999998e-05 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:151770 12.c e370/336 12:38104f25:::benchmark_data_smithi142_261410_object33573:head v 370'9377, mlcod=324'669)", 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:53.752292+0000", 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 12.515264869999999, 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040907019999999999, 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752292+0000", 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752292+0000", 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752290+0000", 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:39:06.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752303+0000", 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2611e-05 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752326+0000", 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3258e-05 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752335+0000", 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5909999999999996e-06 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752405+0000", 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0145e-05 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.752429+0000", 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4165e-05 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.756334+0000", 2024-08-17T17:39:06.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039050740000000001 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.756382+0000", 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8331000000000002e-05 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:39:06.369 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:39:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:06.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:06.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:06.493 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:06.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54875/251652 objects degraded (21.806%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:06.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:06.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:06.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:06.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:06.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:06.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54875/251652 objects degraded (21.806%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54875/251652 objects degraded (21.806%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:06.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:06.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:06.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:07.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:07.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:07.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:39:07.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:07.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:07.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:07.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:07.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:07.435 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:07.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:07.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:07.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:07.608 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:07.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:07.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:07.717 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:07 smithi152 ceph-mon[93456]: pgmap v1367: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54875/251124 objects degraded (21.852%); 32973/251124 objects misplaced (13.130%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:07.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:07.742 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:07 smithi142 ceph-mon[106729]: pgmap v1367: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54875/251124 objects degraded (21.852%); 32973/251124 objects misplaced (13.130%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:07.742 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:07 smithi142 ceph-mon[103614]: pgmap v1367: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54875/251124 objects degraded (21.852%); 32973/251124 objects misplaced (13.130%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:07.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:07.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:07.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:07.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:07.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:07.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:08.032 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:08.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:08.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:08.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:08.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:08.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:08.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:08.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:08.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:08.466 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:39:08.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:08.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:08.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:08.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:08.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3849561158' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:08.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:08 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:08.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3849561158' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:08.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:08 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:08.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3849561158' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:08.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:08 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:08.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:08.744 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:08.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:08.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:08.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:08.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:08.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:09.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:09.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:09.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:09.226 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:39:09.227 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:39:09.227 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:39:09.227 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:39:09.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:09.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:09.342 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:09.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:09.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:09.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:09.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:09.543 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:09 smithi152 ceph-mon[93456]: pgmap v1368: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 54844/250461 objects degraded (21.897%); 32860/250461 objects misplaced (13.120%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:39:09.543 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:09 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:09.543 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:09.543 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:09 smithi152 ceph-mon[93456]: osdmap e372: 8 total, 8 up, 6 in 2024-08-17T17:39:09.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:09.613 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:09.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:09.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:09.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:09.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[106729]: pgmap v1368: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 54844/250461 objects degraded (21.897%); 32860/250461 objects misplaced (13.120%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:39:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[106729]: osdmap e372: 8 total, 8 up, 6 in 2024-08-17T17:39:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[103614]: pgmap v1368: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 54844/250461 objects degraded (21.897%); 32860/250461 objects misplaced (13.120%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:39:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:09.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:09 smithi142 ceph-mon[103614]: osdmap e372: 8 total, 8 up, 6 in 2024-08-17T17:39:09.894 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:39:09.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:09.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:09.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:09.996 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:10.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:10.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:10.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:10.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:10.227 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:10.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:10.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:10.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:10.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:10.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:10.494 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:39:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:39:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:10.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:10.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:10.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:10.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:10.804 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:10 smithi142 ceph-mon[106729]: pgmap v1370: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 54826/250101 objects degraded (21.922%); 32818/250101 objects misplaced (13.122%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:10.805 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:10 smithi142 ceph-mon[103614]: pgmap v1370: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 54826/250101 objects degraded (21.922%); 32818/250101 objects misplaced (13.122%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:10.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:10 smithi152 ceph-mon[93456]: pgmap v1370: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 54826/250101 objects degraded (21.922%); 32818/250101 objects misplaced (13.122%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:10.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:10.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:10.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:10.990 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:11.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:11.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:11.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:11.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:11.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:11.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:11.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:11.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:11.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:11.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:39:11.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:11.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:11.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:11.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54826/250101 objects degraded (21.922%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54826/250101 objects degraded (21.922%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:11.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54826/250101 objects degraded (21.922%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:11.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:11.784 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:11.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:11.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:11.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:11.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:11.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:12.002 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:12.003 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:39:12.042 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:39:12.043 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:39:12.043 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:39:12.043 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:39:12.043 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:39:12.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:12.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:39:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:39:11] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-08-17T17:39:12.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:12.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:12.329 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:12.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:12.410 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:39:12.411 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:39:12.411 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:12.411 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:12.422 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:39:12.422 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:12.438 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:39:12.438 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:12.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:12.454 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:39:12.454 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:12.468 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:39:12.469 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:12.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:12.484 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:39:12.484 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:12.499 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:39:12.500 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:12.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:12.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:12.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:12.723 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:12 smithi152 ceph-mon[93456]: from='client.37398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:12.723 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:12 smithi152 ceph-mon[93456]: pgmap v1371: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 54826/249396 objects degraded (21.984%); 32784/249396 objects misplaced (13.145%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:39:12.723 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1408402648' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:12.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:12.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:12 smithi142 ceph-mon[106729]: from='client.37398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:12.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:12 smithi142 ceph-mon[106729]: pgmap v1371: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 54826/249396 objects degraded (21.984%); 32784/249396 objects misplaced (13.145%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:39:12.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1408402648' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:12.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:12 smithi142 ceph-mon[103614]: from='client.37398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:12.755 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:12 smithi142 ceph-mon[103614]: pgmap v1371: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 54826/249396 objects degraded (21.984%); 32784/249396 objects misplaced (13.145%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:39:12.755 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1408402648' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:12.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:39:12.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:12.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:12.953 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:12.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:13.075 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:13.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:13.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:13.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:13.222 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1370,"stamp":"2024-08-17T17:39:10.305018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5462366181,"num_objects":83367,"num_object_clones":0,"num_object_copies":250101,"num_objects_missing_on_primary":5853,"num_objects_missing":5853,"num_objects_degraded":54826,"num_objects_misplaced":32818,"num_objects_unfound":0,"num_objects_dirty":83367,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":153327,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26739,"num_bytes_recovered":1629495950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104875,"ondisk_log_size":104875,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16773328,"kb_used_data":11350228,"kb_used_omap":27,"kb_used_meta":5423012,"kb_avail":545697584,"statfs":{"total":575970213888,"available":558794326016,"internally_reserved":0,"allocated":11622633472,"data_stored":11610734614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5553165032},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,4,0,1,0,3,2,7,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-53084160,"num_objects":-810,"num_object_clones":0,"num_object_copies":-2430,"num_objects_missing_on_primary":-62,"num_objects_missing":-62,"num_objects_degraded":-107,"num_objects_misplaced":-328,"num_objects_unfound":0,"num_objects_dirty":-810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":810,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.628375"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":584,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681013+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:39:01.681013+0000","last_peered":"2024-08-17T17:39:01.681013+0000","last_clean":"2024-08-17T17:39:01.681013+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:39:01.681013+0000","last_undegraded":"2024-08-17T17:39:01.681013+0000","last_fullsized":"2024-08-17T17:39:01.681013+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":80,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427358+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:38:50.427358+0000","last_peered":"2024-08-17T17:38:50.427358+0000","last_clean":"2024-08-17T17:38:50.427358+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:38:50.427358+0000","last_undegraded":"2024-08-17T17:38:50.427358+0000","last_fullsized":"2024-08-17T17:38:50.427358+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":275,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.716428+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:39:01.716428+0000","last_peered":"2024-08-17T17:39:01.716428+0000","last_clean":"2024-08-17T17:39:01.716428+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:39:01.716428+0000","last_undegraded":"2024-08-17T17:39:01.716428+0000","last_fullsized":"2024-08-17T17:39:01.716428+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":282,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.682674+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:39:01.682674+0000","last_peered":"2024-08-17T17:39:01.682674+0000","last_clean":"2024-08-17T17:39:01.682674+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:39:01.682674+0000","last_undegraded":"2024-08-17T17:39:01.682674+0000","last_fullsized":"2024-08-17T17:39:01.682674+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":459,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681757+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:39:01.681757+0000","last_peered":"2024-08-17T17:39:01.681757+0000","last_clean":"2024-08-17T17:39:01.681757+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:39:01.681757+0000","last_undegraded":"2024-08-17T17:39:01.681757+0000","last_fullsized":"2024-08-17T17:39:01.681757+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":223,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427247+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:38:50.427247+0000","last_peered":"2024-08-17T17:38:50.427247+0000","last_clean":"2024-08-17T17:38:50.427247+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:38:50.427247+0000","last_undegraded":"2024-08-17T17:38:50.427247+0000","last_fullsized":"2024-08-17T17:38:50.427247+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":552,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680477+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:39:01.680477+0000","last_peered":"2024-08-17T17:39:01.680477+0000","last_clean":"2024-08-17T17:39:01.680477+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:39:01.680477+0000","last_undegraded":"2024-08-17T17:39:01.680477+0000","last_fullsized":"2024-08-17T17:39:01.680477+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":391,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400942+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:38:50.400942+0000","last_peered":"2024-08-17T17:38:50.400942+0000","last_clean":"2024-08-17T17:38:50.400942+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:38:50.400942+0000","last_undegraded":"2024-08-17T17:38:50.400942+0000","last_fullsized":"2024-08-17T17:38:50.400942+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"371'9560","reported_seq":12601,"reported_epoch":371,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:39:07.720530+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:39:07.720530+0000","last_peered":"2024-08-17T17:39:07.720530+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:39:07.720530+0000","last_undegraded":"2024-08-17T17:39:07.720530+0000","last_fullsized":"2024-08-17T17:39:07.720530+0000","mapping_epoch":337,"log_start":"325'2663","ondisk_log_start":"325'2663","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337117184,"num_objects":5144,"num_object_clones":0,"num_object_copies":15432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5144,"num_objects_unfound":0,"num_objects_dirty":5144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9560,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1479,"num_bytes_recovered":96927744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5144}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":76,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.502192+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:38:50.502192+0000","last_peered":"2024-08-17T17:38:50.502192+0000","last_clean":"2024-08-17T17:38:50.502192+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:38:50.502192+0000","last_undegraded":"2024-08-17T17:38:50.502192+0000","last_fullsized":"2024-08-17T17:38:50.502192+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":258,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.716136+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:39:01.716136+0000","last_peered":"2024-08-17T17:39:01.716136+0000","last_clean":"2024-08-17T17:39:01.716136+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:39:01.716136+0000","last_undegraded":"2024-08-17T17:39:01.716136+0000","last_fullsized":"2024-08-17T17:39:01.716136+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"371'9768","reported_seq":9837,"reported_epoch":371,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:07.005339+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:39:07.005339+0000","last_peered":"2024-08-17T17:39:07.005339+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:39:07.005339+0000","last_undegraded":"2024-08-17T17:39:07.005339+0000","last_fullsized":"2024-08-17T17:39:07.005339+0000","mapping_epoch":337,"log_start":"325'3182","ondisk_log_start":"325'3182","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347734016,"num_objects":5306,"num_object_clones":0,"num_object_copies":15918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5306,"num_objects_unfound":0,"num_objects_dirty":5306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9768,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5306}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":78,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.428367+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:38:50.428367+0000","last_peered":"2024-08-17T17:38:50.428367+0000","last_clean":"2024-08-17T17:38:50.428367+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:38:50.428367+0000","last_undegraded":"2024-08-17T17:38:50.428367+0000","last_fullsized":"2024-08-17T17:38:50.428367+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":342,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.428528+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:38:50.428528+0000","last_peered":"2024-08-17T17:38:50.428528+0000","last_clean":"2024-08-17T17:38:50.428528+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:38:50.428528+0000","last_undegraded":"2024-08-17T17:38:50.428528+0000","last_fullsized":"2024-08-17T17:38:50.428528+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"371'9493","reported_seq":9565,"reported_epoch":371,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:07.390213+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:39:07.390213+0000","last_peered":"2024-08-17T17:39:07.390213+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:39:07.390213+0000","last_undegraded":"2024-08-17T17:39:07.390213+0000","last_fullsized":"2024-08-17T17:39:07.390213+0000","mapping_epoch":337,"log_start":"325'2653","ondisk_log_start":"325'2653","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6840,"log_dups_size":0,"ondisk_log_size":6840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343474176,"num_objects":5241,"num_object_clones":0,"num_object_copies":15723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5241,"num_objects_unfound":0,"num_objects_dirty":5241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9493,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5241}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":75,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.716366+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:39:01.716366+0000","last_peered":"2024-08-17T17:39:01.716366+0000","last_clean":"2024-08-17T17:39:01.716366+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:39:01.716366+0000","last_undegraded":"2024-08-17T17:39:01.716366+0000","last_fullsized":"2024-08-17T17:39:01.716366+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":352,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400763+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:38:50.400763+0000","last_peered":"2024-08-17T17:38:50.400763+0000","last_clean":"2024-08-17T17:38:50.400763+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:38:50.400763+0000","last_undegraded":"2024-08-17T17:38:50.400763+0000","last_fullsized":"2024-08-17T17:38:50.400763+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"372'9679","reported_seq":9802,"reported_epoch":372,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:09.872837+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:39:09.872837+0000","last_peered":"2024-08-17T17:39:09.872837+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:39:09.872837+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"324'2613","ondisk_log_start":"324'2613","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7066,"log_dups_size":0,"ondisk_log_size":7066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345964544,"num_objects":5279,"num_object_clones":0,"num_object_copies":15837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9679,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1490},{"shards":"0,3,7","objects":5279}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":496,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681151+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:39:01.681151+0000","last_peered":"2024-08-17T17:39:01.681151+0000","last_clean":"2024-08-17T17:39:01.681151+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:39:01.681151+0000","last_undegraded":"2024-08-17T17:39:01.681151+0000","last_fullsized":"2024-08-17T17:39:01.681151+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":264,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.555497+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:39:08.555497+0000","last_peered":"2024-08-17T17:39:08.555497+0000","last_clean":"2024-08-17T17:39:08.555497+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:39:08.555497+0000","last_undegraded":"2024-08-17T17:39:08.555497+0000","last_fullsized":"2024-08-17T17:39:08.555497+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"371'9442","reported_seq":9502,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:07.051669+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:39:07.051669+0000","last_peered":"2024-08-17T17:39:07.051669+0000","last_clean":"2024-08-17T17:39:07.051669+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:39:07.051669+0000","last_undegraded":"2024-08-17T17:39:07.051669+0000","last_fullsized":"2024-08-17T17:39:07.051669+0000","mapping_epoch":319,"log_start":"325'2874","ondisk_log_start":"325'2874","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337641472,"num_objects":5152,"num_object_clones":0,"num_object_copies":15456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9442,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":585,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.683076+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:39:01.683076+0000","last_peered":"2024-08-17T17:39:01.683076+0000","last_clean":"2024-08-17T17:39:01.683076+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:39:01.683076+0000","last_undegraded":"2024-08-17T17:39:01.683076+0000","last_fullsized":"2024-08-17T17:39:01.683076+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427845+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:38:50.427845+0000","last_peered":"2024-08-17T17:38:50.427845+0000","last_clean":"2024-08-17T17:38:50.427845+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:38:50.427845+0000","last_undegraded":"2024-08-17T17:38:50.427845+0000","last_fullsized":"2024-08-17T17:38:50.427845+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"371'9585","reported_seq":9655,"reported_epoch":371,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:06.997558+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:39:06.997558+0000","last_peered":"2024-08-17T17:39:06.997558+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:39:06.997558+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'3035","ondisk_log_start":"325'3035","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350420992,"num_objects":5347,"num_object_clones":0,"num_object_copies":16041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9585,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5347},{"shards":"1,6","objects":-2019}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":326,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400909+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:38:50.400909+0000","last_peered":"2024-08-17T17:38:50.400909+0000","last_clean":"2024-08-17T17:38:50.400909+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:38:50.400909+0000","last_undegraded":"2024-08-17T17:38:50.400909+0000","last_fullsized":"2024-08-17T17:38:50.400909+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":352,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.428139+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:38:50.428139+0000","last_peered":"2024-08-17T17:38:50.428139+0000","last_clean":"2024-08-17T17:38:50.428139+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:38:50.428139+0000","last_undegraded":"2024-08-17T17:38:50.428139+0000","last_fullsized":"2024-08-17T17:38:50.428139+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"372'9728","reported_seq":9852,"reported_epoch":372,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:08.815897+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:39:08.815897+0000","last_peered":"2024-08-17T17:39:08.815897+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:39:08.815897+0000","last_undegraded":"2024-08-17T17:39:08.815897+0000","last_fullsized":"2024-08-17T17:39:08.815897+0000","mapping_epoch":337,"log_start":"326'4161","ondisk_log_start":"326'4161","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346161152,"num_objects":5282,"num_object_clones":0,"num_object_copies":15846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5282,"num_objects_unfound":0,"num_objects_dirty":5282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9728,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5282}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":363,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427381+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:38:50.427381+0000","last_peered":"2024-08-17T17:38:50.427381+0000","last_clean":"2024-08-17T17:38:50.427381+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:38:50.427381+0000","last_undegraded":"2024-08-17T17:38:50.427381+0000","last_fullsized":"2024-08-17T17:38:50.427381+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.428510+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:38:50.428510+0000","last_peered":"2024-08-17T17:38:50.428510+0000","last_clean":"2024-08-17T17:38:50.428510+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:38:50.428510+0000","last_undegraded":"2024-08-17T17:38:50.428510+0000","last_fullsized":"2024-08-17T17:38:50.428510+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"371'9542","reported_seq":12602,"reported_epoch":371,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:07.808983+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:39:07.808983+0000","last_peered":"2024-08-17T17:39:07.808983+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:39:07.808983+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:39:07.808983+0000","mapping_epoch":337,"log_start":"325'2641","ondisk_log_start":"325'2641","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338296832,"num_objects":5162,"num_object_clones":0,"num_object_copies":15486,"num_objects_missing_on_primary":5853,"num_objects_missing":5853,"num_objects_degraded":5162,"num_objects_misplaced":5853,"num_objects_unfound":0,"num_objects_dirty":5162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9542,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":15},{"shards":"1,2,7","objects":5147}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":555,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.684253+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:39:01.684253+0000","last_peered":"2024-08-17T17:39:01.684253+0000","last_clean":"2024-08-17T17:39:01.684253+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:39:01.684253+0000","last_undegraded":"2024-08-17T17:39:01.684253+0000","last_fullsized":"2024-08-17T17:39:01.684253+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":492,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680579+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:39:01.680579+0000","last_peered":"2024-08-17T17:39:01.680579+0000","last_clean":"2024-08-17T17:39:01.680579+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:39:01.680579+0000","last_undegraded":"2024-08-17T17:39:01.680579+0000","last_fullsized":"2024-08-17T17:39:01.680579+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"372'9654","reported_seq":12679,"reported_epoch":372,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:08.553684+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:39:08.553684+0000","last_peered":"2024-08-17T17:39:08.553684+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:39:08.553684+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:39:08.553684+0000","mapping_epoch":337,"log_start":"326'4044","ondisk_log_start":"326'4044","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343474200,"num_objects":5242,"num_object_clones":0,"num_object_copies":15726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5242,"num_objects_misplaced":5992,"num_objects_unfound":0,"num_objects_dirty":5242,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9654,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1456,"num_bytes_recovered":95354880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5242}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":687,"reported_epoch":372,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:08.553621+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:39:08.553621+0000","last_peered":"2024-08-17T17:39:08.553621+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:39:08.553621+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":301,"reported_epoch":371,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:39:01.715847+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:39:01.715847+0000","last_peered":"2024-08-17T17:39:01.715847+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:39:01.715847+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:39:01.715847+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":534,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.682591+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:39:01.682591+0000","last_peered":"2024-08-17T17:39:01.682591+0000","last_clean":"2024-08-17T17:39:01.682591+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:39:01.682591+0000","last_undegraded":"2024-08-17T17:39:01.682591+0000","last_fullsized":"2024-08-17T17:39:01.682591+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":335,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401386+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:38:50.401386+0000","last_peered":"2024-08-17T17:38:50.401386+0000","last_clean":"2024-08-17T17:38:50.401386+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:38:50.401386+0000","last_undegraded":"2024-08-17T17:38:50.401386+0000","last_fullsized":"2024-08-17T17:38:50.401386+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"372'9557","reported_seq":9683,"reported_epoch":372,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:09.897835+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:39:09.897835+0000","last_peered":"2024-08-17T17:39:09.897835+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:39:09.897835+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"324'2521","ondisk_log_start":"324'2521","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339935232,"num_objects":5187,"num_object_clones":0,"num_object_copies":15561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9557,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1460},{"shards":"0,4,7","objects":5187}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":352,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463036+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:39:08.463036+0000","last_peered":"2024-08-17T17:39:08.463036+0000","last_clean":"2024-08-17T17:39:08.463036+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:39:08.463036+0000","last_undegraded":"2024-08-17T17:39:08.463036+0000","last_fullsized":"2024-08-17T17:39:08.463036+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":307,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427131+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:38:50.427131+0000","last_peered":"2024-08-17T17:38:50.427131+0000","last_clean":"2024-08-17T17:38:50.427131+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:38:50.427131+0000","last_undegraded":"2024-08-17T17:38:50.427131+0000","last_fullsized":"2024-08-17T17:38:50.427131+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":278,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.554850+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:39:08.554850+0000","last_peered":"2024-08-17T17:39:08.554850+0000","last_clean":"2024-08-17T17:39:08.554850+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:39:08.554850+0000","last_undegraded":"2024-08-17T17:39:08.554850+0000","last_fullsized":"2024-08-17T17:39:08.554850+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"371'9548","reported_seq":9618,"reported_epoch":371,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:06.944680+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:39:06.944680+0000","last_peered":"2024-08-17T17:39:06.944680+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:39:06.944680+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'3017","ondisk_log_start":"325'3017","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343932928,"num_objects":5248,"num_object_clones":0,"num_object_copies":15744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9548,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5248},{"shards":"5,6","objects":-2048}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":442,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680703+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:39:01.680703+0000","last_peered":"2024-08-17T17:39:01.680703+0000","last_clean":"2024-08-17T17:39:01.680703+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:39:01.680703+0000","last_undegraded":"2024-08-17T17:39:01.680703+0000","last_fullsized":"2024-08-17T17:39:01.680703+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":107,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.400706+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:38:50.400706+0000","last_peered":"2024-08-17T17:38:50.400706+0000","last_clean":"2024-08-17T17:38:50.400706+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:38:50.400706+0000","last_undegraded":"2024-08-17T17:38:50.400706+0000","last_fullsized":"2024-08-17T17:38:50.400706+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":629,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464576+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:39:08.464576+0000","last_peered":"2024-08-17T17:39:08.464576+0000","last_clean":"2024-08-17T17:39:08.464576+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:39:08.464576+0000","last_undegraded":"2024-08-17T17:39:08.464576+0000","last_fullsized":"2024-08-17T17:39:08.464576+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"372'9472","reported_seq":10322,"reported_epoch":372,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:09.596173+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:39:09.596173+0000","last_peered":"2024-08-17T17:39:09.596173+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:39:09.596173+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2416","ondisk_log_start":"324'2416","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339083264,"num_objects":5174,"num_object_clones":0,"num_object_copies":15522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9472,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5174},{"shards":"0,5","objects":-1480}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":420,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463110+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:39:08.463110+0000","last_peered":"2024-08-17T17:39:08.463110+0000","last_clean":"2024-08-17T17:39:08.463110+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:39:08.463110+0000","last_undegraded":"2024-08-17T17:39:08.463110+0000","last_fullsized":"2024-08-17T17:39:08.463110+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":346,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427704+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:38:50.427704+0000","last_peered":"2024-08-17T17:38:50.427704+0000","last_clean":"2024-08-17T17:38:50.427704+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:38:50.427704+0000","last_undegraded":"2024-08-17T17:38:50.427704+0000","last_fullsized":"2024-08-17T17:38:50.427704+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":213,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680946+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:39:01.680946+0000","last_peered":"2024-08-17T17:39:01.680946+0000","last_clean":"2024-08-17T17:39:01.680946+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:39:01.680946+0000","last_undegraded":"2024-08-17T17:39:01.680946+0000","last_fullsized":"2024-08-17T17:39:01.680946+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"371'9611","reported_seq":24578,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:05.131958+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:39:05.131958+0000","last_peered":"2024-08-17T17:39:05.131958+0000","last_clean":"2024-08-17T17:39:05.131958+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:39:05.131958+0000","last_undegraded":"2024-08-17T17:39:05.131958+0000","last_fullsized":"2024-08-17T17:39:05.131958+0000","mapping_epoch":355,"log_start":"325'3330","ondisk_log_start":"325'3330","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6281,"log_dups_size":0,"ondisk_log_size":6281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342425600,"num_objects":5225,"num_object_clones":0,"num_object_copies":15675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9611,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":286,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.716291+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:39:01.716291+0000","last_peered":"2024-08-17T17:39:01.716291+0000","last_clean":"2024-08-17T17:39:01.716291+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:39:01.716291+0000","last_undegraded":"2024-08-17T17:39:01.716291+0000","last_fullsized":"2024-08-17T17:39:01.716291+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":82,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680866+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:39:01.680866+0000","last_peered":"2024-08-17T17:39:01.680866+0000","last_clean":"2024-08-17T17:39:01.680866+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:39:01.680866+0000","last_undegraded":"2024-08-17T17:39:01.680866+0000","last_fullsized":"2024-08-17T17:39:01.680866+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":580,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463427+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:39:08.463427+0000","last_peered":"2024-08-17T17:39:08.463427+0000","last_clean":"2024-08-17T17:39:08.463427+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:39:08.463427+0000","last_undegraded":"2024-08-17T17:39:08.463427+0000","last_fullsized":"2024-08-17T17:39:08.463427+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"371'9321","reported_seq":23856,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:05.255947+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:39:05.255947+0000","last_peered":"2024-08-17T17:39:05.255947+0000","last_clean":"2024-08-17T17:39:05.255947+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:39:05.255947+0000","last_undegraded":"2024-08-17T17:39:05.255947+0000","last_fullsized":"2024-08-17T17:39:05.255947+0000","mapping_epoch":345,"log_start":"325'2972","ondisk_log_start":"325'2972","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6349,"log_dups_size":0,"ondisk_log_size":6349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333119488,"num_objects":5083,"num_object_clones":0,"num_object_copies":15249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9321,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":282,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.716639+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:39:01.716639+0000","last_peered":"2024-08-17T17:39:01.716639+0000","last_clean":"2024-08-17T17:39:01.716639+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:39:01.716639+0000","last_undegraded":"2024-08-17T17:39:01.716639+0000","last_fullsized":"2024-08-17T17:39:01.716639+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":335,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.554183+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:39:08.554183+0000","last_peered":"2024-08-17T17:39:08.554183+0000","last_clean":"2024-08-17T17:39:08.554183+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:39:08.554183+0000","last_undegraded":"2024-08-17T17:39:08.554183+0000","last_fullsized":"2024-08-17T17:39:08.554183+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":424,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.715755+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:39:01.715755+0000","last_peered":"2024-08-17T17:39:01.715755+0000","last_clean":"2024-08-17T17:39:01.715755+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:39:01.715755+0000","last_undegraded":"2024-08-17T17:39:01.715755+0000","last_fullsized":"2024-08-17T17:39:01.715755+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"371'9589","reported_seq":9732,"reported_epoch":371,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:05.243746+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:39:05.243746+0000","last_peered":"2024-08-17T17:39:05.243746+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:39:05.243746+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'3287","ondisk_log_start":"325'3287","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6302,"log_dups_size":0,"ondisk_log_size":6302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340459520,"num_objects":5195,"num_object_clones":0,"num_object_copies":15585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9589,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5195},{"shards":"0,3","objects":-2053}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":608,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680674+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:39:01.680674+0000","last_peered":"2024-08-17T17:39:01.680674+0000","last_clean":"2024-08-17T17:39:01.680674+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:39:01.680674+0000","last_undegraded":"2024-08-17T17:39:01.680674+0000","last_fullsized":"2024-08-17T17:39:01.680674+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":578,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464310+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:39:08.464310+0000","last_peered":"2024-08-17T17:39:08.464310+0000","last_clean":"2024-08-17T17:39:08.464310+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:39:08.464310+0000","last_undegraded":"2024-08-17T17:39:08.464310+0000","last_fullsized":"2024-08-17T17:39:08.464310+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":594,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.684030+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:39:01.684030+0000","last_peered":"2024-08-17T17:39:01.684030+0000","last_clean":"2024-08-17T17:39:01.684030+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:39:01.684030+0000","last_undegraded":"2024-08-17T17:39:01.684030+0000","last_fullsized":"2024-08-17T17:39:01.684030+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"371'9500","reported_seq":24168,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:06.207784+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:39:06.207784+0000","last_peered":"2024-08-17T17:39:06.207784+0000","last_clean":"2024-08-17T17:39:06.207784+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:39:06.207784+0000","last_undegraded":"2024-08-17T17:39:06.207784+0000","last_fullsized":"2024-08-17T17:39:06.207784+0000","mapping_epoch":347,"log_start":"325'2972","ondisk_log_start":"325'2972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332529664,"num_objects":5074,"num_object_clones":0,"num_object_copies":15222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9500,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":268,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.554418+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:39:08.554418+0000","last_peered":"2024-08-17T17:39:08.554418+0000","last_clean":"2024-08-17T17:39:08.554418+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:39:08.554418+0000","last_undegraded":"2024-08-17T17:39:08.554418+0000","last_fullsized":"2024-08-17T17:39:08.554418+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":375,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401305+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:38:50.401305+0000","last_peered":"2024-08-17T17:38:50.401305+0000","last_clean":"2024-08-17T17:38:50.401305+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:38:50.401305+0000","last_undegraded":"2024-08-17T17:38:50.401305+0000","last_fullsized":"2024-08-17T17:38:50.401305+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427313+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:38:50.427313+0000","last_peered":"2024-08-17T17:38:50.427313+0000","last_clean":"2024-08-17T17:38:50.427313+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:38:50.427313+0000","last_undegraded":"2024-08-17T17:38:50.427313+0000","last_fullsized":"2024-08-17T17:38:50.427313+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427589+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:38:50.427589+0000","last_peered":"2024-08-17T17:38:50.427589+0000","last_clean":"2024-08-17T17:38:50.427589+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:38:50.427589+0000","last_undegraded":"2024-08-17T17:38:50.427589+0000","last_fullsized":"2024-08-17T17:38:50.427589+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":368,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.428020+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:38:50.428020+0000","last_peered":"2024-08-17T17:38:50.428020+0000","last_clean":"2024-08-17T17:38:50.428020+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:38:50.428020+0000","last_undegraded":"2024-08-17T17:38:50.428020+0000","last_fullsized":"2024-08-17T17:38:50.428020+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":585,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463207+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:39:08.463207+0000","last_peered":"2024-08-17T17:39:08.463207+0000","last_clean":"2024-08-17T17:39:08.463207+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:39:08.463207+0000","last_undegraded":"2024-08-17T17:39:08.463207+0000","last_fullsized":"2024-08-17T17:39:08.463207+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":123,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.682207+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:39:01.682207+0000","last_peered":"2024-08-17T17:39:01.682207+0000","last_clean":"2024-08-17T17:39:01.682207+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:39:01.682207+0000","last_undegraded":"2024-08-17T17:39:01.682207+0000","last_fullsized":"2024-08-17T17:39:01.682207+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":276,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.716347+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:39:01.716347+0000","last_peered":"2024-08-17T17:39:01.716347+0000","last_clean":"2024-08-17T17:39:01.716347+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:39:01.716347+0000","last_undegraded":"2024-08-17T17:39:01.716347+0000","last_fullsized":"2024-08-17T17:39:01.716347+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":541,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680411+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:39:01.680411+0000","last_peered":"2024-08-17T17:39:01.680411+0000","last_clean":"2024-08-17T17:39:01.680411+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:39:01.680411+0000","last_undegraded":"2024-08-17T17:39:01.680411+0000","last_fullsized":"2024-08-17T17:39:01.680411+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":420,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.717631+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:39:01.717631+0000","last_peered":"2024-08-17T17:39:01.717631+0000","last_clean":"2024-08-17T17:39:01.717631+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:39:01.717631+0000","last_undegraded":"2024-08-17T17:39:01.717631+0000","last_fullsized":"2024-08-17T17:39:01.717631+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":535,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681283+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:39:01.681283+0000","last_peered":"2024-08-17T17:39:01.681283+0000","last_clean":"2024-08-17T17:39:01.681283+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:39:01.681283+0000","last_undegraded":"2024-08-17T17:39:01.681283+0000","last_fullsized":"2024-08-17T17:39:01.681283+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":342,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.401260+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:38:50.401260+0000","last_peered":"2024-08-17T17:38:50.401260+0000","last_clean":"2024-08-17T17:38:50.401260+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:38:50.401260+0000","last_undegraded":"2024-08-17T17:38:50.401260+0000","last_fullsized":"2024-08-17T17:38:50.401260+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":566,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.684144+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:39:01.684144+0000","last_peered":"2024-08-17T17:39:01.684144+0000","last_clean":"2024-08-17T17:39:01.684144+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:39:01.684144+0000","last_undegraded":"2024-08-17T17:39:01.684144+0000","last_fullsized":"2024-08-17T17:39:01.684144+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":282,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681688+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:39:01.681688+0000","last_peered":"2024-08-17T17:39:01.681688+0000","last_clean":"2024-08-17T17:39:01.681688+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:39:01.681688+0000","last_undegraded":"2024-08-17T17:39:01.681688+0000","last_fullsized":"2024-08-17T17:39:01.681688+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":576,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463280+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:39:08.463280+0000","last_peered":"2024-08-17T17:39:08.463280+0000","last_clean":"2024-08-17T17:39:08.463280+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:39:08.463280+0000","last_undegraded":"2024-08-17T17:39:08.463280+0000","last_fullsized":"2024-08-17T17:39:08.463280+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":534,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.682037+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:39:01.682037+0000","last_peered":"2024-08-17T17:39:01.682037+0000","last_clean":"2024-08-17T17:39:01.682037+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:39:01.682037+0000","last_undegraded":"2024-08-17T17:39:01.682037+0000","last_fullsized":"2024-08-17T17:39:01.682037+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":354,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427695+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:38:50.427695+0000","last_peered":"2024-08-17T17:38:50.427695+0000","last_clean":"2024-08-17T17:38:50.427695+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:38:50.427695+0000","last_undegraded":"2024-08-17T17:38:50.427695+0000","last_fullsized":"2024-08-17T17:38:50.427695+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":223,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681082+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:39:01.681082+0000","last_peered":"2024-08-17T17:39:01.681082+0000","last_clean":"2024-08-17T17:39:01.681082+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:39:01.681082+0000","last_undegraded":"2024-08-17T17:39:01.681082+0000","last_fullsized":"2024-08-17T17:39:01.681082+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":281,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.554898+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:39:08.554898+0000","last_peered":"2024-08-17T17:39:08.554898+0000","last_clean":"2024-08-17T17:39:08.554898+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:39:08.554898+0000","last_undegraded":"2024-08-17T17:39:08.554898+0000","last_fullsized":"2024-08-17T17:39:08.554898+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":362,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427529+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:38:50.427529+0000","last_peered":"2024-08-17T17:38:50.427529+0000","last_clean":"2024-08-17T17:38:50.427529+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:38:50.427529+0000","last_undegraded":"2024-08-17T17:38:50.427529+0000","last_fullsized":"2024-08-17T17:38:50.427529+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":225,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427935+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:38:50.427935+0000","last_peered":"2024-08-17T17:38:50.427935+0000","last_clean":"2024-08-17T17:38:50.427935+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:38:50.427935+0000","last_undegraded":"2024-08-17T17:38:50.427935+0000","last_fullsized":"2024-08-17T17:38:50.427935+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":362,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427563+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:38:50.427563+0000","last_peered":"2024-08-17T17:38:50.427563+0000","last_clean":"2024-08-17T17:38:50.427563+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:38:50.427563+0000","last_undegraded":"2024-08-17T17:38:50.427563+0000","last_fullsized":"2024-08-17T17:38:50.427563+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":225,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.428112+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:38:50.428112+0000","last_peered":"2024-08-17T17:38:50.428112+0000","last_clean":"2024-08-17T17:38:50.428112+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:38:50.428112+0000","last_undegraded":"2024-08-17T17:38:50.428112+0000","last_fullsized":"2024-08-17T17:38:50.428112+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":586,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.680777+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:39:01.680777+0000","last_peered":"2024-08-17T17:39:01.680777+0000","last_clean":"2024-08-17T17:39:01.680777+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:39:01.680777+0000","last_undegraded":"2024-08-17T17:39:01.680777+0000","last_fullsized":"2024-08-17T17:39:01.680777+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":560,"reported_epoch":371,"state":"active+clean","last_fresh":"2024-08-17T17:39:01.681826+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:39:01.681826+0000","last_peered":"2024-08-17T17:39:01.681826+0000","last_clean":"2024-08-17T17:39:01.681826+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:39:01.681826+0000","last_undegraded":"2024-08-17T17:39:01.681826+0000","last_fullsized":"2024-08-17T17:39:01.681826+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":325,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-08-17T17:38:50.427468+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:38:50.427468+0000","last_peered":"2024-08-17T17:38:50.427468+0000","last_clean":"2024-08-17T17:38:50.427468+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:38:50.427468+0000","last_undegraded":"2024-08-17T17:38:50.427468+0000","last_fullsized":"2024-08-17T17:38:50.427468+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":284,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.554692+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:39:08.554692+0000","last_peered":"2024-08-17T17:39:08.554692+0000","last_clean":"2024-08-17T17:39:08.554692+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:39:08.554692+0000","last_undegraded":"2024-08-17T17:39:08.554692+0000","last_fullsized":"2024-08-17T17:39:08.554692+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5461770264,"num_objects":83341,"num_object_clones":0,"num_object_copies":250023,"num_objects_missing_on_primary":5853,"num_objects_missing":5853,"num_objects_degraded":54823,"num_objects_misplaced":32818,"num_objects_unfound":0,"num_objects_dirty":83341,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153049,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26698,"num_bytes_recovered":1627127808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17662881792,"data_stored":17662869576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104668,"ondisk_log_size":104668,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170072,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463442,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3223112,"kb_used_data":2333568,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90522040,"statfs":{"total":95995035648,"available":92694568960,"internally_reserved":0,"allocated":2389573632,"data_stored":2387582190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492117,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1057472,"kb_used_data":670452,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92687680,"statfs":{"total":95995035648,"available":94912184320,"internally_reserved":0,"allocated":686542848,"data_stored":684565913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986708,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2522108,"kb_used_data":1328628,"kb_used_omap":1,"kb_used_meta":1193470,"kb_avail":91223044,"statfs":{"total":95995035648,"available":93412397056,"internally_reserved":0,"allocated":1360515072,"data_stored":1358534570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1222113624},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,4,0,0,0,3,2,7,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225535,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3593836,"kb_used_data":2669796,"kb_used_omap":4,"kb_used_meta":924027,"kb_avail":90151316,"statfs":{"total":95995035648,"available":92314947584,"internally_reserved":0,"allocated":2733871104,"data_stored":2731902260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":946204096},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075861,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533754,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3371140,"kb_used_data":2337464,"kb_used_omap":5,"kb_used_meta":1033658,"kb_avail":90374012,"statfs":{"total":95995035648,"available":92542988288,"internally_reserved":0,"allocated":2393563136,"data_stored":2391581177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1058466032},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089731,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3005660,"kb_used_data":2010320,"kb_used_omap":5,"kb_used_meta":995322,"kb_avail":90739492,"statfs":{"total":95995035648,"available":92917239808,"internally_reserved":0,"allocated":2058567680,"data_stored":2056568504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1019209787},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2050097152,"data_stored":2050097152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2387611648,"data_stored":2387607576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3477671936,"data_stored":3477667864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2726887424,"data_stored":2726887424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1355218944,"data_stored":1355218944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":681246720,"data_stored":681246720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2382036992,"data_stored":2382036992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2602110976,"data_stored":2602106904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:13.226 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:39:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:13.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:13.329 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:13.350 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:13.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:13.454 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:13.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:13.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:13.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:13.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:13.724 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:13.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:13.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:13.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:13.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:13.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:13.964 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:13.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:14.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:14.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:14.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:14.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:14.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:14.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:14.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:14.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:14.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:14.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:14.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:14.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:14.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:14.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:14.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:14.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:14.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:14.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:14.893 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:14.893 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:14.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:14.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:14.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:15.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:15.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:15.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:15.197 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:15.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:15.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:15.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:15.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:15.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:15.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:15.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:15 smithi152 ceph-mon[93456]: pgmap v1372: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 54772/248982 objects degraded (21.998%); 32649/248982 objects misplaced (13.113%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:39:15.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:15.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:39:15.595 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:15.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:15 smithi142 ceph-mon[106729]: pgmap v1372: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 54772/248982 objects degraded (21.998%); 32649/248982 objects misplaced (13.113%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:39:15.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:15 smithi142 ceph-mon[103614]: pgmap v1372: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 54772/248982 objects degraded (21.998%); 32649/248982 objects misplaced (13.113%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:39:15.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:15.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:15.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:15.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:15.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:15.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:15.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:16.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:16.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:16.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:16.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:16.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:16.342 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:16.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:16.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:16.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:16.592 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:16.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54772/248982 objects degraded (21.998%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54772/248982 objects degraded (21.998%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:16.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:16.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:16.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54772/248982 objects degraded (21.998%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:16.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:16.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:16.964 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:17.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:17.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:17.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:17.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:17.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:17.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:17.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:17.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:17.482 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:17.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:17.590 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:17.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:17 smithi142 ceph-mon[106729]: pgmap v1373: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 54772/248421 objects degraded (22.048%); 32649/248421 objects misplaced (13.143%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:39:17.597 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:17 smithi142 ceph-mon[103614]: pgmap v1373: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 54772/248421 objects degraded (22.048%); 32649/248421 objects misplaced (13.143%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:39:17.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:17.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:17.749 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:17.749 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 54772/248421 objects degraded (22.048%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2038,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":372,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":82807,"data_bytes":5425666021,"bytes_used":17085157376,"bytes_avail":558885056512,"bytes_total":575970213888,"degraded_objects":54772,"degraded_total":248421,"degraded_ratio":0.22048055518655829,"misplaced_objects":32649,"misplaced_total":248421,"misplaced_ratio":0.13142608716654389,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":936866,"recovering_keys_per_sec":0,"num_objects_recovered":143,"num_bytes_recovered":9371648,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":90},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:39:17.816 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:17 smithi152 ceph-mon[93456]: pgmap v1373: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 54772/248421 objects degraded (22.048%); 32649/248421 objects misplaced (13.143%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:39:17.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:17.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:17.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:18.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:18.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:18.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:18.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:18.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:18.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:18.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:18.370 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:39:18.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:18.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:18.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:18.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:18.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1792807664' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:18.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1863376819' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:18.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1792807664' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:18.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1863376819' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:18.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:18.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:18.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:18.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1792807664' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:18.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1863376819' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:18.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:18.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:18.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:18.978 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:39:18.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:19.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:19.082 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:19.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:19.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:19.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:19.331 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:19.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:19.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:19.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:19.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[106729]: pgmap v1374: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54741/247629 objects degraded (22.106%); 32511/247629 objects misplaced (13.129%); 1011 KiB/s, 15 objects/s recovering 2024-08-17T17:39:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1792807664' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[106729]: osdmap e373: 8 total, 8 up, 6 in 2024-08-17T17:39:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[103614]: pgmap v1374: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54741/247629 objects degraded (22.106%); 32511/247629 objects misplaced (13.129%); 1011 KiB/s, 15 objects/s recovering 2024-08-17T17:39:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1792807664' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:19 smithi142 ceph-mon[103614]: osdmap e373: 8 total, 8 up, 6 in 2024-08-17T17:39:19.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:19.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:19.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:19 smithi152 ceph-mon[93456]: pgmap v1374: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54741/247629 objects degraded (22.106%); 32511/247629 objects misplaced (13.129%); 1011 KiB/s, 15 objects/s recovering 2024-08-17T17:39:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:19 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1792807664' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:19.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:19 smithi152 ceph-mon[93456]: osdmap e373: 8 total, 8 up, 6 in 2024-08-17T17:39:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:19.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:19.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:19.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:20.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:20.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:20.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:20.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:20.189 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:20.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:20.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:20.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:20.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:20.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:20.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:39:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:39:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:20.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:20.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:20.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:20.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:20.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:20.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:20.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:20.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:20.865 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:39:20.866 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:20.886 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:39:20.886 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:20.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:39:20.913 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:20.925 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:20.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:20.933 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:39:20.933 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:20.951 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:39:20.951 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:20.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:20.969 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:39:20.969 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:20.985 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:39:20.985 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:21.000 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:39:21.001 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:21.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:21.074 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:21.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:21.198 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:21.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:21.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:21.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:21.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:21.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:21.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-mon[103614]: pgmap v1376: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 54703/247059 objects degraded (22.142%); 32444/247059 objects misplaced (13.132%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54703/247059 objects degraded (22.142%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1817790364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-mon[106729]: pgmap v1376: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 54703/247059 objects degraded (22.142%); 32444/247059 objects misplaced (13.132%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54703/247059 objects degraded (22.142%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:21.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1817790364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:21.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:21.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:21.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:21.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:21 smithi152 ceph-mon[93456]: pgmap v1376: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 54703/247059 objects degraded (22.142%); 32444/247059 objects misplaced (13.132%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54703/247059 objects degraded (22.142%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1817790364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:21.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:21.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:21.950 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:39:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:39:21] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-08-17T17:39:22.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:22.076 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:22.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:22.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:39:22.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:22.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:22.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:22.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:22.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:22.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:22.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:22.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.578 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:22.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:22.679 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:22.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:22.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:22.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:22.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:22.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:23.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:23.076 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:23.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:23.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:23.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:23.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:23.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:23.446 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:23.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:23.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:23.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:23 smithi142 ceph-mon[103614]: pgmap v1377: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54703/246189 objects degraded (22.220%); 32401/246189 objects misplaced (13.161%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:23.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:23 smithi142 ceph-mon[106729]: pgmap v1377: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54703/246189 objects degraded (22.220%); 32401/246189 objects misplaced (13.161%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:23.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.697 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:23.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:23.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:23 smithi152 ceph-mon[93456]: pgmap v1377: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 54703/246189 objects degraded (22.220%); 32401/246189 objects misplaced (13.161%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:23.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:23.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:23.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:23.948 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:24.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:24.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:24.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:24.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:24.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:24.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:24.419 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:24.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:24.568 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:24.670 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:24.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:24.792 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:154869 12.3 12:c7a0ea05:::benchmark_data_smithi142_261410_object36672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:24.786129+0000", 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0057672449999999998, 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0057874299999999997, 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:24.793 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 154869 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786129+0000", 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.794 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786129+0000", 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786131+0000", 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.81e-06 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.795 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786139+0000", 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0290000000000005e-06 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786156+0000", 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6532e-05 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786167+0000", 2024-08-17T17:39:24.796 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1061e-05 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786245+0000", 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7844999999999998e-05 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786368+0000", 2024-08-17T17:39:24.797 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012330300000000001 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.786474+0000", 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000106148 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.789479+0000", 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030052859999999998 2024-08-17T17:39:24.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.789674+0000", 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019438300000000001 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 1 2024-08-17T17:39:24.799 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:39:24.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:24.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:24.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:24.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:25.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:25.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:25.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:25.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:25.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:25.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:25.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:25.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:25.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:25.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:25.674 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:25.695 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:25.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:25.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:25 smithi142 ceph-mon[106729]: pgmap v1378: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 54635/245628 objects degraded (22.243%); 32216/245628 objects misplaced (13.116%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:25.744 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:25 smithi142 ceph-mon[103614]: pgmap v1378: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 54635/245628 objects degraded (22.243%); 32216/245628 objects misplaced (13.116%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:25.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:25.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:25.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:25 smithi152 ceph-mon[93456]: pgmap v1378: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 54635/245628 objects degraded (22.243%); 32216/245628 objects misplaced (13.116%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:25.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:25.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:26.035 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:26.036 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:39:26.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:26.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:26.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:26.191 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:26.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:26.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:26.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:26.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:26.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:39:26.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:26.562 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:26.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:26.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:26.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:26.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:26.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:26.810 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54635/245628 objects degraded (22.243%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:26.839 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54635/245628 objects degraded (22.243%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54635/245628 objects degraded (22.243%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:26.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:26.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:27.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:27.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:27.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:27.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:27.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:27.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:27.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:27.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:27.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:27.558 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[103614]: from='client.47291 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[103614]: pgmap v1379: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 54635/244992 objects degraded (22.301%); 32216/244992 objects misplaced (13.150%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[106729]: from='client.47291 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[106729]: pgmap v1379: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 54635/244992 objects degraded (22.301%); 32216/244992 objects misplaced (13.150%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:27.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:27.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:27.684 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:27.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:27.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:27.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:27 smithi152 ceph-mon[93456]: from='client.47291 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:27 smithi152 ceph-mon[93456]: pgmap v1379: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 54635/244992 objects degraded (22.301%); 32216/244992 objects misplaced (13.150%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:27.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:27.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:28.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:28.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:28.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:28.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:28.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:28.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:28.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:28.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:28.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:28.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:28.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:28.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:28.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:28.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:28.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:28.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:28.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:28.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:29.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:29.048 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:29.150 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:29.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:29.204 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:29.205 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:29.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:29.296 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:29.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:29.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:29.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:29.444 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:39:29.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:29.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:29.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:29 smithi142 ceph-mon[106729]: pgmap v1380: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54603/244158 objects degraded (22.364%); 32084/244158 objects misplaced (13.141%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:29.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/580293350' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:29.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:29.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:29 smithi142 ceph-mon[103614]: pgmap v1380: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54603/244158 objects degraded (22.364%); 32084/244158 objects misplaced (13.141%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:29.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/580293350' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:29.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:29.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:29.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:29.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:29.788 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1378,"stamp":"2024-08-17T17:39:24.309151+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5364652005,"num_objects":81876,"num_object_clones":0,"num_object_copies":245628,"num_objects_missing_on_primary":5761,"num_objects_missing":5761,"num_objects_degraded":54635,"num_objects_misplaced":32216,"num_objects_unfound":0,"num_objects_dirty":81876,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":154818,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27025,"num_bytes_recovered":1648239246,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104766,"ondisk_log_size":104766,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16561052,"kb_used_data":11149028,"kb_used_omap":27,"kb_used_meta":5411940,"kb_avail":545909860,"statfs":{"total":575970213888,"available":559011696640,"internally_reserved":0,"allocated":11416604672,"data_stored":11404635677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5541827304},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,4,1,2,1,3,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-73269248,"num_objects":-1118,"num_object_clones":0,"num_object_copies":-3354,"num_objects_missing_on_primary":-63,"num_objects_missing":-63,"num_objects_degraded":-137,"num_objects_misplaced":-433,"num_objects_unfound":0,"num_objects_dirty":-1118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1118,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":13959168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003060"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":588,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467823+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:39:18.467823+0000","last_peered":"2024-08-17T17:39:18.467823+0000","last_clean":"2024-08-17T17:39:18.467823+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:39:18.467823+0000","last_undegraded":"2024-08-17T17:39:18.467823+0000","last_fullsized":"2024-08-17T17:39:18.467823+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":82,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464441+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:39:08.464441+0000","last_peered":"2024-08-17T17:39:08.464441+0000","last_clean":"2024-08-17T17:39:08.464441+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:39:08.464441+0000","last_undegraded":"2024-08-17T17:39:08.464441+0000","last_fullsized":"2024-08-17T17:39:08.464441+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.493554+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:39:08.493554+0000","last_peered":"2024-08-17T17:39:08.493554+0000","last_clean":"2024-08-17T17:39:08.493554+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:39:08.493554+0000","last_undegraded":"2024-08-17T17:39:08.493554+0000","last_fullsized":"2024-08-17T17:39:08.493554+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":286,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467857+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:39:18.467857+0000","last_peered":"2024-08-17T17:39:18.467857+0000","last_clean":"2024-08-17T17:39:18.467857+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:39:18.467857+0000","last_undegraded":"2024-08-17T17:39:18.467857+0000","last_fullsized":"2024-08-17T17:39:18.467857+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":463,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.468072+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:39:18.468072+0000","last_peered":"2024-08-17T17:39:18.468072+0000","last_clean":"2024-08-17T17:39:18.468072+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:39:18.468072+0000","last_undegraded":"2024-08-17T17:39:18.468072+0000","last_fullsized":"2024-08-17T17:39:18.468072+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463564+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:39:08.463564+0000","last_peered":"2024-08-17T17:39:08.463564+0000","last_clean":"2024-08-17T17:39:08.463564+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:39:08.463564+0000","last_undegraded":"2024-08-17T17:39:08.463564+0000","last_fullsized":"2024-08-17T17:39:08.463564+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":557,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467974+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:39:18.467974+0000","last_peered":"2024-08-17T17:39:18.467974+0000","last_clean":"2024-08-17T17:39:18.467974+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:39:18.467974+0000","last_undegraded":"2024-08-17T17:39:18.467974+0000","last_fullsized":"2024-08-17T17:39:18.467974+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":393,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.480759+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:39:08.480759+0000","last_peered":"2024-08-17T17:39:08.480759+0000","last_clean":"2024-08-17T17:39:08.480759+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:39:08.480759+0000","last_undegraded":"2024-08-17T17:39:08.480759+0000","last_fullsized":"2024-08-17T17:39:08.480759+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"373'9655","reported_seq":12888,"reported_epoch":373,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:39:23.044783+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:39:23.044783+0000","last_peered":"2024-08-17T17:39:23.044783+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:39:23.044783+0000","last_undegraded":"2024-08-17T17:39:23.044783+0000","last_fullsized":"2024-08-17T17:39:23.044783+0000","mapping_epoch":337,"log_start":"325'2763","ondisk_log_start":"325'2763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330891264,"num_objects":5049,"num_object_clones":0,"num_object_copies":15147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5049,"num_objects_unfound":0,"num_objects_dirty":5049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9655,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1574,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":5049}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":78,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.526037+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:39:08.526037+0000","last_peered":"2024-08-17T17:39:08.526037+0000","last_clean":"2024-08-17T17:39:08.526037+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:39:08.526037+0000","last_undegraded":"2024-08-17T17:39:08.526037+0000","last_fullsized":"2024-08-17T17:39:08.526037+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":259,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.494009+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:39:08.494009+0000","last_peered":"2024-08-17T17:39:08.494009+0000","last_clean":"2024-08-17T17:39:08.494009+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:39:08.494009+0000","last_undegraded":"2024-08-17T17:39:08.494009+0000","last_fullsized":"2024-08-17T17:39:08.494009+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"373'9878","reported_seq":9949,"reported_epoch":373,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:22.062317+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:39:22.062317+0000","last_peered":"2024-08-17T17:39:22.062317+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:39:22.062317+0000","last_undegraded":"2024-08-17T17:39:22.062317+0000","last_fullsized":"2024-08-17T17:39:22.062317+0000","mapping_epoch":337,"log_start":"325'3282","ondisk_log_start":"325'3282","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6596,"log_dups_size":0,"ondisk_log_size":6596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340525056,"num_objects":5196,"num_object_clones":0,"num_object_copies":15588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5196,"num_objects_unfound":0,"num_objects_dirty":5196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9878,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5196}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":80,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464216+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:39:08.464216+0000","last_peered":"2024-08-17T17:39:08.464216+0000","last_clean":"2024-08-17T17:39:08.464216+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:39:08.464216+0000","last_undegraded":"2024-08-17T17:39:08.464216+0000","last_fullsized":"2024-08-17T17:39:08.464216+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":344,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464208+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:39:08.464208+0000","last_peered":"2024-08-17T17:39:08.464208+0000","last_clean":"2024-08-17T17:39:08.464208+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:39:08.464208+0000","last_undegraded":"2024-08-17T17:39:08.464208+0000","last_fullsized":"2024-08-17T17:39:08.464208+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"373'9600","reported_seq":9674,"reported_epoch":373,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:23.084620+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:39:23.084620+0000","last_peered":"2024-08-17T17:39:23.084620+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:39:23.084620+0000","last_undegraded":"2024-08-17T17:39:23.084620+0000","last_fullsized":"2024-08-17T17:39:23.084620+0000","mapping_epoch":337,"log_start":"325'2753","ondisk_log_start":"325'2753","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6847,"log_dups_size":0,"ondisk_log_size":6847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336461824,"num_objects":5134,"num_object_clones":0,"num_object_copies":15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5134,"num_objects_unfound":0,"num_objects_dirty":5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9600,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5134}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":76,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.492931+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:39:08.492931+0000","last_peered":"2024-08-17T17:39:08.492931+0000","last_clean":"2024-08-17T17:39:08.492931+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:39:08.492931+0000","last_undegraded":"2024-08-17T17:39:08.492931+0000","last_fullsized":"2024-08-17T17:39:08.492931+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":354,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.473374+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:39:08.473374+0000","last_peered":"2024-08-17T17:39:08.473374+0000","last_clean":"2024-08-17T17:39:08.473374+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:39:08.473374+0000","last_undegraded":"2024-08-17T17:39:08.473374+0000","last_fullsized":"2024-08-17T17:39:08.473374+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"373'9763","reported_seq":9888,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:19.429233+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:39:19.429233+0000","last_peered":"2024-08-17T17:39:19.429233+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:39:19.429233+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'2713","ondisk_log_start":"325'2713","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340459520,"num_objects":5195,"num_object_clones":0,"num_object_copies":15585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9763,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1574},{"shards":"0,3,7","objects":5195}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":500,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467915+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:39:18.467915+0000","last_peered":"2024-08-17T17:39:18.467915+0000","last_clean":"2024-08-17T17:39:18.467915+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:39:18.467915+0000","last_undegraded":"2024-08-17T17:39:18.467915+0000","last_fullsized":"2024-08-17T17:39:18.467915+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":266,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.473892+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:39:18.473892+0000","last_peered":"2024-08-17T17:39:18.473892+0000","last_clean":"2024-08-17T17:39:18.473892+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:39:18.473892+0000","last_undegraded":"2024-08-17T17:39:18.473892+0000","last_fullsized":"2024-08-17T17:39:18.473892+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"373'9538","reported_seq":9600,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:21.843832+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:39:21.843832+0000","last_peered":"2024-08-17T17:39:21.843832+0000","last_clean":"2024-08-17T17:39:21.843832+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:39:21.843832+0000","last_undegraded":"2024-08-17T17:39:21.843832+0000","last_fullsized":"2024-08-17T17:39:21.843832+0000","mapping_epoch":319,"log_start":"325'2974","ondisk_log_start":"325'2974","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6564,"log_dups_size":0,"ondisk_log_size":6564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331350016,"num_objects":5056,"num_object_clones":0,"num_object_copies":15168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9538,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":589,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.469356+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:39:18.469356+0000","last_peered":"2024-08-17T17:39:18.469356+0000","last_clean":"2024-08-17T17:39:18.469356+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:39:18.469356+0000","last_undegraded":"2024-08-17T17:39:18.469356+0000","last_fullsized":"2024-08-17T17:39:18.469356+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":327,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463785+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:39:08.463785+0000","last_peered":"2024-08-17T17:39:08.463785+0000","last_clean":"2024-08-17T17:39:08.463785+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:39:08.463785+0000","last_undegraded":"2024-08-17T17:39:08.463785+0000","last_fullsized":"2024-08-17T17:39:08.463785+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"373'9691","reported_seq":9763,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:22.025028+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:39:22.025028+0000","last_peered":"2024-08-17T17:39:22.025028+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:39:22.025028+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'3135","ondisk_log_start":"325'3135","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343474176,"num_objects":5241,"num_object_clones":0,"num_object_copies":15723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9691,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5241},{"shards":"1,6","objects":-2125}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":328,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.473498+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:39:08.473498+0000","last_peered":"2024-08-17T17:39:08.473498+0000","last_clean":"2024-08-17T17:39:08.473498+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:39:08.473498+0000","last_undegraded":"2024-08-17T17:39:08.473498+0000","last_fullsized":"2024-08-17T17:39:08.473498+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":354,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464091+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:39:08.464091+0000","last_peered":"2024-08-17T17:39:08.464091+0000","last_clean":"2024-08-17T17:39:08.464091+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:39:08.464091+0000","last_undegraded":"2024-08-17T17:39:08.464091+0000","last_fullsized":"2024-08-17T17:39:08.464091+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"373'9831","reported_seq":9957,"reported_epoch":373,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:23.813641+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:39:23.813641+0000","last_peered":"2024-08-17T17:39:23.813641+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:39:23.813641+0000","last_undegraded":"2024-08-17T17:39:23.813641+0000","last_fullsized":"2024-08-17T17:39:23.813641+0000","mapping_epoch":337,"log_start":"326'4261","ondisk_log_start":"326'4261","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5570,"log_dups_size":0,"ondisk_log_size":5570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339410944,"num_objects":5179,"num_object_clones":0,"num_object_copies":15537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5179,"num_objects_unfound":0,"num_objects_dirty":5179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9831,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5179}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":365,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463700+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:39:08.463700+0000","last_peered":"2024-08-17T17:39:08.463700+0000","last_clean":"2024-08-17T17:39:08.463700+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:39:08.463700+0000","last_undegraded":"2024-08-17T17:39:08.463700+0000","last_fullsized":"2024-08-17T17:39:08.463700+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":327,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464279+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:39:08.464279+0000","last_peered":"2024-08-17T17:39:08.464279+0000","last_clean":"2024-08-17T17:39:08.464279+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:39:08.464279+0000","last_undegraded":"2024-08-17T17:39:08.464279+0000","last_fullsized":"2024-08-17T17:39:08.464279+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"373'9638","reported_seq":12888,"reported_epoch":373,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:23.001699+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:39:23.001699+0000","last_peered":"2024-08-17T17:39:23.001699+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:39:23.001699+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:39:23.001699+0000","mapping_epoch":337,"log_start":"325'2741","ondisk_log_start":"325'2741","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":5761,"num_objects_missing":5761,"num_objects_degraded":5066,"num_objects_misplaced":5761,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9638,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1580,"num_bytes_recovered":103481344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":11},{"shards":"1,2,7","objects":5055}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":559,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467694+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:39:18.467694+0000","last_peered":"2024-08-17T17:39:18.467694+0000","last_clean":"2024-08-17T17:39:18.467694+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:39:18.467694+0000","last_undegraded":"2024-08-17T17:39:18.467694+0000","last_fullsized":"2024-08-17T17:39:18.467694+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":496,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.468058+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:39:18.468058+0000","last_peered":"2024-08-17T17:39:18.468058+0000","last_clean":"2024-08-17T17:39:18.468058+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:39:18.468058+0000","last_undegraded":"2024-08-17T17:39:18.468058+0000","last_fullsized":"2024-08-17T17:39:18.468058+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"373'9749","reported_seq":12966,"reported_epoch":373,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:23.937274+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:39:23.937274+0000","last_peered":"2024-08-17T17:39:23.937274+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:39:23.937274+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:39:23.937274+0000","mapping_epoch":337,"log_start":"326'4144","ondisk_log_start":"326'4144","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337248280,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5147,"num_objects_misplaced":5897,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9749,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1551,"num_bytes_recovered":101580800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5147}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":689,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:18.471122+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:39:18.471122+0000","last_peered":"2024-08-17T17:39:18.471122+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:39:18.471122+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":302,"reported_epoch":372,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:39:08.493346+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:39:08.493346+0000","last_peered":"2024-08-17T17:39:08.493346+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:39:08.493346+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:39:08.493346+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":538,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467765+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:39:18.467765+0000","last_peered":"2024-08-17T17:39:18.467765+0000","last_clean":"2024-08-17T17:39:18.467765+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:39:18.467765+0000","last_undegraded":"2024-08-17T17:39:18.467765+0000","last_fullsized":"2024-08-17T17:39:18.467765+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":337,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.473898+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:39:08.473898+0000","last_peered":"2024-08-17T17:39:08.473898+0000","last_clean":"2024-08-17T17:39:08.473898+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:39:08.473898+0000","last_undegraded":"2024-08-17T17:39:08.473898+0000","last_fullsized":"2024-08-17T17:39:08.473898+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"373'9606","reported_seq":9734,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:19.251601+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:39:19.251601+0000","last_peered":"2024-08-17T17:39:19.251601+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:39:19.251601+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'2621","ondisk_log_start":"325'2621","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336723968,"num_objects":5138,"num_object_clones":0,"num_object_copies":15414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9606,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1509},{"shards":"0,4,7","objects":5138}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":354,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.371950+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:39:18.371950+0000","last_peered":"2024-08-17T17:39:18.371950+0000","last_clean":"2024-08-17T17:39:18.371950+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:39:18.371950+0000","last_undegraded":"2024-08-17T17:39:18.371950+0000","last_fullsized":"2024-08-17T17:39:18.371950+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":309,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463422+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:39:08.463422+0000","last_peered":"2024-08-17T17:39:08.463422+0000","last_clean":"2024-08-17T17:39:08.463422+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:39:08.463422+0000","last_undegraded":"2024-08-17T17:39:08.463422+0000","last_fullsized":"2024-08-17T17:39:08.463422+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":280,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471962+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:39:18.471962+0000","last_peered":"2024-08-17T17:39:18.471962+0000","last_clean":"2024-08-17T17:39:18.471962+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:39:18.471962+0000","last_undegraded":"2024-08-17T17:39:18.471962+0000","last_fullsized":"2024-08-17T17:39:18.471962+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"373'9641","reported_seq":9713,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:21.957763+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:39:21.957763+0000","last_peered":"2024-08-17T17:39:21.957763+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:39:21.957763+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'3117","ondisk_log_start":"325'3117","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337838080,"num_objects":5155,"num_object_clones":0,"num_object_copies":15465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9641,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5155},{"shards":"5,6","objects":-2141}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":446,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467694+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:39:18.467694+0000","last_peered":"2024-08-17T17:39:18.467694+0000","last_clean":"2024-08-17T17:39:18.467694+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:39:18.467694+0000","last_undegraded":"2024-08-17T17:39:18.467694+0000","last_fullsized":"2024-08-17T17:39:18.467694+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.480638+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:39:08.480638+0000","last_peered":"2024-08-17T17:39:08.480638+0000","last_clean":"2024-08-17T17:39:08.480638+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:39:08.480638+0000","last_undegraded":"2024-08-17T17:39:08.480638+0000","last_fullsized":"2024-08-17T17:39:08.480638+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":631,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.373389+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:39:18.373389+0000","last_peered":"2024-08-17T17:39:18.373389+0000","last_clean":"2024-08-17T17:39:18.373389+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:39:18.373389+0000","last_undegraded":"2024-08-17T17:39:18.373389+0000","last_fullsized":"2024-08-17T17:39:18.373389+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"373'9539","reported_seq":10391,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:19.523369+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:39:19.523369+0000","last_peered":"2024-08-17T17:39:19.523369+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:39:19.523369+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"324'2516","ondisk_log_start":"324'2516","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9539,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5107},{"shards":"0,5","objects":-1547}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":423,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.372394+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:39:18.372394+0000","last_peered":"2024-08-17T17:39:18.372394+0000","last_clean":"2024-08-17T17:39:18.372394+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:39:18.372394+0000","last_undegraded":"2024-08-17T17:39:18.372394+0000","last_fullsized":"2024-08-17T17:39:18.372394+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":348,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464725+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:39:08.464725+0000","last_peered":"2024-08-17T17:39:08.464725+0000","last_clean":"2024-08-17T17:39:08.464725+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:39:08.464725+0000","last_undegraded":"2024-08-17T17:39:08.464725+0000","last_fullsized":"2024-08-17T17:39:08.464725+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":217,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467758+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:39:18.467758+0000","last_peered":"2024-08-17T17:39:18.467758+0000","last_clean":"2024-08-17T17:39:18.467758+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:39:18.467758+0000","last_undegraded":"2024-08-17T17:39:18.467758+0000","last_fullsized":"2024-08-17T17:39:18.467758+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"373'9729","reported_seq":24700,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:20.419573+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:39:20.419573+0000","last_peered":"2024-08-17T17:39:20.419573+0000","last_clean":"2024-08-17T17:39:20.419573+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:39:20.419573+0000","last_undegraded":"2024-08-17T17:39:20.419573+0000","last_fullsized":"2024-08-17T17:39:20.419573+0000","mapping_epoch":355,"log_start":"325'3430","ondisk_log_start":"325'3430","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6299,"log_dups_size":0,"ondisk_log_size":6299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9729,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":287,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.492835+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:39:08.492835+0000","last_peered":"2024-08-17T17:39:08.492835+0000","last_clean":"2024-08-17T17:39:08.492835+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:39:08.492835+0000","last_undegraded":"2024-08-17T17:39:08.492835+0000","last_fullsized":"2024-08-17T17:39:08.492835+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":86,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467648+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:39:18.467648+0000","last_peered":"2024-08-17T17:39:18.467648+0000","last_clean":"2024-08-17T17:39:18.467648+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:39:18.467648+0000","last_undegraded":"2024-08-17T17:39:18.467648+0000","last_fullsized":"2024-08-17T17:39:18.467648+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":582,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.372829+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:39:18.372829+0000","last_peered":"2024-08-17T17:39:18.372829+0000","last_clean":"2024-08-17T17:39:18.372829+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:39:18.372829+0000","last_undegraded":"2024-08-17T17:39:18.372829+0000","last_fullsized":"2024-08-17T17:39:18.372829+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"373'9428","reported_seq":23967,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:19.724829+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:39:19.724829+0000","last_peered":"2024-08-17T17:39:19.724829+0000","last_clean":"2024-08-17T17:39:19.724829+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:39:19.724829+0000","last_undegraded":"2024-08-17T17:39:19.724829+0000","last_fullsized":"2024-08-17T17:39:19.724829+0000","mapping_epoch":345,"log_start":"325'3172","ondisk_log_start":"325'3172","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326107136,"num_objects":4976,"num_object_clones":0,"num_object_copies":14928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9428,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":283,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.493024+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:39:08.493024+0000","last_peered":"2024-08-17T17:39:08.493024+0000","last_clean":"2024-08-17T17:39:08.493024+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:39:08.493024+0000","last_undegraded":"2024-08-17T17:39:08.493024+0000","last_fullsized":"2024-08-17T17:39:08.493024+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":337,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471658+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:39:18.471658+0000","last_peered":"2024-08-17T17:39:18.471658+0000","last_clean":"2024-08-17T17:39:18.471658+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:39:18.471658+0000","last_undegraded":"2024-08-17T17:39:18.471658+0000","last_fullsized":"2024-08-17T17:39:18.471658+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":425,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.493244+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:39:08.493244+0000","last_peered":"2024-08-17T17:39:08.493244+0000","last_clean":"2024-08-17T17:39:08.493244+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:39:08.493244+0000","last_undegraded":"2024-08-17T17:39:08.493244+0000","last_fullsized":"2024-08-17T17:39:08.493244+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"373'9672","reported_seq":9819,"reported_epoch":373,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:19.768032+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:39:19.768032+0000","last_peered":"2024-08-17T17:39:19.768032+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:39:19.768032+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'3387","ondisk_log_start":"325'3387","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6285,"log_dups_size":0,"ondisk_log_size":6285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335020032,"num_objects":5112,"num_object_clones":0,"num_object_copies":15336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9672,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":5112},{"shards":"0,3","objects":-2136}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":612,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.468150+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:39:18.468150+0000","last_peered":"2024-08-17T17:39:18.468150+0000","last_clean":"2024-08-17T17:39:18.468150+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:39:18.468150+0000","last_undegraded":"2024-08-17T17:39:18.468150+0000","last_fullsized":"2024-08-17T17:39:18.468150+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":580,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.373666+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:39:18.373666+0000","last_peered":"2024-08-17T17:39:18.373666+0000","last_clean":"2024-08-17T17:39:18.373666+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:39:18.373666+0000","last_undegraded":"2024-08-17T17:39:18.373666+0000","last_fullsized":"2024-08-17T17:39:18.373666+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":598,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467517+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:39:18.467517+0000","last_peered":"2024-08-17T17:39:18.467517+0000","last_clean":"2024-08-17T17:39:18.467517+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:39:18.467517+0000","last_undegraded":"2024-08-17T17:39:18.467517+0000","last_fullsized":"2024-08-17T17:39:18.467517+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"373'9582","reported_seq":24252,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:22.037051+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:39:22.037051+0000","last_peered":"2024-08-17T17:39:22.037051+0000","last_clean":"2024-08-17T17:39:22.037051+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:39:22.037051+0000","last_undegraded":"2024-08-17T17:39:22.037051+0000","last_fullsized":"2024-08-17T17:39:22.037051+0000","mapping_epoch":347,"log_start":"325'2972","ondisk_log_start":"325'2972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9582,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":270,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471854+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:39:18.471854+0000","last_peered":"2024-08-17T17:39:18.471854+0000","last_clean":"2024-08-17T17:39:18.471854+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:39:18.471854+0000","last_undegraded":"2024-08-17T17:39:18.471854+0000","last_fullsized":"2024-08-17T17:39:18.471854+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":377,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.475434+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:39:08.475434+0000","last_peered":"2024-08-17T17:39:08.475434+0000","last_clean":"2024-08-17T17:39:08.475434+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:39:08.475434+0000","last_undegraded":"2024-08-17T17:39:08.475434+0000","last_fullsized":"2024-08-17T17:39:08.475434+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":327,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463632+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:39:08.463632+0000","last_peered":"2024-08-17T17:39:08.463632+0000","last_clean":"2024-08-17T17:39:08.463632+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:39:08.463632+0000","last_undegraded":"2024-08-17T17:39:08.463632+0000","last_fullsized":"2024-08-17T17:39:08.463632+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":327,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464603+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:39:08.464603+0000","last_peered":"2024-08-17T17:39:08.464603+0000","last_clean":"2024-08-17T17:39:08.464603+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:39:08.464603+0000","last_undegraded":"2024-08-17T17:39:08.464603+0000","last_fullsized":"2024-08-17T17:39:08.464603+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":370,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463952+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:39:08.463952+0000","last_peered":"2024-08-17T17:39:08.463952+0000","last_clean":"2024-08-17T17:39:08.463952+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:39:08.463952+0000","last_undegraded":"2024-08-17T17:39:08.463952+0000","last_fullsized":"2024-08-17T17:39:08.463952+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":587,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.372614+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:39:18.372614+0000","last_peered":"2024-08-17T17:39:18.372614+0000","last_clean":"2024-08-17T17:39:18.372614+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:39:18.372614+0000","last_undegraded":"2024-08-17T17:39:18.372614+0000","last_fullsized":"2024-08-17T17:39:18.372614+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":127,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.468303+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:39:18.468303+0000","last_peered":"2024-08-17T17:39:18.468303+0000","last_clean":"2024-08-17T17:39:18.468303+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:39:18.468303+0000","last_undegraded":"2024-08-17T17:39:18.468303+0000","last_fullsized":"2024-08-17T17:39:18.468303+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":277,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.493698+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:39:08.493698+0000","last_peered":"2024-08-17T17:39:08.493698+0000","last_clean":"2024-08-17T17:39:08.493698+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:39:08.493698+0000","last_undegraded":"2024-08-17T17:39:08.493698+0000","last_fullsized":"2024-08-17T17:39:08.493698+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":545,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467495+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:39:18.467495+0000","last_peered":"2024-08-17T17:39:18.467495+0000","last_clean":"2024-08-17T17:39:18.467495+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:39:18.467495+0000","last_undegraded":"2024-08-17T17:39:18.467495+0000","last_fullsized":"2024-08-17T17:39:18.467495+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":421,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.493935+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:39:08.493935+0000","last_peered":"2024-08-17T17:39:08.493935+0000","last_clean":"2024-08-17T17:39:08.493935+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:39:08.493935+0000","last_undegraded":"2024-08-17T17:39:08.493935+0000","last_fullsized":"2024-08-17T17:39:08.493935+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":539,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467421+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:39:18.467421+0000","last_peered":"2024-08-17T17:39:18.467421+0000","last_clean":"2024-08-17T17:39:18.467421+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:39:18.467421+0000","last_undegraded":"2024-08-17T17:39:18.467421+0000","last_fullsized":"2024-08-17T17:39:18.467421+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":344,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.475379+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:39:08.475379+0000","last_peered":"2024-08-17T17:39:08.475379+0000","last_clean":"2024-08-17T17:39:08.475379+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:39:08.475379+0000","last_undegraded":"2024-08-17T17:39:08.475379+0000","last_fullsized":"2024-08-17T17:39:08.475379+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":571,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467599+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:39:18.467599+0000","last_peered":"2024-08-17T17:39:18.467599+0000","last_clean":"2024-08-17T17:39:18.467599+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:39:18.467599+0000","last_undegraded":"2024-08-17T17:39:18.467599+0000","last_fullsized":"2024-08-17T17:39:18.467599+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":286,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467980+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:39:18.467980+0000","last_peered":"2024-08-17T17:39:18.467980+0000","last_clean":"2024-08-17T17:39:18.467980+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:39:18.467980+0000","last_undegraded":"2024-08-17T17:39:18.467980+0000","last_fullsized":"2024-08-17T17:39:18.467980+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":579,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.372292+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:39:18.372292+0000","last_peered":"2024-08-17T17:39:18.372292+0000","last_clean":"2024-08-17T17:39:18.372292+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:39:18.372292+0000","last_undegraded":"2024-08-17T17:39:18.372292+0000","last_fullsized":"2024-08-17T17:39:18.372292+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":538,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.468161+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:39:18.468161+0000","last_peered":"2024-08-17T17:39:18.468161+0000","last_clean":"2024-08-17T17:39:18.468161+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:39:18.468161+0000","last_undegraded":"2024-08-17T17:39:18.468161+0000","last_fullsized":"2024-08-17T17:39:18.468161+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":356,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464066+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:39:08.464066+0000","last_peered":"2024-08-17T17:39:08.464066+0000","last_clean":"2024-08-17T17:39:08.464066+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:39:08.464066+0000","last_undegraded":"2024-08-17T17:39:08.464066+0000","last_fullsized":"2024-08-17T17:39:08.464066+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":227,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467895+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:39:18.467895+0000","last_peered":"2024-08-17T17:39:18.467895+0000","last_clean":"2024-08-17T17:39:18.467895+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:39:18.467895+0000","last_undegraded":"2024-08-17T17:39:18.467895+0000","last_fullsized":"2024-08-17T17:39:18.467895+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":283,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471748+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:39:18.471748+0000","last_peered":"2024-08-17T17:39:18.471748+0000","last_clean":"2024-08-17T17:39:18.471748+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:39:18.471748+0000","last_undegraded":"2024-08-17T17:39:18.471748+0000","last_fullsized":"2024-08-17T17:39:18.471748+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":364,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.464542+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:39:08.464542+0000","last_peered":"2024-08-17T17:39:08.464542+0000","last_clean":"2024-08-17T17:39:08.464542+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:39:08.464542+0000","last_undegraded":"2024-08-17T17:39:08.464542+0000","last_fullsized":"2024-08-17T17:39:08.464542+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":227,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463869+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:39:08.463869+0000","last_peered":"2024-08-17T17:39:08.463869+0000","last_clean":"2024-08-17T17:39:08.463869+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:39:08.463869+0000","last_undegraded":"2024-08-17T17:39:08.463869+0000","last_fullsized":"2024-08-17T17:39:08.463869+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":364,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463898+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:39:08.463898+0000","last_peered":"2024-08-17T17:39:08.463898+0000","last_clean":"2024-08-17T17:39:08.463898+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:39:08.463898+0000","last_undegraded":"2024-08-17T17:39:08.463898+0000","last_fullsized":"2024-08-17T17:39:08.463898+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":227,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463964+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:39:08.463964+0000","last_peered":"2024-08-17T17:39:08.463964+0000","last_clean":"2024-08-17T17:39:08.463964+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:39:08.463964+0000","last_undegraded":"2024-08-17T17:39:08.463964+0000","last_fullsized":"2024-08-17T17:39:08.463964+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":591,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.467840+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:39:18.467840+0000","last_peered":"2024-08-17T17:39:18.467840+0000","last_clean":"2024-08-17T17:39:18.467840+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:39:18.467840+0000","last_undegraded":"2024-08-17T17:39:18.467840+0000","last_fullsized":"2024-08-17T17:39:18.467840+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":564,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.468163+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:39:18.468163+0000","last_peered":"2024-08-17T17:39:18.468163+0000","last_clean":"2024-08-17T17:39:18.468163+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:39:18.468163+0000","last_undegraded":"2024-08-17T17:39:18.468163+0000","last_fullsized":"2024-08-17T17:39:18.468163+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":327,"reported_epoch":372,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.463458+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:39:08.463458+0000","last_peered":"2024-08-17T17:39:08.463458+0000","last_clean":"2024-08-17T17:39:08.463458+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:39:08.463458+0000","last_undegraded":"2024-08-17T17:39:08.463458+0000","last_fullsized":"2024-08-17T17:39:08.463458+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":286,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.472113+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:39:18.472113+0000","last_peered":"2024-08-17T17:39:18.472113+0000","last_clean":"2024-08-17T17:39:18.472113+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:39:18.472113+0000","last_undegraded":"2024-08-17T17:39:18.472113+0000","last_fullsized":"2024-08-17T17:39:18.472113+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5364056088,"num_objects":81850,"num_object_clones":0,"num_object_copies":245550,"num_objects_missing_on_primary":5761,"num_objects_missing":5761,"num_objects_degraded":54632,"num_objects_misplaced":32216,"num_objects_unfound":0,"num_objects_dirty":81850,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154540,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26984,"num_bytes_recovered":1645871104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17406373888,"data_stored":17406361672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104559,"ondisk_log_size":104559,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170075,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463445,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3183912,"kb_used_data":2290208,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90561240,"statfs":{"total":95995035648,"available":92734709760,"internally_reserved":0,"allocated":2345172992,"data_stored":2343165964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492119,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049168,"kb_used_data":662148,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92695984,"statfs":{"total":95995035648,"available":94920687616,"internally_reserved":0,"allocated":678039552,"data_stored":676054824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986711,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2484444,"kb_used_data":1303444,"kb_used_omap":1,"kb_used_meta":1180990,"kb_avail":91260708,"statfs":{"total":95995035648,"available":93450964992,"internally_reserved":0,"allocated":1334726656,"data_stored":1332730568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1209334104},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,4,1,2,1,3,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225538,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3530060,"kb_used_data":2618500,"kb_used_omap":4,"kb_used_meta":911547,"kb_avail":90215092,"statfs":{"total":95995035648,"available":92380254208,"internally_reserved":0,"allocated":2681344000,"data_stored":2679359570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":933424576},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075863,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533757,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3329872,"kb_used_data":2292040,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90415280,"statfs":{"total":95995035648,"available":92585246720,"internally_reserved":0,"allocated":2347048960,"data_stored":2345059208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089733,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2983596,"kb_used_data":1982688,"kb_used_omap":5,"kb_used_meta":1000890,"kb_avail":90761556,"statfs":{"total":95995035648,"available":92939833344,"internally_reserved":0,"allocated":2030272512,"data_stored":2028265543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1024911419},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2023292928,"data_stored":2023292928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2339835904,"data_stored":2339831832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3458011136,"data_stored":3458007064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2674917376,"data_stored":2674917376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1328349184,"data_stored":1328349184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":673120256,"data_stored":673120256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2337275904,"data_stored":2337275904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2571571200,"data_stored":2571567128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:29.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:29 smithi152 ceph-mon[93456]: pgmap v1380: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54603/244158 objects degraded (22.364%); 32084/244158 objects misplaced (13.141%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/580293350' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:29.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:29.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:29.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:30.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:30.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:30.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:30.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:30.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:30.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:30.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:30.385 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:39:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:39:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:30.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:30.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:30.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:30.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:30.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:30.602 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:30 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:30.602 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:30.602 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:30 smithi142 ceph-mon[103614]: osdmap e374: 8 total, 8 up, 6 in 2024-08-17T17:39:30.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:30 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:30.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:30.602 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:30 smithi142 ceph-mon[106729]: osdmap e374: 8 total, 8 up, 6 in 2024-08-17T17:39:30.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:30.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:30 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:30 smithi152 ceph-mon[93456]: osdmap e374: 8 total, 8 up, 6 in 2024-08-17T17:39:30.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:30.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:30.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:30.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:30.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:31.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:31.098 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:31.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:31.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:31.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:31.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:31.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:31.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:31.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:31.592 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:31.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:31.676 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:31 smithi142 ceph-mon[106729]: pgmap v1382: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54570/243588 objects degraded (22.403%); 32019/243588 objects misplaced (13.145%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:31.676 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54570/243588 objects degraded (22.403%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:31.677 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:31 smithi142 ceph-mon[103614]: pgmap v1382: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54570/243588 objects degraded (22.403%); 32019/243588 objects misplaced (13.145%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:31.677 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54570/243588 objects degraded (22.403%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:31.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:31.715 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:31.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:31 smithi152 ceph-mon[93456]: pgmap v1382: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54570/243588 objects degraded (22.403%); 32019/243588 objects misplaced (13.145%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54570/243588 objects degraded (22.403%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:31.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:31.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:31.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:32.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:39:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:39:31] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:39:32.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:32.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:32.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:32.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:32.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:32.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:32.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:32.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:32.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:32.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:32.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:32.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:32.835 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:32.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:32.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:33.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:33.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:33.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:33.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:33.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:33.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:33.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:33.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:39:33.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:33.503 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:33.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:33.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:33.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:33.701 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:33 smithi142 ceph-mon[103614]: pgmap v1383: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 54570/242811 objects degraded (22.474%); 31993/242811 objects misplaced (13.176%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:33.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:33 smithi142 ceph-mon[106729]: pgmap v1383: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 54570/242811 objects degraded (22.474%); 31993/242811 objects misplaced (13.176%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:33.728 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:33.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:33.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:33 smithi152 ceph-mon[93456]: pgmap v1383: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 54570/242811 objects degraded (22.474%); 31993/242811 objects misplaced (13.176%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:33.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:33.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:33.909 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:39:33.910 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:39:33.910 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:39:33.910 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:39:33.910 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:39:33.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:33.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:34.101 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:34.123 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:34.130 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:34.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:39:34.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:34.322 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:34.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:34.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:34.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:34.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:34.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1601081752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:34.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1601081752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:34.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:34.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1601081752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:34.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.871 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:34.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:34.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:35.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:35.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:35.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:35.243 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:35.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:35.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:35.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:35.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:35.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:35.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:35.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:35.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:35.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:35.744 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:35 smithi152 ceph-mon[93456]: pgmap v1384: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 54496/242265 objects degraded (22.494%); 31813/242265 objects misplaced (13.131%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:35.842 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:35 smithi142 ceph-mon[106729]: pgmap v1384: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 54496/242265 objects degraded (22.494%); 31813/242265 objects misplaced (13.131%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:35.842 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:35 smithi142 ceph-mon[103614]: pgmap v1384: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 54496/242265 objects degraded (22.494%); 31813/242265 objects misplaced (13.131%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:35.847 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:35.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:39:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:35.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:35.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:35.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:36.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:36.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:36.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:36.244 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:36.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:36.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:36.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:36.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:36.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:36.591 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:36.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:36.714 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:36.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54496/242265 objects degraded (22.494%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:36.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:36.849 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54496/242265 objects degraded (22.494%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:36.849 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54496/242265 objects degraded (22.494%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:36.855 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:36.866 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:36.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:36.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:36.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:37.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:37.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:37.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:37.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:37.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:37.342 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:37.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:37.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:37.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:37.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:37.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:37.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:37.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:37 smithi152 ceph-mon[93456]: pgmap v1385: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 54496/241584 objects degraded (22.558%); 31813/241584 objects misplaced (13.169%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:37.823 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:37 smithi142 ceph-mon[106729]: pgmap v1385: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 54496/241584 objects degraded (22.558%); 31813/241584 objects misplaced (13.169%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:37.824 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:37 smithi142 ceph-mon[103614]: pgmap v1385: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 54496/241584 objects degraded (22.558%); 31813/241584 objects misplaced (13.169%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:39:37.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:37.890 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:37.891 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:39:37.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:38.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:38.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:38.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:38.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:38.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:38.362 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:38.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:38.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:38.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:38.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:38.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:38.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:38 smithi152 ceph-mon[93456]: from='client.37446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:38 smithi152 ceph-mon[93456]: pgmap v1386: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 54465/240537 objects degraded (22.643%); 31636/240537 objects misplaced (13.152%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:38.829 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:38.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:38.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:38 smithi142 ceph-mon[103614]: from='client.37446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:38 smithi142 ceph-mon[103614]: pgmap v1386: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 54465/240537 objects degraded (22.643%); 31636/240537 objects misplaced (13.152%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:38 smithi142 ceph-mon[106729]: from='client.37446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:38 smithi142 ceph-mon[106729]: pgmap v1386: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 54465/240537 objects degraded (22.643%); 31636/240537 objects misplaced (13.152%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:38.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:38.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:39.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:39.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:39.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:39.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:39.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:39.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:39.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:39.357 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1385,"stamp":"2024-08-17T17:39:36.312703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5276309477,"num_objects":80528,"num_object_clones":0,"num_object_copies":241584,"num_objects_missing_on_primary":5696,"num_objects_missing":5696,"num_objects_degraded":54496,"num_objects_misplaced":31813,"num_objects_unfound":0,"num_objects_dirty":80528,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":156166,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27226,"num_bytes_recovered":1661411982,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104714,"ondisk_log_size":104714,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16384884,"kb_used_data":10960388,"kb_used_omap":27,"kb_used_meta":5424420,"kb_avail":546086028,"statfs":{"total":575970213888,"available":559192092672,"internally_reserved":0,"allocated":11223437312,"data_stored":11211421559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5554606824},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,0,0,0,3,3,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-74448896,"num_objects":-1136,"num_object_clones":0,"num_object_copies":-3408,"num_objects_missing_on_primary":-65,"num_objects_missing":-65,"num_objects_degraded":-139,"num_objects_misplaced":-403,"num_objects_unfound":0,"num_objects_dirty":-1136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1136,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":201,"num_bytes_recovered":13172736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002954"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":590,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451842+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:39:29.451842+0000","last_peered":"2024-08-17T17:39:29.451842+0000","last_clean":"2024-08-17T17:39:29.451842+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:39:29.451842+0000","last_undegraded":"2024-08-17T17:39:29.451842+0000","last_fullsized":"2024-08-17T17:39:29.451842+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":83,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389026+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:39:18.389026+0000","last_peered":"2024-08-17T17:39:18.389026+0000","last_clean":"2024-08-17T17:39:18.389026+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:39:18.389026+0000","last_undegraded":"2024-08-17T17:39:18.389026+0000","last_fullsized":"2024-08-17T17:39:18.389026+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":277,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374450+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:39:18.374450+0000","last_peered":"2024-08-17T17:39:18.374450+0000","last_clean":"2024-08-17T17:39:18.374450+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:39:18.374450+0000","last_undegraded":"2024-08-17T17:39:18.374450+0000","last_fullsized":"2024-08-17T17:39:18.374450+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":288,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451827+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:39:29.451827+0000","last_peered":"2024-08-17T17:39:29.451827+0000","last_clean":"2024-08-17T17:39:29.451827+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:39:29.451827+0000","last_undegraded":"2024-08-17T17:39:29.451827+0000","last_fullsized":"2024-08-17T17:39:29.451827+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":465,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.453032+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:39:29.453032+0000","last_peered":"2024-08-17T17:39:29.453032+0000","last_clean":"2024-08-17T17:39:29.453032+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:39:29.453032+0000","last_undegraded":"2024-08-17T17:39:29.453032+0000","last_fullsized":"2024-08-17T17:39:29.453032+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":226,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.388935+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:39:18.388935+0000","last_peered":"2024-08-17T17:39:18.388935+0000","last_clean":"2024-08-17T17:39:18.388935+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:39:18.388935+0000","last_undegraded":"2024-08-17T17:39:18.388935+0000","last_fullsized":"2024-08-17T17:39:18.388935+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":559,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452162+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:39:29.452162+0000","last_peered":"2024-08-17T17:39:29.452162+0000","last_clean":"2024-08-17T17:39:29.452162+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:39:29.452162+0000","last_undegraded":"2024-08-17T17:39:29.452162+0000","last_fullsized":"2024-08-17T17:39:29.452162+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":394,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.470761+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:39:18.470761+0000","last_peered":"2024-08-17T17:39:18.470761+0000","last_clean":"2024-08-17T17:39:18.470761+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:39:18.470761+0000","last_undegraded":"2024-08-17T17:39:18.470761+0000","last_fullsized":"2024-08-17T17:39:18.470761+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"374'9717","reported_seq":13075,"reported_epoch":374,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:39:33.167828+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:39:33.167828+0000","last_peered":"2024-08-17T17:39:33.167828+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:39:33.167828+0000","last_undegraded":"2024-08-17T17:39:33.167828+0000","last_fullsized":"2024-08-17T17:39:33.167828+0000","mapping_epoch":337,"log_start":"325'2863","ondisk_log_start":"325'2863","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6854,"log_dups_size":0,"ondisk_log_size":6854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326828032,"num_objects":4987,"num_object_clones":0,"num_object_copies":14961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4987,"num_objects_unfound":0,"num_objects_dirty":4987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9717,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1636,"num_bytes_recovered":107216896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4987}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":79,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.451908+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:39:18.451908+0000","last_peered":"2024-08-17T17:39:18.451908+0000","last_clean":"2024-08-17T17:39:18.451908+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:39:18.451908+0000","last_undegraded":"2024-08-17T17:39:18.451908+0000","last_fullsized":"2024-08-17T17:39:18.451908+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":260,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374082+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:39:18.374082+0000","last_peered":"2024-08-17T17:39:18.374082+0000","last_clean":"2024-08-17T17:39:18.374082+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:39:18.374082+0000","last_undegraded":"2024-08-17T17:39:18.374082+0000","last_fullsized":"2024-08-17T17:39:18.374082+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"374'9936","reported_seq":10008,"reported_epoch":374,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:32.137223+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:39:32.137223+0000","last_peered":"2024-08-17T17:39:32.137223+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:39:32.137223+0000","last_undegraded":"2024-08-17T17:39:32.137223+0000","last_fullsized":"2024-08-17T17:39:32.137223+0000","mapping_epoch":337,"log_start":"325'3382","ondisk_log_start":"325'3382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336723968,"num_objects":5138,"num_object_clones":0,"num_object_copies":15414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5138,"num_objects_unfound":0,"num_objects_dirty":5138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9936,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":5138}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":81,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389730+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:39:18.389730+0000","last_peered":"2024-08-17T17:39:18.389730+0000","last_clean":"2024-08-17T17:39:18.389730+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:39:18.389730+0000","last_undegraded":"2024-08-17T17:39:18.389730+0000","last_fullsized":"2024-08-17T17:39:18.389730+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":345,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.388914+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:39:18.388914+0000","last_peered":"2024-08-17T17:39:18.388914+0000","last_clean":"2024-08-17T17:39:18.388914+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:39:18.388914+0000","last_undegraded":"2024-08-17T17:39:18.388914+0000","last_fullsized":"2024-08-17T17:39:18.388914+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"374'9672","reported_seq":9747,"reported_epoch":374,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:33.007996+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:39:33.007996+0000","last_peered":"2024-08-17T17:39:33.007996+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:39:33.007996+0000","last_undegraded":"2024-08-17T17:39:33.007996+0000","last_fullsized":"2024-08-17T17:39:33.007996+0000","mapping_epoch":337,"log_start":"325'2853","ondisk_log_start":"325'2853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331743232,"num_objects":5062,"num_object_clones":0,"num_object_copies":15186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5062,"num_objects_unfound":0,"num_objects_dirty":5062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9672,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":5062}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":77,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374290+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:39:18.374290+0000","last_peered":"2024-08-17T17:39:18.374290+0000","last_clean":"2024-08-17T17:39:18.374290+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:39:18.374290+0000","last_undegraded":"2024-08-17T17:39:18.374290+0000","last_fullsized":"2024-08-17T17:39:18.374290+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":355,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.470817+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:39:18.470817+0000","last_peered":"2024-08-17T17:39:18.470817+0000","last_clean":"2024-08-17T17:39:18.470817+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:39:18.470817+0000","last_undegraded":"2024-08-17T17:39:18.470817+0000","last_fullsized":"2024-08-17T17:39:18.470817+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"374'9887","reported_seq":10014,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:34.910114+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:39:34.910114+0000","last_peered":"2024-08-17T17:39:34.910114+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:39:34.910114+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'2813","ondisk_log_start":"325'2813","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332333056,"num_objects":5071,"num_object_clones":0,"num_object_copies":15213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9887,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1698},{"shards":"0,3,7","objects":5071}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":502,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452584+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:39:29.452584+0000","last_peered":"2024-08-17T17:39:29.452584+0000","last_clean":"2024-08-17T17:39:29.452584+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:39:29.452584+0000","last_undegraded":"2024-08-17T17:39:29.452584+0000","last_fullsized":"2024-08-17T17:39:29.452584+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":268,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.456029+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:39:29.456029+0000","last_peered":"2024-08-17T17:39:29.456029+0000","last_clean":"2024-08-17T17:39:29.456029+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:39:29.456029+0000","last_undegraded":"2024-08-17T17:39:29.456029+0000","last_fullsized":"2024-08-17T17:39:29.456029+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"374'9615","reported_seq":9678,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.101369+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:39:32.101369+0000","last_peered":"2024-08-17T17:39:32.101369+0000","last_clean":"2024-08-17T17:39:32.101369+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:39:32.101369+0000","last_undegraded":"2024-08-17T17:39:32.101369+0000","last_fullsized":"2024-08-17T17:39:32.101369+0000","mapping_epoch":319,"log_start":"325'3074","ondisk_log_start":"325'3074","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6541,"log_dups_size":0,"ondisk_log_size":6541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326303744,"num_objects":4979,"num_object_clones":0,"num_object_copies":14937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9615,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":591,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451439+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:39:29.451439+0000","last_peered":"2024-08-17T17:39:29.451439+0000","last_clean":"2024-08-17T17:39:29.451439+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:39:29.451439+0000","last_undegraded":"2024-08-17T17:39:29.451439+0000","last_fullsized":"2024-08-17T17:39:29.451439+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":328,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389480+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:39:18.389480+0000","last_peered":"2024-08-17T17:39:18.389480+0000","last_clean":"2024-08-17T17:39:18.389480+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:39:18.389480+0000","last_undegraded":"2024-08-17T17:39:18.389480+0000","last_fullsized":"2024-08-17T17:39:18.389480+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"374'9760","reported_seq":9833,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:32.074684+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:39:32.074684+0000","last_peered":"2024-08-17T17:39:32.074684+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:39:32.074684+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'3235","ondisk_log_start":"325'3235","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338952192,"num_objects":5172,"num_object_clones":0,"num_object_copies":15516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9760,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5172},{"shards":"1,6","objects":-2194}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":329,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.470919+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:39:18.470919+0000","last_peered":"2024-08-17T17:39:18.470919+0000","last_clean":"2024-08-17T17:39:18.470919+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:39:18.470919+0000","last_undegraded":"2024-08-17T17:39:18.470919+0000","last_fullsized":"2024-08-17T17:39:18.470919+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":355,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389659+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:39:18.389659+0000","last_peered":"2024-08-17T17:39:18.389659+0000","last_clean":"2024-08-17T17:39:18.389659+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:39:18.389659+0000","last_undegraded":"2024-08-17T17:39:18.389659+0000","last_fullsized":"2024-08-17T17:39:18.389659+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"374'9902","reported_seq":10030,"reported_epoch":374,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:33.452545+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:39:33.452545+0000","last_peered":"2024-08-17T17:39:33.452545+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:39:33.452545+0000","last_undegraded":"2024-08-17T17:39:33.452545+0000","last_fullsized":"2024-08-17T17:39:33.452545+0000","mapping_epoch":337,"log_start":"326'4261","ondisk_log_start":"326'4261","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334757888,"num_objects":5108,"num_object_clones":0,"num_object_copies":15324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5108,"num_objects_unfound":0,"num_objects_dirty":5108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9902,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":5108}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":366,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389020+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:39:18.389020+0000","last_peered":"2024-08-17T17:39:18.389020+0000","last_clean":"2024-08-17T17:39:18.389020+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:39:18.389020+0000","last_undegraded":"2024-08-17T17:39:18.389020+0000","last_fullsized":"2024-08-17T17:39:18.389020+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":328,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389827+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:39:18.389827+0000","last_peered":"2024-08-17T17:39:18.389827+0000","last_clean":"2024-08-17T17:39:18.389827+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:39:18.389827+0000","last_undegraded":"2024-08-17T17:39:18.389827+0000","last_fullsized":"2024-08-17T17:39:18.389827+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"374'9702","reported_seq":13082,"reported_epoch":374,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:33.111649+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:39:33.111649+0000","last_peered":"2024-08-17T17:39:33.111649+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:39:33.111649+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:39:33.111649+0000","mapping_epoch":337,"log_start":"325'2841","ondisk_log_start":"325'2841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6861,"log_dups_size":0,"ondisk_log_size":6861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811072,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":5696,"num_objects_missing":5696,"num_objects_degraded":5002,"num_objects_misplaced":5696,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9702,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1644,"num_bytes_recovered":107675648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":4990}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":561,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451828+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:39:29.451828+0000","last_peered":"2024-08-17T17:39:29.451828+0000","last_clean":"2024-08-17T17:39:29.451828+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:39:29.451828+0000","last_undegraded":"2024-08-17T17:39:29.451828+0000","last_fullsized":"2024-08-17T17:39:29.451828+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":498,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452258+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:39:29.452258+0000","last_peered":"2024-08-17T17:39:29.452258+0000","last_clean":"2024-08-17T17:39:29.452258+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:39:29.452258+0000","last_undegraded":"2024-08-17T17:39:29.452258+0000","last_fullsized":"2024-08-17T17:39:29.452258+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"374'9824","reported_seq":13193,"reported_epoch":374,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:33.942177+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:39:33.942177+0000","last_peered":"2024-08-17T17:39:33.942177+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:39:33.942177+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:39:33.942177+0000","mapping_epoch":337,"log_start":"326'4244","ondisk_log_start":"326'4244","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332333080,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5072,"num_objects_misplaced":5822,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9824,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1626,"num_bytes_recovered":106496000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5072}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":691,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:29.451462+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:39:29.451462+0000","last_peered":"2024-08-17T17:39:29.451462+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:39:29.451462+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":303,"reported_epoch":373,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:39:18.374257+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:39:18.374257+0000","last_peered":"2024-08-17T17:39:18.374257+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:39:18.374257+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:39:18.374257+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":540,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451730+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:39:29.451730+0000","last_peered":"2024-08-17T17:39:29.451730+0000","last_clean":"2024-08-17T17:39:29.451730+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:39:29.451730+0000","last_undegraded":"2024-08-17T17:39:29.451730+0000","last_fullsized":"2024-08-17T17:39:29.451730+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":338,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471563+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:39:18.471563+0000","last_peered":"2024-08-17T17:39:18.471563+0000","last_clean":"2024-08-17T17:39:18.471563+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:39:18.471563+0000","last_undegraded":"2024-08-17T17:39:18.471563+0000","last_fullsized":"2024-08-17T17:39:18.471563+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"374'9722","reported_seq":9852,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:34.890358+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:39:34.890358+0000","last_peered":"2024-08-17T17:39:34.890358+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:39:34.890358+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'2721","ondisk_log_start":"325'2721","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329121792,"num_objects":5022,"num_object_clones":0,"num_object_copies":15066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9722,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1625},{"shards":"0,4,7","objects":5022}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":356,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.441797+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:39:29.441797+0000","last_peered":"2024-08-17T17:39:29.441797+0000","last_clean":"2024-08-17T17:39:29.441797+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:39:29.441797+0000","last_undegraded":"2024-08-17T17:39:29.441797+0000","last_fullsized":"2024-08-17T17:39:29.441797+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":310,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.388799+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:39:18.388799+0000","last_peered":"2024-08-17T17:39:18.388799+0000","last_clean":"2024-08-17T17:39:18.388799+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:39:18.388799+0000","last_undegraded":"2024-08-17T17:39:18.388799+0000","last_fullsized":"2024-08-17T17:39:18.388799+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":282,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.456162+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:39:29.456162+0000","last_peered":"2024-08-17T17:39:29.456162+0000","last_clean":"2024-08-17T17:39:29.456162+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:39:29.456162+0000","last_undegraded":"2024-08-17T17:39:29.456162+0000","last_fullsized":"2024-08-17T17:39:29.456162+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"374'9706","reported_seq":9779,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:30.880454+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:39:30.880454+0000","last_peered":"2024-08-17T17:39:30.880454+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:39:30.880454+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'3117","ondisk_log_start":"325'3117","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6589,"log_dups_size":0,"ondisk_log_size":6589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333578240,"num_objects":5090,"num_object_clones":0,"num_object_copies":15270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9706,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5090},{"shards":"5,6","objects":-2206}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":449,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452223+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:39:29.452223+0000","last_peered":"2024-08-17T17:39:29.452223+0000","last_clean":"2024-08-17T17:39:29.452223+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:39:29.452223+0000","last_undegraded":"2024-08-17T17:39:29.452223+0000","last_fullsized":"2024-08-17T17:39:29.452223+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":110,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.470961+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:39:18.470961+0000","last_peered":"2024-08-17T17:39:18.470961+0000","last_clean":"2024-08-17T17:39:18.470961+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:39:18.470961+0000","last_undegraded":"2024-08-17T17:39:18.470961+0000","last_fullsized":"2024-08-17T17:39:18.470961+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":633,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.442323+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:39:29.442323+0000","last_peered":"2024-08-17T17:39:29.442323+0000","last_clean":"2024-08-17T17:39:29.442323+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:39:29.442323+0000","last_undegraded":"2024-08-17T17:39:29.442323+0000","last_fullsized":"2024-08-17T17:39:29.442323+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"374'9657","reported_seq":10511,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:34.859061+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:39:34.859061+0000","last_peered":"2024-08-17T17:39:34.859061+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:39:34.859061+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'2616","ondisk_log_start":"325'2616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7041,"log_dups_size":0,"ondisk_log_size":7041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326959104,"num_objects":4989,"num_object_clones":0,"num_object_copies":14967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9657,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4989},{"shards":"0,5","objects":-1665}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":425,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.441910+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:39:29.441910+0000","last_peered":"2024-08-17T17:39:29.441910+0000","last_clean":"2024-08-17T17:39:29.441910+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:39:29.441910+0000","last_undegraded":"2024-08-17T17:39:29.441910+0000","last_fullsized":"2024-08-17T17:39:29.441910+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":349,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389392+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:39:18.389392+0000","last_peered":"2024-08-17T17:39:18.389392+0000","last_clean":"2024-08-17T17:39:18.389392+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:39:18.389392+0000","last_undegraded":"2024-08-17T17:39:18.389392+0000","last_fullsized":"2024-08-17T17:39:18.389392+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":220,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451260+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:39:29.451260+0000","last_peered":"2024-08-17T17:39:29.451260+0000","last_clean":"2024-08-17T17:39:29.451260+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:39:29.451260+0000","last_undegraded":"2024-08-17T17:39:29.451260+0000","last_fullsized":"2024-08-17T17:39:29.451260+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"374'9826","reported_seq":24799,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:35.236663+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:39:35.236663+0000","last_peered":"2024-08-17T17:39:35.236663+0000","last_clean":"2024-08-17T17:39:35.236663+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:39:35.236663+0000","last_undegraded":"2024-08-17T17:39:35.236663+0000","last_fullsized":"2024-08-17T17:39:35.236663+0000","mapping_epoch":355,"log_start":"325'3530","ondisk_log_start":"325'3530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328335360,"num_objects":5010,"num_object_clones":0,"num_object_copies":15030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9826,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":288,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374273+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:39:18.374273+0000","last_peered":"2024-08-17T17:39:18.374273+0000","last_clean":"2024-08-17T17:39:18.374273+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:39:18.374273+0000","last_undegraded":"2024-08-17T17:39:18.374273+0000","last_fullsized":"2024-08-17T17:39:18.374273+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451138+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:39:29.451138+0000","last_peered":"2024-08-17T17:39:29.451138+0000","last_clean":"2024-08-17T17:39:29.451138+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:39:29.451138+0000","last_undegraded":"2024-08-17T17:39:29.451138+0000","last_fullsized":"2024-08-17T17:39:29.451138+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":584,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.442147+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:39:29.442147+0000","last_peered":"2024-08-17T17:39:29.442147+0000","last_clean":"2024-08-17T17:39:29.442147+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:39:29.442147+0000","last_undegraded":"2024-08-17T17:39:29.442147+0000","last_fullsized":"2024-08-17T17:39:29.442147+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"374'9525","reported_seq":24066,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:35.333030+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:39:35.333030+0000","last_peered":"2024-08-17T17:39:35.333030+0000","last_clean":"2024-08-17T17:39:35.333030+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:39:35.333030+0000","last_undegraded":"2024-08-17T17:39:35.333030+0000","last_fullsized":"2024-08-17T17:39:35.333030+0000","mapping_epoch":345,"log_start":"325'3272","ondisk_log_start":"325'3272","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319750144,"num_objects":4879,"num_object_clones":0,"num_object_copies":14637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9525,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":284,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374278+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:39:18.374278+0000","last_peered":"2024-08-17T17:39:18.374278+0000","last_clean":"2024-08-17T17:39:18.374278+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:39:18.374278+0000","last_undegraded":"2024-08-17T17:39:18.374278+0000","last_fullsized":"2024-08-17T17:39:18.374278+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":339,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452517+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:39:29.452517+0000","last_peered":"2024-08-17T17:39:29.452517+0000","last_clean":"2024-08-17T17:39:29.452517+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:39:29.452517+0000","last_undegraded":"2024-08-17T17:39:29.452517+0000","last_fullsized":"2024-08-17T17:39:29.452517+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":426,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374119+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:39:18.374119+0000","last_peered":"2024-08-17T17:39:18.374119+0000","last_clean":"2024-08-17T17:39:18.374119+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:39:18.374119+0000","last_undegraded":"2024-08-17T17:39:18.374119+0000","last_fullsized":"2024-08-17T17:39:18.374119+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"374'9785","reported_seq":9934,"reported_epoch":374,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:35.408145+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:39:35.408145+0000","last_peered":"2024-08-17T17:39:35.408145+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:39:35.408145+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"325'3487","ondisk_log_start":"325'3487","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6298,"log_dups_size":0,"ondisk_log_size":6298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327614464,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9785,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4999},{"shards":"0,3","objects":-2249}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":614,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452353+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:39:29.452353+0000","last_peered":"2024-08-17T17:39:29.452353+0000","last_clean":"2024-08-17T17:39:29.452353+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:39:29.452353+0000","last_undegraded":"2024-08-17T17:39:29.452353+0000","last_fullsized":"2024-08-17T17:39:29.452353+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":582,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.442567+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:39:29.442567+0000","last_peered":"2024-08-17T17:39:29.442567+0000","last_clean":"2024-08-17T17:39:29.442567+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:39:29.442567+0000","last_undegraded":"2024-08-17T17:39:29.442567+0000","last_fullsized":"2024-08-17T17:39:29.442567+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":600,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451646+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:39:29.451646+0000","last_peered":"2024-08-17T17:39:29.451646+0000","last_clean":"2024-08-17T17:39:29.451646+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:39:29.451646+0000","last_undegraded":"2024-08-17T17:39:29.451646+0000","last_fullsized":"2024-08-17T17:39:29.451646+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"374'9652","reported_seq":24323,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:31.526416+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:39:31.526416+0000","last_peered":"2024-08-17T17:39:31.526416+0000","last_clean":"2024-08-17T17:39:31.526416+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:39:31.526416+0000","last_undegraded":"2024-08-17T17:39:31.526416+0000","last_fullsized":"2024-08-17T17:39:31.526416+0000","mapping_epoch":347,"log_start":"325'3072","ondisk_log_start":"325'3072","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322568192,"num_objects":4922,"num_object_clones":0,"num_object_copies":14766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9652,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":272,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452119+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:39:29.452119+0000","last_peered":"2024-08-17T17:39:29.452119+0000","last_clean":"2024-08-17T17:39:29.452119+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:39:29.452119+0000","last_undegraded":"2024-08-17T17:39:29.452119+0000","last_fullsized":"2024-08-17T17:39:29.452119+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":378,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471474+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:39:18.471474+0000","last_peered":"2024-08-17T17:39:18.471474+0000","last_clean":"2024-08-17T17:39:18.471474+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:39:18.471474+0000","last_undegraded":"2024-08-17T17:39:18.471474+0000","last_fullsized":"2024-08-17T17:39:18.471474+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":328,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.388983+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:39:18.388983+0000","last_peered":"2024-08-17T17:39:18.388983+0000","last_clean":"2024-08-17T17:39:18.388983+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:39:18.388983+0000","last_undegraded":"2024-08-17T17:39:18.388983+0000","last_fullsized":"2024-08-17T17:39:18.388983+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":328,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389238+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:39:18.389238+0000","last_peered":"2024-08-17T17:39:18.389238+0000","last_clean":"2024-08-17T17:39:18.389238+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:39:18.389238+0000","last_undegraded":"2024-08-17T17:39:18.389238+0000","last_fullsized":"2024-08-17T17:39:18.389238+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":371,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389567+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:39:18.389567+0000","last_peered":"2024-08-17T17:39:18.389567+0000","last_clean":"2024-08-17T17:39:18.389567+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:39:18.389567+0000","last_undegraded":"2024-08-17T17:39:18.389567+0000","last_fullsized":"2024-08-17T17:39:18.389567+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":589,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.442004+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:39:29.442004+0000","last_peered":"2024-08-17T17:39:29.442004+0000","last_clean":"2024-08-17T17:39:29.442004+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:39:29.442004+0000","last_undegraded":"2024-08-17T17:39:29.442004+0000","last_fullsized":"2024-08-17T17:39:29.442004+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":129,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.453491+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:39:29.453491+0000","last_peered":"2024-08-17T17:39:29.453491+0000","last_clean":"2024-08-17T17:39:29.453491+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:39:29.453491+0000","last_undegraded":"2024-08-17T17:39:29.453491+0000","last_fullsized":"2024-08-17T17:39:29.453491+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":278,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374360+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:39:18.374360+0000","last_peered":"2024-08-17T17:39:18.374360+0000","last_clean":"2024-08-17T17:39:18.374360+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:39:18.374360+0000","last_undegraded":"2024-08-17T17:39:18.374360+0000","last_fullsized":"2024-08-17T17:39:18.374360+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":547,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451950+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:39:29.451950+0000","last_peered":"2024-08-17T17:39:29.451950+0000","last_clean":"2024-08-17T17:39:29.451950+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:39:29.451950+0000","last_undegraded":"2024-08-17T17:39:29.451950+0000","last_fullsized":"2024-08-17T17:39:29.451950+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":422,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.374587+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:39:18.374587+0000","last_peered":"2024-08-17T17:39:18.374587+0000","last_clean":"2024-08-17T17:39:18.374587+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:39:18.374587+0000","last_undegraded":"2024-08-17T17:39:18.374587+0000","last_fullsized":"2024-08-17T17:39:18.374587+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":541,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451027+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:39:29.451027+0000","last_peered":"2024-08-17T17:39:29.451027+0000","last_clean":"2024-08-17T17:39:29.451027+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:39:29.451027+0000","last_undegraded":"2024-08-17T17:39:29.451027+0000","last_fullsized":"2024-08-17T17:39:29.451027+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":345,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.471411+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:39:18.471411+0000","last_peered":"2024-08-17T17:39:18.471411+0000","last_clean":"2024-08-17T17:39:18.471411+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:39:18.471411+0000","last_undegraded":"2024-08-17T17:39:18.471411+0000","last_fullsized":"2024-08-17T17:39:18.471411+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":573,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451728+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:39:29.451728+0000","last_peered":"2024-08-17T17:39:29.451728+0000","last_clean":"2024-08-17T17:39:29.451728+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:39:29.451728+0000","last_undegraded":"2024-08-17T17:39:29.451728+0000","last_fullsized":"2024-08-17T17:39:29.451728+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":288,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452963+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:39:29.452963+0000","last_peered":"2024-08-17T17:39:29.452963+0000","last_clean":"2024-08-17T17:39:29.452963+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:39:29.452963+0000","last_undegraded":"2024-08-17T17:39:29.452963+0000","last_fullsized":"2024-08-17T17:39:29.452963+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":581,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.441978+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:39:29.441978+0000","last_peered":"2024-08-17T17:39:29.441978+0000","last_clean":"2024-08-17T17:39:29.441978+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:39:29.441978+0000","last_undegraded":"2024-08-17T17:39:29.441978+0000","last_fullsized":"2024-08-17T17:39:29.441978+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":540,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.453312+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:39:29.453312+0000","last_peered":"2024-08-17T17:39:29.453312+0000","last_clean":"2024-08-17T17:39:29.453312+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:39:29.453312+0000","last_undegraded":"2024-08-17T17:39:29.453312+0000","last_fullsized":"2024-08-17T17:39:29.453312+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":357,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389201+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:39:18.389201+0000","last_peered":"2024-08-17T17:39:18.389201+0000","last_clean":"2024-08-17T17:39:18.389201+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:39:18.389201+0000","last_undegraded":"2024-08-17T17:39:18.389201+0000","last_fullsized":"2024-08-17T17:39:18.389201+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":229,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452517+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:39:29.452517+0000","last_peered":"2024-08-17T17:39:29.452517+0000","last_clean":"2024-08-17T17:39:29.452517+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:39:29.452517+0000","last_undegraded":"2024-08-17T17:39:29.452517+0000","last_fullsized":"2024-08-17T17:39:29.452517+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":285,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452163+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:39:29.452163+0000","last_peered":"2024-08-17T17:39:29.452163+0000","last_clean":"2024-08-17T17:39:29.452163+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:39:29.452163+0000","last_undegraded":"2024-08-17T17:39:29.452163+0000","last_fullsized":"2024-08-17T17:39:29.452163+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":365,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389145+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:39:18.389145+0000","last_peered":"2024-08-17T17:39:18.389145+0000","last_clean":"2024-08-17T17:39:18.389145+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:39:18.389145+0000","last_undegraded":"2024-08-17T17:39:18.389145+0000","last_fullsized":"2024-08-17T17:39:18.389145+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":228,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389495+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:39:18.389495+0000","last_peered":"2024-08-17T17:39:18.389495+0000","last_clean":"2024-08-17T17:39:18.389495+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:39:18.389495+0000","last_undegraded":"2024-08-17T17:39:18.389495+0000","last_fullsized":"2024-08-17T17:39:18.389495+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":365,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389077+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:39:18.389077+0000","last_peered":"2024-08-17T17:39:18.389077+0000","last_clean":"2024-08-17T17:39:18.389077+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:39:18.389077+0000","last_undegraded":"2024-08-17T17:39:18.389077+0000","last_fullsized":"2024-08-17T17:39:18.389077+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":228,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.389586+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:39:18.389586+0000","last_peered":"2024-08-17T17:39:18.389586+0000","last_clean":"2024-08-17T17:39:18.389586+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:39:18.389586+0000","last_undegraded":"2024-08-17T17:39:18.389586+0000","last_fullsized":"2024-08-17T17:39:18.389586+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":593,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.452263+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:39:29.452263+0000","last_peered":"2024-08-17T17:39:29.452263+0000","last_clean":"2024-08-17T17:39:29.452263+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:39:29.452263+0000","last_undegraded":"2024-08-17T17:39:29.452263+0000","last_fullsized":"2024-08-17T17:39:29.452263+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":566,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.453103+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:39:29.453103+0000","last_peered":"2024-08-17T17:39:29.453103+0000","last_clean":"2024-08-17T17:39:29.453103+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:39:29.453103+0000","last_undegraded":"2024-08-17T17:39:29.453103+0000","last_fullsized":"2024-08-17T17:39:29.453103+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":328,"reported_epoch":373,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.388819+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:39:18.388819+0000","last_peered":"2024-08-17T17:39:18.388819+0000","last_clean":"2024-08-17T17:39:18.388819+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:39:18.388819+0000","last_undegraded":"2024-08-17T17:39:18.388819+0000","last_fullsized":"2024-08-17T17:39:18.388819+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":288,"reported_epoch":374,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.451806+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:39:29.451806+0000","last_peered":"2024-08-17T17:39:29.451806+0000","last_clean":"2024-08-17T17:39:29.451806+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:39:29.451806+0000","last_undegraded":"2024-08-17T17:39:29.451806+0000","last_fullsized":"2024-08-17T17:39:29.451806+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5275713560,"num_objects":80502,"num_object_clones":0,"num_object_copies":241506,"num_objects_missing_on_primary":5696,"num_objects_missing":5696,"num_objects_degraded":54493,"num_objects_misplaced":31813,"num_objects_unfound":0,"num_objects_dirty":80502,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155888,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27185,"num_bytes_recovered":1659043840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17146064896,"data_stored":17146052680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104507,"ondisk_log_size":104507,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170078,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463447,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3152952,"kb_used_data":2259248,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90592200,"statfs":{"total":95995035648,"available":92766412800,"internally_reserved":0,"allocated":2313469952,"data_stored":2311455131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492122,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1039772,"kb_used_data":648596,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92705380,"statfs":{"total":95995035648,"available":94930309120,"internally_reserved":0,"allocated":664162304,"data_stored":662169783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986713,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2466668,"kb_used_data":1285668,"kb_used_omap":1,"kb_used_meta":1180990,"kb_avail":91278484,"statfs":{"total":95995035648,"available":93469167616,"internally_reserved":0,"allocated":1316524032,"data_stored":1314520151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1209334104},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,0,0,0,2,3,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225541,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3480860,"kb_used_data":2565140,"kb_used_omap":4,"kb_used_meta":915707,"kb_avail":90264292,"statfs":{"total":95995035648,"available":92430635008,"internally_reserved":0,"allocated":2626703360,"data_stored":2624711137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":937684416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075866,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533759,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3298592,"kb_used_data":2260760,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90446560,"statfs":{"total":95995035648,"available":92617277440,"internally_reserved":0,"allocated":2315018240,"data_stored":2313020695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089736,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2946040,"kb_used_data":1940976,"kb_used_omap":5,"kb_used_meta":1005050,"kb_avail":90799112,"statfs":{"total":95995035648,"available":92978290688,"internally_reserved":0,"allocated":1987559424,"data_stored":1985544662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1029171259},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1979056128,"data_stored":1979056128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2309033984,"data_stored":2309029912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3428519936,"data_stored":3428515864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2617769984,"data_stored":2617769984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1310130176,"data_stored":1310130176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":659095552,"data_stored":659095552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2306408448,"data_stored":2306408448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2536050688,"data_stored":2536046616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:39.361 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:39:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.362 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:39.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:39.401 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:39.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:39.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:39.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:39.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:39.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:39.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:39.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:39.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:39.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:39.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:40.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:40.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:40.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:40.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:40.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:40.334 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:40.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:40.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:40.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:39:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:39:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:40.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.603 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:40.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:40.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:40.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:40.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:40.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:40.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:40.973 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:41.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:41.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:41.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:41.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:41.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:41.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:41.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:41.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:41.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:41.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:41.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:41.615 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:41 smithi142 ceph-mon[106729]: pgmap v1387: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 54414/239877 objects degraded (22.684%); 31531/239877 objects misplaced (13.145%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:41.615 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54414/239877 objects degraded (22.684%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:41.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:41 smithi142 ceph-mon[103614]: pgmap v1387: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 54414/239877 objects degraded (22.684%); 31531/239877 objects misplaced (13.145%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:41.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54414/239877 objects degraded (22.684%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:41.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:41.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:41.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:41.816 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:41 smithi152 ceph-mon[93456]: pgmap v1387: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 54414/239877 objects degraded (22.684%); 31531/239877 objects misplaced (13.145%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:39:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54414/239877 objects degraded (22.684%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:41.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:41.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:39:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:39:41] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:39:41.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:41.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:42.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:42.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:42.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:42.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:42.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:42.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:42.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:42.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:42.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3444872017' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:42.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:42 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:42.375 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:39:42.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:42.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:42.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:42.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:42.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3444872017' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:42 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:42.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:42.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:42.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3444872017' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:42 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:42.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:42.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:42.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:42.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:43.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:43.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:43.211 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:43.305 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:43.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.426 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[106729]: pgmap v1388: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 54414/238986 objects degraded (22.769%); 31494/238986 objects misplaced (13.178%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:39:43.426 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:43.426 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:43.426 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[106729]: osdmap e375: 8 total, 8 up, 6 in 2024-08-17T17:39:43.426 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[103614]: pgmap v1388: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 54414/238986 objects degraded (22.769%); 31494/238986 objects misplaced (13.178%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:39:43.427 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:43.427 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:43.427 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:43 smithi142 ceph-mon[103614]: osdmap e375: 8 total, 8 up, 6 in 2024-08-17T17:39:43.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:43.450 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:43.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:43.575 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:43.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:43.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:43 smithi152 ceph-mon[93456]: pgmap v1388: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 54414/238986 objects degraded (22.769%); 31494/238986 objects misplaced (13.178%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:39:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:43 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:43 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:43 smithi152 ceph-mon[93456]: osdmap e375: 8 total, 8 up, 6 in 2024-08-17T17:39:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:43.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:43.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:43.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:43.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:44.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:44.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:44.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:44.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:44.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:44.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:44.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:44.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:44.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:44.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:44.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:44.596 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:39:44.596 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:39:44.596 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:44.596 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:44.597 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:44.598 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:44.598 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:44.598 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:44.598 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:44.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3358759203' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:44.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3358759203' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:44.678 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:44.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.700 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:44.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:44.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3358759203' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:44.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:44.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:44.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:44.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236392 11.1 e314/283 11:80697271:::benchmark_data_smithi142_227468_object112973:head v 314'14844, mlcod=314'14841)", 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.700037+0000", 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 522.26135899799999, 2024-08-17T17:39:44.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26028967400000003, 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700037+0000", 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:44.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700037+0000", 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700038+0000", 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.268e-06 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700045+0000", 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700059+0000", 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3743e-05 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700067+0000", 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1669999999999999e-06 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:44.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700122+0000", 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4629999999999997e-05 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.700142+0000", 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0307000000000001e-05 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.705124+0000", 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0049820059999999998 2024-08-17T17:39:44.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960326+0000", 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25520210999999998 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236396 11.b e314/283 11:de9f86a9:::benchmark_data_smithi142_227468_object112977:head v 314'14927, mlcod=314'14925)", 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.701912+0000", 2024-08-17T17:39:44.996 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 522.25948351099998, 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25917337200000001, 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701912+0000", 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:44.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701912+0000", 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701913+0000", 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0499999999999999e-06 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701920+0000", 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0099999999999998e-06 2024-08-17T17:39:44.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701932+0000", 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.242e-05 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701940+0000", 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7889999999999995e-06 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:44.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.701996+0000", 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5615999999999997e-05 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.702015+0000", 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9304999999999999e-05 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960459+0000", 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25844405799999998 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.961085+0000", 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00062612400000000004 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236400 11.b e314/283 11:d7d22f27:::benchmark_data_smithi142_227468_object112981:head v 314'14928, mlcod=314'14925)", 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.707407+0000", 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 522.25398884599997, 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26579524300000001, 2024-08-17T17:39:45.001 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707407+0000", 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707407+0000", 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707408+0000", 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.519e-06 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.707416+0000", 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6890000000000004e-06 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960286+0000", 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25287000500000001 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960303+0000", 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6773e-05 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.961120+0000", 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000817061 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.961134+0000", 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4035e-05 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973137+0000", 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012003315000000001 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973202+0000", 2024-08-17T17:39:45.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4845999999999995e-05 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.36603.0:236411 11.1 e314/283 11:86e88955:::benchmark_data_smithi142_227468_object112992:head v 314'14845, mlcod=314'14842)", 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:31:02.716916+0000", 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 522.24447943300004, 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.25703896100000001, 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.006 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716916+0000", 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716916+0000", 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716917+0000", 2024-08-17T17:39:45.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.373e-06 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.716929+0000", 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1445e-05 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960320+0000", 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.243391145 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960329+0000", 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0110000000000006e-06 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960521+0000", 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019235199999999999 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.960546+0000", 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4553999999999999e-05 2024-08-17T17:39:45.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973867+0000", 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013321058 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:31:02.973955+0000", 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8022999999999994e-05 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.010 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:123775 12.c 12:321760c7:::benchmark_data_smithi142_261410_object5578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:28.712782+0000", 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 376.24861317099999, 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19839264200000001, 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.011 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 123775 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712782+0000", 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712782+0000", 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712784+0000", 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.968e-06 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712792+0000", 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1310000000000006e-06 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712813+0000", 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0407999999999998e-05 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.712824+0000", 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0662000000000001e-05 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.725136+0000", 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012312498 2024-08-17T17:39:45.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.015 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.015 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.725258+0000", 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012247700000000001 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.814574+0000", 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.089315806999999997 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.814682+0000", 2024-08-17T17:39:45.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010725 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.814822+0000", 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0001404 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.820109+0000", 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052871250000000002 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.850287+0000", 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030177679999999998 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.906624+0000", 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.056337320000000003 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.906667+0000", 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2571999999999998e-05 2024-08-17T17:39:45.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.911175+0000", 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045083440000000001 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:123795 12.c 12:340d23f4:::benchmark_data_smithi142_261410_object5598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:28.752468+0000", 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 376.20892713900002, 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.150072178, 2024-08-17T17:39:45.019 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 123795 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752468+0000", 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752468+0000", 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752471+0000", 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6000000000000001e-06 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752479+0000", 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2390000000000002e-06 2024-08-17T17:39:45.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752502+0000", 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3040999999999998e-05 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.752512+0000", 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4609999999999997e-06 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.768199+0000", 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015687741000000002 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.768370+0000", 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000170428 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.768528+0000", 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015771700000000001 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.820044+0000", 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.051516861999999997 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.820077+0000", 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2141000000000003e-05 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.897089+0000", 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.077012238999999996 2024-08-17T17:39:45.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.897143+0000", 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4277000000000002e-05 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:28.902541+0000", 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053974319999999998 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:125894 12.c 12:3c54b205:::benchmark_data_smithi142_261410_object7697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:33.095034+0000", 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 371.86636165099998, 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.137142285, 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 125894 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.026 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095034+0000", 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095034+0000", 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095036+0000", 2024-08-17T17:39:45.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.796e-06 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095041+0000", 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.711e-06 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095056+0000", 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4293e-05 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.095064+0000", 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5709999999999998e-06 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.103028+0000", 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079640660000000005 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.103167+0000", 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013898799999999999 2024-08-17T17:39:45.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.174613+0000", 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.071445896999999994 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.174706+0000", 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2662000000000002e-05 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:45.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.174916+0000", 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000210476 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.203685+0000", 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.028769045 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.203942+0000", 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025621599999999997 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.228054+0000", 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024112858000000001 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.228154+0000", 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9561000000000004e-05 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:33.232176+0000", 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040221450000000004 2024-08-17T17:39:45.032 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:129215 12.c 12:35a663db:::benchmark_data_smithi142_261410_object11018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:39.247687+0000", 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 365.713708156, 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.14726345299999999, 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.033 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 129215 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247687+0000", 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247687+0000", 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247690+0000", 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4480000000000001e-06 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247698+0000", 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9279999999999993e-06 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247721+0000", 2024-08-17T17:39:45.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3683e-05 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.247731+0000", 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8519999999999999e-06 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.266425+0000", 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018693624999999998 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:45.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.266559+0000", 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013370499999999999 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.344578+0000", 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078019644999999999 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.344892+0000", 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00031342499999999999 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.345349+0000", 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00045739100000000003 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.357632+0000", 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01228248 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.388347+0000", 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030715884999999998 2024-08-17T17:39:45.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.388424+0000", 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6260000000000005e-05 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.388481+0000", 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7651000000000001e-05 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:39.394951+0000", 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064694749999999997 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:135776 12.e 12:732b4aff:::benchmark_data_smithi142_261410_object17579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:35:58.793819+0000", 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 226.16757606499999, 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027107873000000001, 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 135776 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.793819+0000", 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.793819+0000", 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.793820+0000", 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.006e-06 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.793825+0000", 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6990000000000002e-06 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.793835+0000", 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3510000000000008e-06 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.793841+0000", 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8859999999999998e-06 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.797192+0000", 2024-08-17T17:39:45.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033509 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.797534+0000", 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000341768 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.797698+0000", 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000163719 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.801643+0000", 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039454879999999996 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.801663+0000", 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9678999999999999e-05 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.815796+0000", 2024-08-17T17:39:45.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014132710999999999 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.815863+0000", 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7410999999999995e-05 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:58.820927+0000", 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0050642550000000001 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:140566 12.c 12:3de8a603:::benchmark_data_smithi142_261410_object22369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:57.672397+0000", 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 167.288998773, 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.028627118999999999, 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 140566 2024-08-17T17:39:45.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672397+0000", 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672397+0000", 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672399+0000", 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1320000000000001e-06 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672410+0000", 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0645000000000001e-05 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672429+0000", 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9612000000000001e-05 2024-08-17T17:39:45.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672440+0000", 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1122e-05 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672514+0000", 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3365e-05 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672690+0000", 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017643100000000001 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.672871+0000", 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018074500000000001 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.693773+0000", 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.020902303000000001 2024-08-17T17:39:45.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.693816+0000", 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3389000000000001e-05 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.696507+0000", 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026907979999999999 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.696576+0000", 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8987999999999997e-05 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:57.701024+0000", 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044475890000000001 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146751 12.c 12:3397af55:::benchmark_data_smithi142_261410_object28554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.871835+0000", 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.089560844, 2024-08-17T17:39:45.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027802812999999999, 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146751 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.871835+0000", 2024-08-17T17:39:45.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.871835+0000", 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.871836+0000", 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8059999999999999e-06 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.871840+0000", 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9990000000000002e-06 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.871850+0000", 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7070000000000004e-06 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.871857+0000", 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1790000000000002e-06 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.876981+0000", 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0051232999999999999 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.877098+0000", 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117095 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.877202+0000", 2024-08-17T17:39:45.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010384 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.883663+0000", 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064615669999999997 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884249+0000", 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058576199999999998 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.895288+0000", 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01103927 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.895325+0000", 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6327e-05 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.899637+0000", 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043129609999999997 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.059 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146754 12.c 12:3f152483:::benchmark_data_smithi142_261410_object28557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.875872+0000", 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.08552333900001, 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.029187397, 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.060 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146754 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.875872+0000", 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.875872+0000", 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.875873+0000", 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.08e-06 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.875883+0000", 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7440000000000002e-06 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.875896+0000", 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3127e-05 2024-08-17T17:39:45.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.875904+0000", 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9910000000000002e-06 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.880033+0000", 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.004128662 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.880196+0000", 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000162771 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.880360+0000", 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016455500000000001 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.883811+0000", 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034506559999999999 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.886939+0000", 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031283090000000001 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.899713+0000", 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012774120999999999 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.065 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.899757+0000", 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3591000000000003e-05 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.905060+0000", 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0053027899999999999 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146757 12.c 12:33e279d8:::benchmark_data_smithi142_261410_object28560:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:39:45.066 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.884241+0000", 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 102.07715451999999, 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027072565999999999, 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146757 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884241+0000", 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884241+0000", 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884242+0000", 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.28e-06 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884246+0000", 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8809999999999998e-06 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884256+0000", 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0019e-05 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884263+0000", 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6900000000000003e-06 2024-08-17T17:39:45.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884307+0000", 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3643999999999998e-05 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884410+0000", 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103223 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:39:45.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.884823+0000", 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000412769 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.887082+0000", 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0022596579999999999 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.888949+0000", 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0018668739999999999 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.905433+0000", 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016483898 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.905499+0000", 2024-08-17T17:39:45.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5587999999999996e-05 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.911314+0000", 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0058150420000000003 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:147242 12.3 12:ce77bc63:::benchmark_data_smithi142_261410_object29045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:07.391467+0000", 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 97.569928763999997, 2024-08-17T17:39:45.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024831003000000001, 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 147242 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.391467+0000", 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.391467+0000", 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.391468+0000", 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1540000000000001e-06 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.391472+0000", 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4240000000000003e-06 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.391483+0000", 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0975000000000001e-05 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.391488+0000", 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4859999999999999e-06 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.395722+0000", 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042342669999999999 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.395858+0000", 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013565199999999999 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.395973+0000", 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115017 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.398950+0000", 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029776469999999999 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.399236+0000", 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000285858 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.413064+0000", 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013827973 2024-08-17T17:39:45.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.413142+0000", 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8071999999999994e-05 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.416298+0000", 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031554780000000002 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:147284 12.3 12:c242c3a8:::benchmark_data_smithi142_261410_object29087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:08.125648+0000", 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 96.835747726999998, 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025315126, 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 147284 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.125648+0000", 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.125648+0000", 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.125649+0000", 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5200000000000001e-06 2024-08-17T17:39:45.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.125653+0000", 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6990000000000001e-06 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.125662+0000", 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6629999999999999e-06 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.125667+0000", 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9250000000000003e-06 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.128792+0000", 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031257920000000001 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.128910+0000", 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117798 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.129012+0000", 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010199700000000001 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.131818+0000", 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028056349999999999 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.132299+0000", 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00048158199999999998 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.147466+0000", 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015166176 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.147504+0000", 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8266000000000003e-05 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:08.150963+0000", 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003459073 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.085 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:149992 12.e 12:7b0d7c66:::benchmark_data_smithi142_261410_object31795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:34.461202+0000", 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 70.500193530000004, 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030123817000000001, 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.086 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 149992 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461202+0000", 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461202+0000", 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461204+0000", 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4430000000000002e-06 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461214+0000", 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3270000000000007e-06 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461235+0000", 2024-08-17T17:39:45.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1169999999999999e-05 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461245+0000", 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6199999999999994e-06 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461316+0000", 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1415000000000007e-05 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op must wait for map", 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.461325+0000", 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6340000000000007e-06 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.477633+0000", 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016308644000000001 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.477842+0000", 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020923900000000001 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.478051+0000", 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000208786 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.484248+0000", 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0061964000000000003 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.484272+0000", 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4524000000000001e-05 2024-08-17T17:39:45.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.487056+0000", 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027837949999999999 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.487099+0000", 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3393000000000003e-05 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:34.491326+0000", 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042264269999999996 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.093 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:154855 12.3 12:c1cb9528:::benchmark_data_smithi142_261410_object36658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-17T17:39:45.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:24.686379+0000", 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 20.275016691000001, 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.026114482000000001, 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 154855 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.095 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.686379+0000", 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.686379+0000", 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.686380+0000", 2024-08-17T17:39:45.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4500000000000001e-06 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.686385+0000", 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.169e-06 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.686398+0000", 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2777e-05 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.686406+0000", 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5410000000000003e-06 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.690514+0000", 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041086309999999997 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.690629+0000", 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114543 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.691143+0000", 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00051372899999999999 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.696013+0000", 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048705520000000002 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.699233+0000", 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032195330000000001 2024-08-17T17:39:45.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.708851+0000", 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0096180899999999993 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.708910+0000", 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9147999999999997e-05 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:24.712493+0000", 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035833190000000002 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:155105 12.c 12:36eae967:::benchmark_data_smithi142_261410_object36908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:27.035465+0000", 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 17.925930689000001, 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027433214000000001, 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.101 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 155105 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035465+0000", 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035465+0000", 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035466+0000", 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4470000000000001e-06 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035472+0000", 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4280000000000004e-06 2024-08-17T17:39:45.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035484+0000", 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1844e-05 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035489+0000", 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7169999999999996e-06 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035549+0000", 2024-08-17T17:39:45.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0207999999999997e-05 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035713+0000", 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016370100000000001 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.035913+0000", 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000199423 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.044599+0000", 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.008686203 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.044643+0000", 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4678000000000002e-05 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.058357+0000", 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013713695999999999 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.058416+0000", 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8869000000000002e-05 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.062898+0000", 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044819999999999999 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:155136 12.3 12:c0ff2365:::benchmark_data_smithi142_261410_object36939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:27.111873+0000", 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 17.849522286999999, 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025216432, 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 155136 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.111873+0000", 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.111873+0000", 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.111874+0000", 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8199999999999999e-07 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.111882+0000", 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.729e-06 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.111890+0000", 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5490000000000006e-06 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.111896+0000", 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9730000000000002e-06 2024-08-17T17:39:45.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.119167+0000", 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072710099999999996 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.119353+0000", 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018539899999999999 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.119534+0000", 2024-08-17T17:39:45.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018174900000000001 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.123430+0000", 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038961030000000002 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.128223+0000", 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0047926720000000004 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.132435+0000", 2024-08-17T17:39:45.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042115299999999998 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.132498+0000", 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3212000000000004e-05 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:27.137090+0000", 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045918240000000004 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:155707 12.c 12:3e4d574c:::benchmark_data_smithi142_261410_object37510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:32.235347+0000", 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 12.726048619, 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024297261000000001, 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 155707 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.114 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.235347+0000", 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.235347+0000", 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.235348+0000", 2024-08-17T17:39:45.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7700000000000004e-07 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.235351+0000", 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.348e-06 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.235360+0000", 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7830000000000004e-06 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.235366+0000", 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1020000000000002e-06 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.238451+0000", 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030852729999999999 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.238572+0000", 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012095300000000001 2024-08-17T17:39:45.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.238681+0000", 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010944500000000001 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.241549+0000", 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028671539999999998 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.241866+0000", 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00031786299999999999 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.255677+0000", 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013810471 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.255721+0000", 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4020000000000002e-05 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:45.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.259644+0000", 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039231719999999999 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:45.120 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:39:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:45.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:45.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:39:45.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:45.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:45.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:45.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:45.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:45.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:45.507 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:45.558 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:45 smithi142 ceph-mon[106729]: pgmap v1390: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 54346/238539 objects degraded (22.783%); 31345/238539 objects misplaced (13.140%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:39:45.558 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:45 smithi142 ceph-mon[103614]: pgmap v1390: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 54346/238539 objects degraded (22.783%); 31345/238539 objects misplaced (13.140%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:39:45.609 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:45.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:45.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:45.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:45 smithi152 ceph-mon[93456]: pgmap v1390: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 54346/238539 objects degraded (22.783%); 31345/238539 objects misplaced (13.140%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:39:45.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:45.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:45.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:46.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:46.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:39:46.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:46.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:46.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:46.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:46.274 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:46.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:46.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:46.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:46.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:46.522 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:46.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:46.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:46.660 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54346/238539 objects degraded (22.783%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:46.660 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54346/238539 objects degraded (22.783%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:46.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:46.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:46.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54346/238539 objects degraded (22.783%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:46.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:46.901 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:47.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:47.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:47.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.152 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:47.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:47.155 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:47.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:47.275 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:47.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:47.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:47.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:47.443 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:47.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:47.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:47.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:47.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.648 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:47.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:47.699 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:47 smithi142 ceph-mon[106729]: pgmap v1391: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54346/238128 objects degraded (22.822%); 31345/238128 objects misplaced (13.163%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:39:47.699 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:47 smithi142 ceph-mon[103614]: pgmap v1391: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54346/238128 objects degraded (22.822%); 31345/238128 objects misplaced (13.163%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:39:47.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:47.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:47 smithi152 ceph-mon[93456]: pgmap v1391: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 54346/238128 objects degraded (22.822%); 31345/238128 objects misplaced (13.163%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:39:47.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:47.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:47.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:48.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:48.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:48.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:48.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:48.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:48.272 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:48.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:48.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:48.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:48.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:48.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:48.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:48.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:48.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:48.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:48.662 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:48.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:48.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:48.869 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:48.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:48.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:48.994 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:49.016 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:49.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:49.142 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:49.142 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 54346/237705 objects degraded (22.863%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2069,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":375,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":79235,"data_bytes":5191571429,"bytes_used":16624222208,"bytes_avail":559345991680,"bytes_total":575970213888,"degraded_objects":54346,"degraded_total":237705,"degraded_ratio":0.2286279211627858,"misplaced_objects":31318,"misplaced_total":237705,"misplaced_ratio":0.13175154077533077,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":923803,"recovering_keys_per_sec":0,"num_objects_recovered":141,"num_bytes_recovered":9240576,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":94},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:39:49.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:49.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:49.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:49.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:49.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:49.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:49.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:49.420 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:49 smithi142 ceph-mon[106729]: pgmap v1392: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54346/237705 objects degraded (22.863%); 31318/237705 objects misplaced (13.175%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:39:49.420 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/737570285' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:49.421 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:49 smithi142 ceph-mon[103614]: pgmap v1392: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54346/237705 objects degraded (22.863%); 31318/237705 objects misplaced (13.175%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:39:49.421 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/737570285' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:49.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:49.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:49.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:49.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:49.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:49.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:49.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:49 smithi152 ceph-mon[93456]: pgmap v1392: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 54346/237705 objects degraded (22.863%); 31318/237705 objects misplaced (13.175%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:39:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/737570285' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:49.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:49.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:49.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:50.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:50.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:50.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:50.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:50.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:50.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:50.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:50.450 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:39:50.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:50.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:50.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:50.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:39:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:39:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:50.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:50.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:50.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:50.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:50.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:50.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:50.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:50.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:50.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:51.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:51.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:51.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:51.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:51.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:51.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:51.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:51.389 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:39:51.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:51.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:51.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:51.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-mon[103614]: pgmap v1393: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 54284/236901 objects degraded (22.914%); 31155/236901 objects misplaced (13.151%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54284/236901 objects degraded (22.914%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4041543685' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-mon[106729]: pgmap v1393: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 54284/236901 objects degraded (22.914%); 31155/236901 objects misplaced (13.151%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54284/236901 objects degraded (22.914%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:51.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4041543685' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:51.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:51.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:51.812 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:51 smithi152 ceph-mon[93456]: pgmap v1393: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 54284/236901 objects degraded (22.914%); 31155/236901 objects misplaced (13.151%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54284/236901 objects degraded (22.914%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4041543685' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:51.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:51.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:51.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:52.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:52.074 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:39:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:39:51] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:39:52.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:52.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:52.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:52.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:52.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:52.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:52.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:52.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:52.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:52.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:52.562 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:52.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:52.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:52.634 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:52.656 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:52 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:52.656 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4041543685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:52.656 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:52 smithi142 ceph-mon[106729]: osdmap e376: 8 total, 8 up, 6 in 2024-08-17T17:39:52.656 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:52 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:52.656 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4041543685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:52.656 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:52 smithi142 ceph-mon[103614]: osdmap e376: 8 total, 8 up, 6 in 2024-08-17T17:39:52.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:52.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:52.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:52 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4041543685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:52 smithi152 ceph-mon[93456]: osdmap e376: 8 total, 8 up, 6 in 2024-08-17T17:39:52.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:52.853 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:52.867 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:39:52.867 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:39:52.867 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.867 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:158031 12.6 12:6c7eacc6:::benchmark_data_smithi142_261410_object39834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e376)", 2024-08-17T17:39:52.867 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:52.862225+0000", 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0043489490000000004, 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043768289999999996, 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 158031 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.868 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862225+0000", 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862225+0000", 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862227+0000", 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.869 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862234+0000", 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2239999999999998e-06 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862252+0000", 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7910999999999998e-05 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:52.870 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862262+0000", 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4790000000000002e-06 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862340+0000", 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8879000000000002e-05 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862524+0000", 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018368200000000001 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.871 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6,7", 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.862690+0000", 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016631299999999999 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866052+0000", 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033619000000000001 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866181+0000", 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012819599999999999 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.872 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866200+0000", 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9287999999999999e-05 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866427+0000", 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022709999999999999 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866498+0000", 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1033999999999998e-05 2024-08-17T17:39:52.873 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:158031 12.6 e376/337 ondisk, result = 0)", 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:52.866347+0000", 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.000227653, 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00032457200000000002, 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:39:52.874 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866347+0000", 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866347+0000", 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:39:52.875 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866347+0000", 2024-08-17T17:39:52.876 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5600000000000005e-07 2024-08-17T17:39:52.876 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.876 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.876 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:39:52.876 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866352+0000", 2024-08-17T17:39:52.876 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6360000000000002e-06 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866362+0000", 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.825e-06 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866366+0000", 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5460000000000002e-06 2024-08-17T17:39:52.877 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866411+0000", 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4682999999999999e-05 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866421+0000", 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0226e-05 2024-08-17T17:39:52.878 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:52.866671+0000", 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025000000000000001 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 2 2024-08-17T17:39:52.879 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:39:52.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:52.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:52.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:53.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:53.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:53.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:53.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:53.211 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:53.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:53.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:53.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:53.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:53.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:53.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:53.490 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:53 smithi142 ceph-mon[106729]: pgmap v1395: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 54284/236016 objects degraded (23.000%); 31112/236016 objects misplaced (13.182%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:53.491 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:53 smithi142 ceph-mon[103614]: pgmap v1395: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 54284/236016 objects degraded (23.000%); 31112/236016 objects misplaced (13.182%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:53.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:53.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:53.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:53.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:53.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:53.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:53 smithi152 ceph-mon[93456]: pgmap v1395: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 54284/236016 objects degraded (23.000%); 31112/236016 objects misplaced (13.182%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:39:53.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:53.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:53.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:54.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:54.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:54.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:54.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:54.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:54.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:54.334 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:54.436 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:54.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:54.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:54.582 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:54.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:39:54.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:54.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:54.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:54.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:54.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:54.934 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:54.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:54.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:55.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:55.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:55.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:55.209 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:55.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:55.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:55.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:55.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:55.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:55.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:55.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:55 smithi142 ceph-mon[103614]: pgmap v1396: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 54218/235518 objects degraded (23.021%); 30946/235518 objects misplaced (13.140%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:55.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:55 smithi142 ceph-mon[106729]: pgmap v1396: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 54218/235518 objects degraded (23.021%); 30946/235518 objects misplaced (13.140%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:55.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:55.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:55.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:55 smithi152 ceph-mon[93456]: pgmap v1396: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 54218/235518 objects degraded (23.021%); 30946/235518 objects misplaced (13.140%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:55.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:55.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:55.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:55.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:56.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:56.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:56.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:56.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:56.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:39:56.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:56.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:56.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:56.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:56.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:56.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:56.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:56.600 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54218/235518 objects degraded (23.021%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:56.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54218/235518 objects degraded (23.021%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:56.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:56.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:56.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54218/235518 objects degraded (23.021%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:56.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:56.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:56.934 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:56.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:57.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:57.082 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:57.096 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:39:57.097 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:39:57.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:57.205 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:57.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:57.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:57.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:57.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:57.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:57.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:57.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:57.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:57.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:57.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:57.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:57.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:57 smithi152 ceph-mon[93456]: pgmap v1397: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 54218/234876 objects degraded (23.084%); 30946/234876 objects misplaced (13.175%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:57.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:57.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[106729]: pgmap v1397: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 54218/234876 objects degraded (23.084%); 30946/234876 objects misplaced (13.175%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[103614]: pgmap v1397: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 54218/234876 objects degraded (23.084%); 30946/234876 objects misplaced (13.175%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:39:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:57.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:57.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:58.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:58.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:58.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:58.209 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:58.225 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:58.249 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:58.259 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1397,"stamp":"2024-08-17T17:39:56.317945+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5129770981,"num_objects":78292,"num_object_clones":0,"num_object_copies":234876,"num_objects_missing_on_primary":5558,"num_objects_missing":5558,"num_objects_degraded":54218,"num_objects_misplaced":30946,"num_objects_unfound":0,"num_objects_dirty":78292,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":158402,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27625,"num_bytes_recovered":1687560846,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105050,"ondisk_log_size":105050,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16110636,"kb_used_data":10657012,"kb_used_omap":27,"kb_used_meta":5453540,"kb_avail":546360276,"statfs":{"total":575970213888,"available":559472922624,"internally_reserved":0,"allocated":10912780288,"data_stored":10900678812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5584425704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,1,3,3,0,0,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-71041024,"num_objects":-1084,"num_object_clones":0,"num_object_copies":-3252,"num_objects_missing_on_primary":-69,"num_objects_missing":-69,"num_objects_degraded":-128,"num_objects_misplaced":-399,"num_objects_unfound":0,"num_objects_dirty":-1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1084,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":182,"num_bytes_recovered":11927552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002513"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":594,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392123+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:39:51.392123+0000","last_peered":"2024-08-17T17:39:51.392123+0000","last_clean":"2024-08-17T17:39:51.392123+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:39:51.392123+0000","last_undegraded":"2024-08-17T17:39:51.392123+0000","last_fullsized":"2024-08-17T17:39:51.392123+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":85,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.455418+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:39:42.455418+0000","last_peered":"2024-08-17T17:39:42.455418+0000","last_clean":"2024-08-17T17:39:42.455418+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:39:42.455418+0000","last_undegraded":"2024-08-17T17:39:42.455418+0000","last_fullsized":"2024-08-17T17:39:42.455418+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":279,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374928+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:39:42.374928+0000","last_peered":"2024-08-17T17:39:42.374928+0000","last_clean":"2024-08-17T17:39:42.374928+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:39:42.374928+0000","last_undegraded":"2024-08-17T17:39:42.374928+0000","last_fullsized":"2024-08-17T17:39:42.374928+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":292,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393450+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:39:51.393450+0000","last_peered":"2024-08-17T17:39:51.393450+0000","last_clean":"2024-08-17T17:39:51.393450+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:39:51.393450+0000","last_undegraded":"2024-08-17T17:39:51.393450+0000","last_fullsized":"2024-08-17T17:39:51.393450+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":469,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393004+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:39:51.393004+0000","last_peered":"2024-08-17T17:39:51.393004+0000","last_clean":"2024-08-17T17:39:51.393004+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:39:51.393004+0000","last_undegraded":"2024-08-17T17:39:51.393004+0000","last_fullsized":"2024-08-17T17:39:51.393004+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":228,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454599+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:39:42.454599+0000","last_peered":"2024-08-17T17:39:42.454599+0000","last_clean":"2024-08-17T17:39:42.454599+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:39:42.454599+0000","last_undegraded":"2024-08-17T17:39:42.454599+0000","last_fullsized":"2024-08-17T17:39:42.454599+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":563,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.391821+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:39:51.391821+0000","last_peered":"2024-08-17T17:39:51.391821+0000","last_clean":"2024-08-17T17:39:51.391821+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:39:51.391821+0000","last_undegraded":"2024-08-17T17:39:51.391821+0000","last_fullsized":"2024-08-17T17:39:51.391821+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":396,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.379813+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:39:42.379813+0000","last_peered":"2024-08-17T17:39:42.379813+0000","last_clean":"2024-08-17T17:39:42.379813+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:39:42.379813+0000","last_undegraded":"2024-08-17T17:39:42.379813+0000","last_fullsized":"2024-08-17T17:39:42.379813+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"376'9836","reported_seq":13434,"reported_epoch":376,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:39:53.262292+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:39:53.262292+0000","last_peered":"2024-08-17T17:39:53.262292+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:39:53.262292+0000","last_undegraded":"2024-08-17T17:39:53.262292+0000","last_fullsized":"2024-08-17T17:39:53.262292+0000","mapping_epoch":337,"log_start":"325'2963","ondisk_log_start":"325'2963","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6873,"log_dups_size":0,"ondisk_log_size":6873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319029248,"num_objects":4868,"num_object_clones":0,"num_object_copies":14604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4868,"num_objects_unfound":0,"num_objects_dirty":4868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9836,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1755,"num_bytes_recovered":115015680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4868}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":81,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.478327+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:39:42.478327+0000","last_peered":"2024-08-17T17:39:42.478327+0000","last_clean":"2024-08-17T17:39:42.478327+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:39:42.478327+0000","last_undegraded":"2024-08-17T17:39:42.478327+0000","last_fullsized":"2024-08-17T17:39:42.478327+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":262,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374121+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:39:42.374121+0000","last_peered":"2024-08-17T17:39:42.374121+0000","last_clean":"2024-08-17T17:39:42.374121+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:39:42.374121+0000","last_undegraded":"2024-08-17T17:39:42.374121+0000","last_fullsized":"2024-08-17T17:39:42.374121+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"376'10090","reported_seq":10164,"reported_epoch":376,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:51.885587+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:39:51.885587+0000","last_peered":"2024-08-17T17:39:51.885587+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:39:51.885587+0000","last_undegraded":"2024-08-17T17:39:51.885587+0000","last_fullsized":"2024-08-17T17:39:51.885587+0000","mapping_epoch":337,"log_start":"325'3482","ondisk_log_start":"325'3482","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326631424,"num_objects":4984,"num_object_clones":0,"num_object_copies":14952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4984,"num_objects_unfound":0,"num_objects_dirty":4984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10090,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4984}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":83,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454857+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:39:42.454857+0000","last_peered":"2024-08-17T17:39:42.454857+0000","last_clean":"2024-08-17T17:39:42.454857+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:39:42.454857+0000","last_undegraded":"2024-08-17T17:39:42.454857+0000","last_fullsized":"2024-08-17T17:39:42.454857+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":347,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454523+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:39:42.454523+0000","last_peered":"2024-08-17T17:39:42.454523+0000","last_clean":"2024-08-17T17:39:42.454523+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:39:42.454523+0000","last_undegraded":"2024-08-17T17:39:42.454523+0000","last_fullsized":"2024-08-17T17:39:42.454523+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"376'9824","reported_seq":9901,"reported_epoch":376,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:53.293672+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:39:53.293672+0000","last_peered":"2024-08-17T17:39:53.293672+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:39:53.293672+0000","last_undegraded":"2024-08-17T17:39:53.293672+0000","last_fullsized":"2024-08-17T17:39:53.293672+0000","mapping_epoch":337,"log_start":"325'2953","ondisk_log_start":"325'2953","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6871,"log_dups_size":0,"ondisk_log_size":6871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321781760,"num_objects":4910,"num_object_clones":0,"num_object_copies":14730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4910,"num_objects_unfound":0,"num_objects_dirty":4910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9824,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4910}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":79,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374260+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:39:42.374260+0000","last_peered":"2024-08-17T17:39:42.374260+0000","last_clean":"2024-08-17T17:39:42.374260+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:39:42.374260+0000","last_undegraded":"2024-08-17T17:39:42.374260+0000","last_fullsized":"2024-08-17T17:39:42.374260+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":357,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.379701+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:39:42.379701+0000","last_peered":"2024-08-17T17:39:42.379701+0000","last_clean":"2024-08-17T17:39:42.379701+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:39:42.379701+0000","last_undegraded":"2024-08-17T17:39:42.379701+0000","last_fullsized":"2024-08-17T17:39:42.379701+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"376'10035","reported_seq":10166,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:54.812834+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:39:54.812834+0000","last_peered":"2024-08-17T17:39:54.812834+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:39:54.812834+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'3013","ondisk_log_start":"325'3013","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322633728,"num_objects":4923,"num_object_clones":0,"num_object_copies":14769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10035,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1846},{"shards":"0,3,7","objects":4923}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":506,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392281+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:39:51.392281+0000","last_peered":"2024-08-17T17:39:51.392281+0000","last_clean":"2024-08-17T17:39:51.392281+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:39:51.392281+0000","last_undegraded":"2024-08-17T17:39:51.392281+0000","last_fullsized":"2024-08-17T17:39:51.392281+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":272,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.745026+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:39:51.745026+0000","last_peered":"2024-08-17T17:39:51.745026+0000","last_clean":"2024-08-17T17:39:51.745026+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:39:51.745026+0000","last_undegraded":"2024-08-17T17:39:51.745026+0000","last_fullsized":"2024-08-17T17:39:51.745026+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"376'9751","reported_seq":9816,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.931270+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:39:51.931270+0000","last_peered":"2024-08-17T17:39:51.931270+0000","last_clean":"2024-08-17T17:39:51.931270+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:39:51.931270+0000","last_undegraded":"2024-08-17T17:39:51.931270+0000","last_fullsized":"2024-08-17T17:39:51.931270+0000","mapping_epoch":319,"log_start":"325'3174","ondisk_log_start":"325'3174","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317390848,"num_objects":4843,"num_object_clones":0,"num_object_copies":14529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9751,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":596,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393721+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:39:51.393721+0000","last_peered":"2024-08-17T17:39:51.393721+0000","last_clean":"2024-08-17T17:39:51.393721+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:39:51.393721+0000","last_undegraded":"2024-08-17T17:39:51.393721+0000","last_fullsized":"2024-08-17T17:39:51.393721+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":330,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454408+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:39:42.454408+0000","last_peered":"2024-08-17T17:39:42.454408+0000","last_clean":"2024-08-17T17:39:42.454408+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:39:42.454408+0000","last_undegraded":"2024-08-17T17:39:42.454408+0000","last_fullsized":"2024-08-17T17:39:42.454408+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"376'9909","reported_seq":9984,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:51.896097+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:39:51.896097+0000","last_peered":"2024-08-17T17:39:51.896097+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:39:51.896097+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'3335","ondisk_log_start":"325'3335","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329187328,"num_objects":5023,"num_object_clones":0,"num_object_copies":15069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5023},{"shards":"1,6","objects":-2343}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":331,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.379809+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:39:42.379809+0000","last_peered":"2024-08-17T17:39:42.379809+0000","last_clean":"2024-08-17T17:39:42.379809+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:39:42.379809+0000","last_undegraded":"2024-08-17T17:39:42.379809+0000","last_fullsized":"2024-08-17T17:39:42.379809+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":357,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454895+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:39:42.454895+0000","last_peered":"2024-08-17T17:39:42.454895+0000","last_clean":"2024-08-17T17:39:42.454895+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:39:42.454895+0000","last_undegraded":"2024-08-17T17:39:42.454895+0000","last_fullsized":"2024-08-17T17:39:42.454895+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"376'10063","reported_seq":10195,"reported_epoch":376,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:39:53.910411+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:39:53.910411+0000","last_peered":"2024-08-17T17:39:53.910411+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:39:53.910411+0000","last_undegraded":"2024-08-17T17:39:53.910411+0000","last_fullsized":"2024-08-17T17:39:53.910411+0000","mapping_epoch":337,"log_start":"326'4461","ondisk_log_start":"326'4461","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324206592,"num_objects":4947,"num_object_clones":0,"num_object_copies":14841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4947,"num_objects_unfound":0,"num_objects_dirty":4947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10063,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4947}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":368,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454743+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:39:42.454743+0000","last_peered":"2024-08-17T17:39:42.454743+0000","last_clean":"2024-08-17T17:39:42.454743+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:39:42.454743+0000","last_undegraded":"2024-08-17T17:39:42.454743+0000","last_fullsized":"2024-08-17T17:39:42.454743+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":330,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454992+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:39:42.454992+0000","last_peered":"2024-08-17T17:39:42.454992+0000","last_clean":"2024-08-17T17:39:42.454992+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:39:42.454992+0000","last_undegraded":"2024-08-17T17:39:42.454992+0000","last_fullsized":"2024-08-17T17:39:42.454992+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"376'9838","reported_seq":13495,"reported_epoch":376,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:53.390061+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:39:53.390061+0000","last_peered":"2024-08-17T17:39:53.390061+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:39:53.390061+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:39:53.390061+0000","mapping_epoch":337,"log_start":"325'2941","ondisk_log_start":"325'2941","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318898176,"num_objects":4866,"num_object_clones":0,"num_object_copies":14598,"num_objects_missing_on_primary":5558,"num_objects_missing":5558,"num_objects_degraded":4866,"num_objects_misplaced":5558,"num_objects_unfound":0,"num_objects_dirty":4866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9838,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1781,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":4852},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":565,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.394093+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:39:51.394093+0000","last_peered":"2024-08-17T17:39:51.394093+0000","last_clean":"2024-08-17T17:39:51.394093+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:39:51.394093+0000","last_undegraded":"2024-08-17T17:39:51.394093+0000","last_fullsized":"2024-08-17T17:39:51.394093+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":502,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.391926+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:39:51.391926+0000","last_peered":"2024-08-17T17:39:51.391926+0000","last_clean":"2024-08-17T17:39:51.391926+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:39:51.391926+0000","last_undegraded":"2024-08-17T17:39:51.391926+0000","last_fullsized":"2024-08-17T17:39:51.391926+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"376'9966","reported_seq":13625,"reported_epoch":376,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:39:53.967020+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:39:53.967020+0000","last_peered":"2024-08-17T17:39:53.967020+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:39:53.967020+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:39:53.967020+0000","mapping_epoch":337,"log_start":"326'4344","ondisk_log_start":"326'4344","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5622,"log_dups_size":0,"ondisk_log_size":5622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323026968,"num_objects":4930,"num_object_clones":0,"num_object_copies":14790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4930,"num_objects_misplaced":5679,"num_objects_unfound":0,"num_objects_dirty":4930,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9966,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1769,"num_bytes_recovered":115867648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4929},{"shards":"1,3,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":696,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:51.744560+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:39:51.744560+0000","last_peered":"2024-08-17T17:39:51.744560+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:39:51.744560+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":305,"reported_epoch":375,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:39:42.374536+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:39:42.374536+0000","last_peered":"2024-08-17T17:39:42.374536+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:39:42.374536+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:39:42.374536+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":545,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393049+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:39:51.393049+0000","last_peered":"2024-08-17T17:39:51.393049+0000","last_clean":"2024-08-17T17:39:51.393049+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:39:51.393049+0000","last_undegraded":"2024-08-17T17:39:51.393049+0000","last_fullsized":"2024-08-17T17:39:51.393049+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":340,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.380457+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:39:42.380457+0000","last_peered":"2024-08-17T17:39:42.380457+0000","last_clean":"2024-08-17T17:39:42.380457+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:39:42.380457+0000","last_undegraded":"2024-08-17T17:39:42.380457+0000","last_fullsized":"2024-08-17T17:39:42.380457+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"376'9843","reported_seq":9977,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:54.621030+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:39:54.621030+0000","last_peered":"2024-08-17T17:39:54.621030+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:39:54.621030+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'2821","ondisk_log_start":"325'2821","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9843,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1746},{"shards":"0,4,7","objects":4901}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":361,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.390953+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:39:51.390953+0000","last_peered":"2024-08-17T17:39:51.390953+0000","last_clean":"2024-08-17T17:39:51.390953+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:39:51.390953+0000","last_undegraded":"2024-08-17T17:39:51.390953+0000","last_fullsized":"2024-08-17T17:39:51.390953+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":312,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454398+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:39:42.454398+0000","last_peered":"2024-08-17T17:39:42.454398+0000","last_clean":"2024-08-17T17:39:42.454398+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:39:42.454398+0000","last_undegraded":"2024-08-17T17:39:42.454398+0000","last_fullsized":"2024-08-17T17:39:42.454398+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":287,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.745264+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:39:51.745264+0000","last_peered":"2024-08-17T17:39:51.745264+0000","last_clean":"2024-08-17T17:39:51.745264+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:39:51.745264+0000","last_undegraded":"2024-08-17T17:39:51.745264+0000","last_fullsized":"2024-08-17T17:39:51.745264+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"376'9859","reported_seq":9934,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:51.614018+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:39:51.614018+0000","last_peered":"2024-08-17T17:39:51.614018+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:39:51.614018+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'3317","ondisk_log_start":"325'3317","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323551232,"num_objects":4937,"num_object_clones":0,"num_object_copies":14811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4937},{"shards":"5,6","objects":-2359}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":453,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392890+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:39:51.392890+0000","last_peered":"2024-08-17T17:39:51.392890+0000","last_clean":"2024-08-17T17:39:51.392890+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:39:51.392890+0000","last_undegraded":"2024-08-17T17:39:51.392890+0000","last_fullsized":"2024-08-17T17:39:51.392890+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.379607+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:39:42.379607+0000","last_peered":"2024-08-17T17:39:42.379607+0000","last_clean":"2024-08-17T17:39:42.379607+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:39:42.379607+0000","last_undegraded":"2024-08-17T17:39:42.379607+0000","last_fullsized":"2024-08-17T17:39:42.379607+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":637,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392648+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:39:51.392648+0000","last_peered":"2024-08-17T17:39:51.392648+0000","last_clean":"2024-08-17T17:39:51.392648+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:39:51.392648+0000","last_undegraded":"2024-08-17T17:39:51.392648+0000","last_fullsized":"2024-08-17T17:39:51.392648+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"376'9792","reported_seq":10650,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:54.658122+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:39:54.658122+0000","last_peered":"2024-08-17T17:39:54.658122+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:39:54.658122+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'2716","ondisk_log_start":"325'2716","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318111744,"num_objects":4854,"num_object_clones":0,"num_object_copies":14562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9792,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4854},{"shards":"0,5","objects":-1800}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":429,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.391528+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:39:51.391528+0000","last_peered":"2024-08-17T17:39:51.391528+0000","last_clean":"2024-08-17T17:39:51.391528+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:39:51.391528+0000","last_undegraded":"2024-08-17T17:39:51.391528+0000","last_fullsized":"2024-08-17T17:39:51.391528+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":351,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.455824+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:39:42.455824+0000","last_peered":"2024-08-17T17:39:42.455824+0000","last_clean":"2024-08-17T17:39:42.455824+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:39:42.455824+0000","last_undegraded":"2024-08-17T17:39:42.455824+0000","last_fullsized":"2024-08-17T17:39:42.455824+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":224,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392044+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:39:51.392044+0000","last_peered":"2024-08-17T17:39:51.392044+0000","last_clean":"2024-08-17T17:39:51.392044+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:39:51.392044+0000","last_undegraded":"2024-08-17T17:39:51.392044+0000","last_fullsized":"2024-08-17T17:39:51.392044+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"376'9961","reported_seq":24939,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:55.246390+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:39:55.246390+0000","last_peered":"2024-08-17T17:39:55.246390+0000","last_clean":"2024-08-17T17:39:55.246390+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:39:55.246390+0000","last_undegraded":"2024-08-17T17:39:55.246390+0000","last_fullsized":"2024-08-17T17:39:55.246390+0000","mapping_epoch":355,"log_start":"325'3630","ondisk_log_start":"325'3630","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319488000,"num_objects":4875,"num_object_clones":0,"num_object_copies":14625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9961,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":290,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374173+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:39:42.374173+0000","last_peered":"2024-08-17T17:39:42.374173+0000","last_clean":"2024-08-17T17:39:42.374173+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:39:42.374173+0000","last_undegraded":"2024-08-17T17:39:42.374173+0000","last_fullsized":"2024-08-17T17:39:42.374173+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.391996+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:39:51.391996+0000","last_peered":"2024-08-17T17:39:51.391996+0000","last_clean":"2024-08-17T17:39:51.391996+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:39:51.391996+0000","last_undegraded":"2024-08-17T17:39:51.391996+0000","last_fullsized":"2024-08-17T17:39:51.391996+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":588,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.391387+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:39:51.391387+0000","last_peered":"2024-08-17T17:39:51.391387+0000","last_clean":"2024-08-17T17:39:51.391387+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:39:51.391387+0000","last_undegraded":"2024-08-17T17:39:51.391387+0000","last_fullsized":"2024-08-17T17:39:51.391387+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"376'9656","reported_seq":24201,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:55.159259+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:39:55.159259+0000","last_peered":"2024-08-17T17:39:55.159259+0000","last_clean":"2024-08-17T17:39:55.159259+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:39:55.159259+0000","last_undegraded":"2024-08-17T17:39:55.159259+0000","last_fullsized":"2024-08-17T17:39:55.159259+0000","mapping_epoch":345,"log_start":"325'3372","ondisk_log_start":"325'3372","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311164928,"num_objects":4748,"num_object_clones":0,"num_object_copies":14244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9656,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":286,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374722+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:39:42.374722+0000","last_peered":"2024-08-17T17:39:42.374722+0000","last_clean":"2024-08-17T17:39:42.374722+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:39:42.374722+0000","last_undegraded":"2024-08-17T17:39:42.374722+0000","last_fullsized":"2024-08-17T17:39:42.374722+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":343,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.745008+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:39:51.745008+0000","last_peered":"2024-08-17T17:39:51.745008+0000","last_clean":"2024-08-17T17:39:51.745008+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:39:51.745008+0000","last_undegraded":"2024-08-17T17:39:51.745008+0000","last_fullsized":"2024-08-17T17:39:51.745008+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":428,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374464+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:39:42.374464+0000","last_peered":"2024-08-17T17:39:42.374464+0000","last_clean":"2024-08-17T17:39:42.374464+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:39:42.374464+0000","last_undegraded":"2024-08-17T17:39:42.374464+0000","last_fullsized":"2024-08-17T17:39:42.374464+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"376'9916","reported_seq":10069,"reported_epoch":376,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:55.358762+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:39:55.358762+0000","last_peered":"2024-08-17T17:39:55.358762+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:39:55.358762+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'3587","ondisk_log_start":"326'3587","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319029248,"num_objects":4868,"num_object_clones":0,"num_object_copies":14604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9916,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4868},{"shards":"0,3","objects":-2380}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":618,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392015+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:39:51.392015+0000","last_peered":"2024-08-17T17:39:51.392015+0000","last_clean":"2024-08-17T17:39:51.392015+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:39:51.392015+0000","last_undegraded":"2024-08-17T17:39:51.392015+0000","last_fullsized":"2024-08-17T17:39:51.392015+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":586,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392339+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:39:51.392339+0000","last_peered":"2024-08-17T17:39:51.392339+0000","last_clean":"2024-08-17T17:39:51.392339+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:39:51.392339+0000","last_undegraded":"2024-08-17T17:39:51.392339+0000","last_fullsized":"2024-08-17T17:39:51.392339+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":604,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393926+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:39:51.393926+0000","last_peered":"2024-08-17T17:39:51.393926+0000","last_clean":"2024-08-17T17:39:51.393926+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:39:51.393926+0000","last_undegraded":"2024-08-17T17:39:51.393926+0000","last_fullsized":"2024-08-17T17:39:51.393926+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"376'9785","reported_seq":24458,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.934508+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:39:51.934508+0000","last_peered":"2024-08-17T17:39:51.934508+0000","last_clean":"2024-08-17T17:39:51.934508+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:39:51.934508+0000","last_undegraded":"2024-08-17T17:39:51.934508+0000","last_fullsized":"2024-08-17T17:39:51.934508+0000","mapping_epoch":347,"log_start":"325'3172","ondisk_log_start":"325'3172","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313851904,"num_objects":4789,"num_object_clones":0,"num_object_copies":14367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9785,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":277,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.745167+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:39:51.745167+0000","last_peered":"2024-08-17T17:39:51.745167+0000","last_clean":"2024-08-17T17:39:51.745167+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:39:51.745167+0000","last_undegraded":"2024-08-17T17:39:51.745167+0000","last_fullsized":"2024-08-17T17:39:51.745167+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":380,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.381376+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:39:42.381376+0000","last_peered":"2024-08-17T17:39:42.381376+0000","last_clean":"2024-08-17T17:39:42.381376+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:39:42.381376+0000","last_undegraded":"2024-08-17T17:39:42.381376+0000","last_fullsized":"2024-08-17T17:39:42.381376+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":330,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454667+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:39:42.454667+0000","last_peered":"2024-08-17T17:39:42.454667+0000","last_clean":"2024-08-17T17:39:42.454667+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:39:42.454667+0000","last_undegraded":"2024-08-17T17:39:42.454667+0000","last_fullsized":"2024-08-17T17:39:42.454667+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":330,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.455676+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:39:42.455676+0000","last_peered":"2024-08-17T17:39:42.455676+0000","last_clean":"2024-08-17T17:39:42.455676+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:39:42.455676+0000","last_undegraded":"2024-08-17T17:39:42.455676+0000","last_fullsized":"2024-08-17T17:39:42.455676+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":373,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454677+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:39:42.454677+0000","last_peered":"2024-08-17T17:39:42.454677+0000","last_clean":"2024-08-17T17:39:42.454677+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:39:42.454677+0000","last_undegraded":"2024-08-17T17:39:42.454677+0000","last_fullsized":"2024-08-17T17:39:42.454677+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":593,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.391183+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:39:51.391183+0000","last_peered":"2024-08-17T17:39:51.391183+0000","last_clean":"2024-08-17T17:39:51.391183+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:39:51.391183+0000","last_undegraded":"2024-08-17T17:39:51.391183+0000","last_fullsized":"2024-08-17T17:39:51.391183+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":133,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392530+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:39:51.392530+0000","last_peered":"2024-08-17T17:39:51.392530+0000","last_clean":"2024-08-17T17:39:51.392530+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:39:51.392530+0000","last_undegraded":"2024-08-17T17:39:51.392530+0000","last_fullsized":"2024-08-17T17:39:51.392530+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":280,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374312+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:39:42.374312+0000","last_peered":"2024-08-17T17:39:42.374312+0000","last_clean":"2024-08-17T17:39:42.374312+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:39:42.374312+0000","last_undegraded":"2024-08-17T17:39:42.374312+0000","last_fullsized":"2024-08-17T17:39:42.374312+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":551,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392562+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:39:51.392562+0000","last_peered":"2024-08-17T17:39:51.392562+0000","last_clean":"2024-08-17T17:39:51.392562+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:39:51.392562+0000","last_undegraded":"2024-08-17T17:39:51.392562+0000","last_fullsized":"2024-08-17T17:39:51.392562+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":424,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.374822+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:39:42.374822+0000","last_peered":"2024-08-17T17:39:42.374822+0000","last_clean":"2024-08-17T17:39:42.374822+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:39:42.374822+0000","last_undegraded":"2024-08-17T17:39:42.374822+0000","last_fullsized":"2024-08-17T17:39:42.374822+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":545,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393831+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:39:51.393831+0000","last_peered":"2024-08-17T17:39:51.393831+0000","last_clean":"2024-08-17T17:39:51.393831+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:39:51.393831+0000","last_undegraded":"2024-08-17T17:39:51.393831+0000","last_fullsized":"2024-08-17T17:39:51.393831+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":347,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.381320+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:39:42.381320+0000","last_peered":"2024-08-17T17:39:42.381320+0000","last_clean":"2024-08-17T17:39:42.381320+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:39:42.381320+0000","last_undegraded":"2024-08-17T17:39:42.381320+0000","last_fullsized":"2024-08-17T17:39:42.381320+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":577,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.394008+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:39:51.394008+0000","last_peered":"2024-08-17T17:39:51.394008+0000","last_clean":"2024-08-17T17:39:51.394008+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:39:51.394008+0000","last_undegraded":"2024-08-17T17:39:51.394008+0000","last_fullsized":"2024-08-17T17:39:51.394008+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":293,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392849+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:39:51.392849+0000","last_peered":"2024-08-17T17:39:51.392849+0000","last_clean":"2024-08-17T17:39:51.392849+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:39:51.392849+0000","last_undegraded":"2024-08-17T17:39:51.392849+0000","last_fullsized":"2024-08-17T17:39:51.392849+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":585,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393108+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:39:51.393108+0000","last_peered":"2024-08-17T17:39:51.393108+0000","last_clean":"2024-08-17T17:39:51.393108+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:39:51.393108+0000","last_undegraded":"2024-08-17T17:39:51.393108+0000","last_fullsized":"2024-08-17T17:39:51.393108+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":544,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392331+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:39:51.392331+0000","last_peered":"2024-08-17T17:39:51.392331+0000","last_clean":"2024-08-17T17:39:51.392331+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:39:51.392331+0000","last_undegraded":"2024-08-17T17:39:51.392331+0000","last_fullsized":"2024-08-17T17:39:51.392331+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":359,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.455191+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:39:42.455191+0000","last_peered":"2024-08-17T17:39:42.455191+0000","last_clean":"2024-08-17T17:39:42.455191+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:39:42.455191+0000","last_undegraded":"2024-08-17T17:39:42.455191+0000","last_fullsized":"2024-08-17T17:39:42.455191+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":233,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392201+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:39:51.392201+0000","last_peered":"2024-08-17T17:39:51.392201+0000","last_clean":"2024-08-17T17:39:51.392201+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:39:51.392201+0000","last_undegraded":"2024-08-17T17:39:51.392201+0000","last_fullsized":"2024-08-17T17:39:51.392201+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":290,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.744842+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:39:51.744842+0000","last_peered":"2024-08-17T17:39:51.744842+0000","last_clean":"2024-08-17T17:39:51.744842+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:39:51.744842+0000","last_undegraded":"2024-08-17T17:39:51.744842+0000","last_fullsized":"2024-08-17T17:39:51.744842+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":367,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.455608+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:39:42.455608+0000","last_peered":"2024-08-17T17:39:42.455608+0000","last_clean":"2024-08-17T17:39:42.455608+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:39:42.455608+0000","last_undegraded":"2024-08-17T17:39:42.455608+0000","last_fullsized":"2024-08-17T17:39:42.455608+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":230,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454451+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:39:42.454451+0000","last_peered":"2024-08-17T17:39:42.454451+0000","last_clean":"2024-08-17T17:39:42.454451+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:39:42.454451+0000","last_undegraded":"2024-08-17T17:39:42.454451+0000","last_fullsized":"2024-08-17T17:39:42.454451+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":367,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.455061+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:39:42.455061+0000","last_peered":"2024-08-17T17:39:42.455061+0000","last_clean":"2024-08-17T17:39:42.455061+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:39:42.455061+0000","last_undegraded":"2024-08-17T17:39:42.455061+0000","last_fullsized":"2024-08-17T17:39:42.455061+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":230,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454760+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:39:42.454760+0000","last_peered":"2024-08-17T17:39:42.454760+0000","last_clean":"2024-08-17T17:39:42.454760+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:39:42.454760+0000","last_undegraded":"2024-08-17T17:39:42.454760+0000","last_fullsized":"2024-08-17T17:39:42.454760+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":597,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.392981+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:39:51.392981+0000","last_peered":"2024-08-17T17:39:51.392981+0000","last_clean":"2024-08-17T17:39:51.392981+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:39:51.392981+0000","last_undegraded":"2024-08-17T17:39:51.392981+0000","last_fullsized":"2024-08-17T17:39:51.392981+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":570,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.393610+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:39:51.393610+0000","last_peered":"2024-08-17T17:39:51.393610+0000","last_clean":"2024-08-17T17:39:51.393610+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:39:51.393610+0000","last_undegraded":"2024-08-17T17:39:51.393610+0000","last_fullsized":"2024-08-17T17:39:51.393610+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":330,"reported_epoch":375,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.454319+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:39:42.454319+0000","last_peered":"2024-08-17T17:39:42.454319+0000","last_clean":"2024-08-17T17:39:42.454319+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:39:42.454319+0000","last_undegraded":"2024-08-17T17:39:42.454319+0000","last_fullsized":"2024-08-17T17:39:42.454319+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":292,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.745306+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:39:51.745306+0000","last_peered":"2024-08-17T17:39:51.745306+0000","last_clean":"2024-08-17T17:39:51.745306+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:39:51.745306+0000","last_undegraded":"2024-08-17T17:39:51.745306+0000","last_fullsized":"2024-08-17T17:39:51.745306+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5129175064,"num_objects":78266,"num_object_clones":0,"num_object_copies":234798,"num_objects_missing_on_primary":5558,"num_objects_missing":5558,"num_objects_degraded":54215,"num_objects_misplaced":30946,"num_objects_unfound":0,"num_objects_dirty":78266,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158124,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27584,"num_bytes_recovered":1685192704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16751341568,"data_stored":16751329352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104843,"ondisk_log_size":104843,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170082,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463451,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3099720,"kb_used_data":2197696,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":90645432,"statfs":{"total":95995035648,"available":92820922368,"internally_reserved":0,"allocated":2250440704,"data_stored":2248418090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492126,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1021372,"kb_used_data":630196,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92723780,"statfs":{"total":95995035648,"available":94949150720,"internally_reserved":0,"allocated":645320704,"data_stored":643312597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986717,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2440336,"kb_used_data":1251012,"kb_used_omap":1,"kb_used_meta":1189310,"kb_avail":91304816,"statfs":{"total":95995035648,"available":93496131584,"internally_reserved":0,"allocated":1281036288,"data_stored":1279016821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1217853784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1,3,2,0,0,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225545,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3416380,"kb_used_data":2492340,"kb_used_omap":4,"kb_used_meta":924027,"kb_avail":90328772,"statfs":{"total":95995035648,"available":92496662528,"internally_reserved":0,"allocated":2552156160,"data_stored":2550148351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":946204096},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075870,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533763,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3240640,"kb_used_data":2198648,"kb_used_omap":5,"kb_used_meta":1041978,"kb_avail":90504512,"statfs":{"total":95995035648,"available":92676620288,"internally_reserved":0,"allocated":2251415552,"data_stored":2249402421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1066985712},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089740,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2892188,"kb_used_data":1887120,"kb_used_omap":5,"kb_used_meta":1005050,"kb_avail":90852964,"statfs":{"total":95995035648,"available":93033435136,"internally_reserved":0,"allocated":1932410880,"data_stored":1930380532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1029171259},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1924923392,"data_stored":1924923392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2244415488,"data_stored":2244411416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3389198336,"data_stored":3389194264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2545025024,"data_stored":2545025024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1276051456,"data_stored":1276051456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":640352256,"data_stored":640352256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2243035136,"data_stored":2243035136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2488340480,"data_stored":2488336408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:58.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:58.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:58.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:58.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:58.481 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:58 smithi142 ceph-mon[106729]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:58.481 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:58 smithi142 ceph-mon[103614]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:58.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:58.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:58.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:58.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:58.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:58.806 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:58 smithi152 ceph-mon[93456]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:58.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:58.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:58.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:58.965 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:39:58.965 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:58.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:58.991 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:39:58.991 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:59.013 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:39:59.013 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:59.030 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:39:59.031 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:59.048 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:39:59.048 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:59.063 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:39:59.064 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:59.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:59.079 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:39:59.079 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:59.097 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:39:59.097 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:59.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:59.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:59.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:59.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:59.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:59.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:59.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:39:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:59.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:59.732 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:59 smithi152 ceph-mon[93456]: pgmap v1398: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 54218/234381 objects degraded (23.132%); 30909/234381 objects misplaced (13.188%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:59.732 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:39:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4130937722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:59.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:59.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:59 smithi142 ceph-mon[106729]: pgmap v1398: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 54218/234381 objects degraded (23.132%); 30909/234381 objects misplaced (13.188%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:39:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4130937722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:59 smithi142 ceph-mon[103614]: pgmap v1398: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 54218/234381 objects degraded (23.132%); 30909/234381 objects misplaced (13.188%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:39:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:39:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4130937722' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:59.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:59.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:39:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:00.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:00.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:00.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:00.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:00.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:00.322 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:40:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:00.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:00.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:00.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:00.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:00.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:00.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:00.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: Health detail: HEALTH_WARN Degraded data redundancy: 54218/234381 objects degraded (23.132%), 10 pgs degraded, 7 pgs undersized 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: [WRN] PG_DEGRADED: Degraded data redundancy: 54218/234381 objects degraded (23.132%), 10 pgs degraded, 7 pgs undersized 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 1.0 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 2.3 is active+recovery_wait+degraded, acting [5,3,6] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.3 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.5 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.7 is active+recovery_wait+degraded+remapped, acting [4,1,7] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.8 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,0] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.c is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.d is active+recovery_wait+degraded+remapped, acting [1,3,7] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.e is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2024-08-17T17:40:00.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[106729]: pg 12.f is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: Health detail: HEALTH_WARN Degraded data redundancy: 54218/234381 objects degraded (23.132%), 10 pgs degraded, 7 pgs undersized 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: [WRN] PG_DEGRADED: Degraded data redundancy: 54218/234381 objects degraded (23.132%), 10 pgs degraded, 7 pgs undersized 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 1.0 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 2.3 is active+recovery_wait+degraded, acting [5,3,6] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.3 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.5 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.7 is active+recovery_wait+degraded+remapped, acting [4,1,7] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.8 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,0] 2024-08-17T17:40:00.624 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.c is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2024-08-17T17:40:00.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.d is active+recovery_wait+degraded+remapped, acting [1,3,7] 2024-08-17T17:40:00.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.e is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2024-08-17T17:40:00.625 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:00 smithi142 ceph-mon[103614]: pg 12.f is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2024-08-17T17:40:00.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:00.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: Health detail: HEALTH_WARN Degraded data redundancy: 54218/234381 objects degraded (23.132%), 10 pgs degraded, 7 pgs undersized 2024-08-17T17:40:00.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: [WRN] PG_DEGRADED: Degraded data redundancy: 54218/234381 objects degraded (23.132%), 10 pgs degraded, 7 pgs undersized 2024-08-17T17:40:00.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 1.0 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2024-08-17T17:40:00.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 2.3 is active+recovery_wait+degraded, acting [5,3,6] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.3 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.5 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.7 is active+recovery_wait+degraded+remapped, acting [4,1,7] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.8 is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,0] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.c is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.d is active+recovery_wait+degraded+remapped, acting [1,3,7] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.e is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2024-08-17T17:40:00.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:00 smithi152 ceph-mon[93456]: pg 12.f is stuck undersized for 6m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2024-08-17T17:40:00.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:00.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:00.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:00.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:00.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:01.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:01.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:01.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:01.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:01.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:40:01.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:01.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:01.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:01.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:01.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:01.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:01.592 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:01 smithi142 ceph-mon[106729]: pgmap v1399: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 54147/233640 objects degraded (23.175%); 30749/233640 objects misplaced (13.161%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54147/233640 objects degraded (23.175%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:01 smithi142 ceph-mon[103614]: pgmap v1399: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 54147/233640 objects degraded (23.175%); 30749/233640 objects misplaced (13.161%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54147/233640 objects degraded (23.175%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:01.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:01.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:01 smithi152 ceph-mon[93456]: pgmap v1399: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 54147/233640 objects degraded (23.175%); 30749/233640 objects misplaced (13.161%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:01.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54147/233640 objects degraded (23.175%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:01.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:01.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:01.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:01.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:01.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:01.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:01.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:02.015 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:40:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:40:01] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:40:02.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:02.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:02.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:02.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:02.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:02.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:02.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:02.458 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:40:02.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:02.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:02.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1990852433' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1990852433' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:02.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:02.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:02.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:02.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1990852433' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:02.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:02.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:02.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:02.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:02.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:02.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:02.980 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:03.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:03.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:03.140 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:03.140 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:03.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:03.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:03.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:03.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:03.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:03.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:03.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:03.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:03.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:03.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:03.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:03.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[106729]: pgmap v1400: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 54147/232908 objects degraded (23.248%); 30719/232908 objects misplaced (13.189%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1990852433' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[106729]: osdmap e377: 8 total, 8 up, 6 in 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[103614]: pgmap v1400: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 54147/232908 objects degraded (23.248%); 30719/232908 objects misplaced (13.189%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:03.747 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1990852433' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:03.748 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:03 smithi142 ceph-mon[103614]: osdmap e377: 8 total, 8 up, 6 in 2024-08-17T17:40:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:03 smithi152 ceph-mon[93456]: pgmap v1400: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 54147/232908 objects degraded (23.248%); 30719/232908 objects misplaced (13.189%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:40:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:03 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1990852433' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:03 smithi152 ceph-mon[93456]: osdmap e377: 8 total, 8 up, 6 in 2024-08-17T17:40:03.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:03.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:03.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:03.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:04.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:04.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:40:04.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:04.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:04.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:04.229 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:04.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:04.286 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:04.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:04.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:04.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:04.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:04.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:04.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:04.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:04.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:04.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:05.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:05.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:05.203 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:05.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:05.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:05.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:05.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:05.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:05.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:05.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:05.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:05.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:05.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:05 smithi152 ceph-mon[93456]: pgmap v1402: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 54075/232359 objects degraded (23.272%); 30537/232359 objects misplaced (13.142%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:05.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:05.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:05 smithi142 ceph-mon[106729]: pgmap v1402: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 54075/232359 objects degraded (23.272%); 30537/232359 objects misplaced (13.142%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:05 smithi142 ceph-mon[103614]: pgmap v1402: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 54075/232359 objects degraded (23.272%); 30537/232359 objects misplaced (13.142%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:05.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:05.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:05.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:06.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:06.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:06.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:06.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:06.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:06.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:06.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:06.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:06.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:06.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:06.568 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:06.570 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:06.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:06.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:06.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:06.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:06.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54075/232359 objects degraded (23.272%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:06.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54075/232359 objects degraded (23.272%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:06.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54075/232359 objects degraded (23.272%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:06.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:06.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:06.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:07.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:07.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:07.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:07.211 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:07.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:07.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:07.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:07.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:07.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:07.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:07.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:07.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:07.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:07.718 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:07.820 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:07 smithi152 ceph-mon[93456]: pgmap v1403: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 54075/231648 objects degraded (23.344%); 30537/231648 objects misplaced (13.183%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:07.841 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:07 smithi142 ceph-mon[103614]: pgmap v1403: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 54075/231648 objects degraded (23.344%); 30537/231648 objects misplaced (13.183%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:07.841 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:07 smithi142 ceph-mon[106729]: pgmap v1403: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 54075/231648 objects degraded (23.344%); 30537/231648 objects misplaced (13.183%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:07.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:07.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:07.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:08.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:08.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:08.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:08.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:40:08.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:08.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:08.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:08.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:08.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:08.477 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:08.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:08.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:08.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:08.728 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:08.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:08.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:08.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:08.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:08.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:09.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:09.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:09.234 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:09.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:09.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:09.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:09.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:09.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:09.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:09.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:09.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:09.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:09.649 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:09.649 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:09.664 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:09 smithi142 ceph-mon[106729]: pgmap v1404: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 54075/231156 objects degraded (23.393%); 30514/231156 objects misplaced (13.201%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:09.664 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:09 smithi142 ceph-mon[103614]: pgmap v1404: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 54075/231156 objects degraded (23.393%); 30514/231156 objects misplaced (13.201%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:09.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:09.736 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:09 smithi152 ceph-mon[93456]: pgmap v1404: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 54075/231156 objects degraded (23.393%); 30514/231156 objects misplaced (13.201%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:09.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:09.864 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:09.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:09.880 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:09.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:09.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:10.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:10.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:10.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:10.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:10.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:40:10.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:10.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:10.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:10.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:10.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:10.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:40:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:40:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:10.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:10.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:10.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:10.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:10.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:10.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:10.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.912 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:10.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:10.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:11.050 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:40:11.051 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:40:11.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:11.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:11.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:11.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:11.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:11.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:11.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:11.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:11.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:11.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:11.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:11.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[106729]: pgmap v1405: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 54001/230157 objects degraded (23.463%); 30327/230157 objects misplaced (13.177%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:11.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 54001/230157 objects degraded (23.463%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:11.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:11.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:11.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[103614]: pgmap v1405: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 54001/230157 objects degraded (23.463%); 30327/230157 objects misplaced (13.177%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:11.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 54001/230157 objects degraded (23.463%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:11.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:11.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:11.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:11.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:11.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:11.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:11.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:11 smithi152 ceph-mon[93456]: pgmap v1405: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 54001/230157 objects degraded (23.463%); 30327/230157 objects misplaced (13.177%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 54001/230157 objects degraded (23.463%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:11.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:11.851 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:40:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:40:11] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:40:11.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:11.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:11.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:12.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:12.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:12.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:12.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:12.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:12.334 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:12.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:12.460 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:40:12.461 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:12.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:12.481 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:40:12.481 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:12.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:12.500 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:40:12.501 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:12.520 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:40:12.520 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:12.535 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:40:12.535 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:12.550 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:40:12.550 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:12.565 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:40:12.566 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:12.581 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:40:12.581 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:12.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:12.615 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:12 smithi142 ceph-mon[106729]: from='client.56854 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/264999262' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:12 smithi142 ceph-mon[103614]: from='client.56854 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/264999262' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:12.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:12.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:12 smithi152 ceph-mon[93456]: from='client.56854 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/264999262' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:12.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:12.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:12.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:12.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:13.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:13.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:13.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:13.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:13.262 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1405,"stamp":"2024-08-17T17:40:10.321996+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5026682853,"num_objects":76719,"num_object_clones":0,"num_object_copies":230157,"num_objects_missing_on_primary":5460,"num_objects_missing":5460,"num_objects_degraded":54001,"num_objects_misplaced":30327,"num_objects_unfound":0,"num_objects_dirty":76719,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":159975,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27939,"num_bytes_recovered":1708139150,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105023,"ondisk_log_size":105023,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15912148,"kb_used_data":10448804,"kb_used_omap":27,"kb_used_meta":5463268,"kb_avail":546558764,"statfs":{"total":575970213888,"available":559676174336,"internally_reserved":0,"allocated":10699575296,"data_stored":10687419269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5594387176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-76087296,"num_objects":-1161,"num_object_clones":0,"num_object_copies":-3483,"num_objects_missing_on_primary":-65,"num_objects_missing":-65,"num_objects_degraded":-146,"num_objects_misplaced":-422,"num_objects_unfound":0,"num_objects_dirty":-1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1161,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":212,"num_bytes_recovered":13893632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003042"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":596,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471423+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:40:02.471423+0000","last_peered":"2024-08-17T17:40:02.471423+0000","last_clean":"2024-08-17T17:40:02.471423+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:40:02.471423+0000","last_undegraded":"2024-08-17T17:40:02.471423+0000","last_fullsized":"2024-08-17T17:40:02.471423+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":86,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481512+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:39:51.481512+0000","last_peered":"2024-08-17T17:39:51.481512+0000","last_clean":"2024-08-17T17:39:51.481512+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:39:51.481512+0000","last_undegraded":"2024-08-17T17:39:51.481512+0000","last_fullsized":"2024-08-17T17:39:51.481512+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":280,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.645187+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:39:51.645187+0000","last_peered":"2024-08-17T17:39:51.645187+0000","last_clean":"2024-08-17T17:39:51.645187+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:39:51.645187+0000","last_undegraded":"2024-08-17T17:39:51.645187+0000","last_fullsized":"2024-08-17T17:39:51.645187+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":294,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.472467+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:40:02.472467+0000","last_peered":"2024-08-17T17:40:02.472467+0000","last_clean":"2024-08-17T17:40:02.472467+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:40:02.472467+0000","last_undegraded":"2024-08-17T17:40:02.472467+0000","last_fullsized":"2024-08-17T17:40:02.472467+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":471,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471777+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:40:02.471777+0000","last_peered":"2024-08-17T17:40:02.471777+0000","last_clean":"2024-08-17T17:40:02.471777+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:40:02.471777+0000","last_undegraded":"2024-08-17T17:40:02.471777+0000","last_fullsized":"2024-08-17T17:40:02.471777+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":229,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481403+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:39:51.481403+0000","last_peered":"2024-08-17T17:39:51.481403+0000","last_clean":"2024-08-17T17:39:51.481403+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:39:51.481403+0000","last_undegraded":"2024-08-17T17:39:51.481403+0000","last_fullsized":"2024-08-17T17:39:51.481403+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":565,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471840+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:40:02.471840+0000","last_peered":"2024-08-17T17:40:02.471840+0000","last_clean":"2024-08-17T17:40:02.471840+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:40:02.471840+0000","last_undegraded":"2024-08-17T17:40:02.471840+0000","last_fullsized":"2024-08-17T17:40:02.471840+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":397,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.610408+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:39:51.610408+0000","last_peered":"2024-08-17T17:39:51.610408+0000","last_clean":"2024-08-17T17:39:51.610408+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:39:51.610408+0000","last_undegraded":"2024-08-17T17:39:51.610408+0000","last_fullsized":"2024-08-17T17:39:51.610408+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"377'9933","reported_seq":13728,"reported_epoch":377,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:40:08.632928+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:40:08.632928+0000","last_peered":"2024-08-17T17:40:08.632928+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:40:08.632928+0000","last_undegraded":"2024-08-17T17:40:08.632928+0000","last_fullsized":"2024-08-17T17:40:08.632928+0000","mapping_epoch":337,"log_start":"325'3063","ondisk_log_start":"325'3063","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312672256,"num_objects":4771,"num_object_clones":0,"num_object_copies":14313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4770,"num_objects_unfound":0,"num_objects_dirty":4771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9933,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1853,"num_bytes_recovered":121438208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4770},{"shards":"4,6,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":82,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.516063+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:39:51.516063+0000","last_peered":"2024-08-17T17:39:51.516063+0000","last_clean":"2024-08-17T17:39:51.516063+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:39:51.516063+0000","last_undegraded":"2024-08-17T17:39:51.516063+0000","last_fullsized":"2024-08-17T17:39:51.516063+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":263,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.644583+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:39:51.644583+0000","last_peered":"2024-08-17T17:39:51.644583+0000","last_clean":"2024-08-17T17:39:51.644583+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:39:51.644583+0000","last_undegraded":"2024-08-17T17:39:51.644583+0000","last_fullsized":"2024-08-17T17:39:51.644583+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"377'10180","reported_seq":10255,"reported_epoch":377,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:05.613718+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:40:05.613718+0000","last_peered":"2024-08-17T17:40:05.613718+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:40:05.613718+0000","last_undegraded":"2024-08-17T17:40:05.613718+0000","last_fullsized":"2024-08-17T17:40:05.613718+0000","mapping_epoch":337,"log_start":"325'3582","ondisk_log_start":"325'3582","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320733184,"num_objects":4894,"num_object_clones":0,"num_object_copies":14682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4894,"num_objects_unfound":0,"num_objects_dirty":4894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10180,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4894}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":84,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481890+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:39:51.481890+0000","last_peered":"2024-08-17T17:39:51.481890+0000","last_clean":"2024-08-17T17:39:51.481890+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:39:51.481890+0000","last_undegraded":"2024-08-17T17:39:51.481890+0000","last_fullsized":"2024-08-17T17:39:51.481890+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":348,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481375+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:39:51.481375+0000","last_peered":"2024-08-17T17:39:51.481375+0000","last_clean":"2024-08-17T17:39:51.481375+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:39:51.481375+0000","last_undegraded":"2024-08-17T17:39:51.481375+0000","last_fullsized":"2024-08-17T17:39:51.481375+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"377'9927","reported_seq":10005,"reported_epoch":377,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:07.790870+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:40:07.790870+0000","last_peered":"2024-08-17T17:40:07.790870+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:40:07.790870+0000","last_undegraded":"2024-08-17T17:40:07.790870+0000","last_fullsized":"2024-08-17T17:40:07.790870+0000","mapping_epoch":337,"log_start":"325'3053","ondisk_log_start":"325'3053","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315031552,"num_objects":4807,"num_object_clones":0,"num_object_copies":14421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4807,"num_objects_unfound":0,"num_objects_dirty":4807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9927,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4807}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":80,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.645108+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:39:51.645108+0000","last_peered":"2024-08-17T17:39:51.645108+0000","last_clean":"2024-08-17T17:39:51.645108+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:39:51.645108+0000","last_undegraded":"2024-08-17T17:39:51.645108+0000","last_fullsized":"2024-08-17T17:39:51.645108+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":358,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.610433+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:39:51.610433+0000","last_peered":"2024-08-17T17:39:51.610433+0000","last_clean":"2024-08-17T17:39:51.610433+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:39:51.610433+0000","last_undegraded":"2024-08-17T17:39:51.610433+0000","last_fullsized":"2024-08-17T17:39:51.610433+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"377'10147","reported_seq":10281,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:09.938401+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:40:09.938401+0000","last_peered":"2024-08-17T17:40:09.938401+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:40:09.938401+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'3113","ondisk_log_start":"325'3113","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315293696,"num_objects":4811,"num_object_clones":0,"num_object_copies":14433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10147,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-1958},{"shards":"0,3,7","objects":4811}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":508,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471576+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:40:02.471576+0000","last_peered":"2024-08-17T17:40:02.471576+0000","last_clean":"2024-08-17T17:40:02.471576+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:40:02.471576+0000","last_undegraded":"2024-08-17T17:40:02.471576+0000","last_fullsized":"2024-08-17T17:40:02.471576+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":274,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.480775+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:40:02.480775+0000","last_peered":"2024-08-17T17:40:02.480775+0000","last_clean":"2024-08-17T17:40:02.480775+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:40:02.480775+0000","last_undegraded":"2024-08-17T17:40:02.480775+0000","last_fullsized":"2024-08-17T17:40:02.480775+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"377'9856","reported_seq":9922,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:06.957377+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:40:06.957377+0000","last_peered":"2024-08-17T17:40:06.957377+0000","last_clean":"2024-08-17T17:40:06.957377+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:40:06.957377+0000","last_undegraded":"2024-08-17T17:40:06.957377+0000","last_fullsized":"2024-08-17T17:40:06.957377+0000","mapping_epoch":319,"log_start":"325'3274","ondisk_log_start":"325'3274","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310509568,"num_objects":4738,"num_object_clones":0,"num_object_copies":14214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9856,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":598,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.472769+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:40:02.472769+0000","last_peered":"2024-08-17T17:40:02.472769+0000","last_clean":"2024-08-17T17:40:02.472769+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:40:02.472769+0000","last_undegraded":"2024-08-17T17:40:02.472769+0000","last_fullsized":"2024-08-17T17:40:02.472769+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":331,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481553+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:39:51.481553+0000","last_peered":"2024-08-17T17:39:51.481553+0000","last_clean":"2024-08-17T17:39:51.481553+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:39:51.481553+0000","last_undegraded":"2024-08-17T17:39:51.481553+0000","last_fullsized":"2024-08-17T17:39:51.481553+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"377'10004","reported_seq":10080,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:07.009572+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:40:07.009572+0000","last_peered":"2024-08-17T17:40:07.009572+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:40:07.009572+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"325'3435","ondisk_log_start":"325'3435","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322961408,"num_objects":4928,"num_object_clones":0,"num_object_copies":14784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10004,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4928},{"shards":"1,6","objects":-2438}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":332,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.610159+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:39:51.610159+0000","last_peered":"2024-08-17T17:39:51.610159+0000","last_clean":"2024-08-17T17:39:51.610159+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:39:51.610159+0000","last_undegraded":"2024-08-17T17:39:51.610159+0000","last_fullsized":"2024-08-17T17:39:51.610159+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":358,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481915+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:39:51.481915+0000","last_peered":"2024-08-17T17:39:51.481915+0000","last_clean":"2024-08-17T17:39:51.481915+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:39:51.481915+0000","last_undegraded":"2024-08-17T17:39:51.481915+0000","last_fullsized":"2024-08-17T17:39:51.481915+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"377'10177","reported_seq":10311,"reported_epoch":377,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:08.898693+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:40:08.898693+0000","last_peered":"2024-08-17T17:40:08.898693+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:40:08.898693+0000","last_undegraded":"2024-08-17T17:40:08.898693+0000","last_fullsized":"2024-08-17T17:40:08.898693+0000","mapping_epoch":337,"log_start":"326'4561","ondisk_log_start":"326'4561","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316735488,"num_objects":4833,"num_object_clones":0,"num_object_copies":14499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4833,"num_objects_unfound":0,"num_objects_dirty":4833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10177,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4833}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":369,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481492+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:39:51.481492+0000","last_peered":"2024-08-17T17:39:51.481492+0000","last_clean":"2024-08-17T17:39:51.481492+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:39:51.481492+0000","last_undegraded":"2024-08-17T17:39:51.481492+0000","last_fullsized":"2024-08-17T17:39:51.481492+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":331,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481954+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:39:51.481954+0000","last_peered":"2024-08-17T17:39:51.481954+0000","last_clean":"2024-08-17T17:39:51.481954+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:39:51.481954+0000","last_undegraded":"2024-08-17T17:39:51.481954+0000","last_fullsized":"2024-08-17T17:39:51.481954+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"377'9938","reported_seq":13793,"reported_epoch":377,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:40:08.499059+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:40:08.499059+0000","last_peered":"2024-08-17T17:40:08.499059+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:40:08.499059+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:40:08.499059+0000","mapping_epoch":337,"log_start":"325'3041","ondisk_log_start":"325'3041","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312344576,"num_objects":4766,"num_object_clones":0,"num_object_copies":14298,"num_objects_missing_on_primary":5460,"num_objects_missing":5460,"num_objects_degraded":4766,"num_objects_misplaced":5460,"num_objects_unfound":0,"num_objects_dirty":4766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9938,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1881,"num_bytes_recovered":123207680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":11},{"shards":"1,2,7","objects":4754},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":567,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.480726+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:40:02.480726+0000","last_peered":"2024-08-17T17:40:02.480726+0000","last_clean":"2024-08-17T17:40:02.480726+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:40:02.480726+0000","last_undegraded":"2024-08-17T17:40:02.480726+0000","last_fullsized":"2024-08-17T17:40:02.480726+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":504,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471937+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:40:02.471937+0000","last_peered":"2024-08-17T17:40:02.471937+0000","last_clean":"2024-08-17T17:40:02.471937+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:40:02.471937+0000","last_undegraded":"2024-08-17T17:40:02.471937+0000","last_fullsized":"2024-08-17T17:40:02.471937+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"377'10083","reported_seq":13976,"reported_epoch":377,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:40:08.834930+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:40:08.834930+0000","last_peered":"2024-08-17T17:40:08.834930+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:40:08.834930+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:40:08.834930+0000","mapping_epoch":337,"log_start":"326'4444","ondisk_log_start":"326'4444","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315359256,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4813,"num_objects_misplaced":5563,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10083,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1885,"num_bytes_recovered":123469824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4813}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":698,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:02.478511+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:40:02.478511+0000","last_peered":"2024-08-17T17:40:02.478511+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:40:02.478511+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":306,"reported_epoch":376,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:39:51.645014+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:39:51.645014+0000","last_peered":"2024-08-17T17:39:51.645014+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:39:51.645014+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:39:51.645014+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":547,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.472375+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:40:02.472375+0000","last_peered":"2024-08-17T17:40:02.472375+0000","last_clean":"2024-08-17T17:40:02.472375+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:40:02.472375+0000","last_undegraded":"2024-08-17T17:40:02.472375+0000","last_fullsized":"2024-08-17T17:40:02.472375+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":341,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.610832+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:39:51.610832+0000","last_peered":"2024-08-17T17:39:51.610832+0000","last_clean":"2024-08-17T17:39:51.610832+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:39:51.610832+0000","last_undegraded":"2024-08-17T17:39:51.610832+0000","last_fullsized":"2024-08-17T17:39:51.610832+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"377'9959","reported_seq":10095,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:09.769544+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:40:09.769544+0000","last_peered":"2024-08-17T17:40:09.769544+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:40:09.769544+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'2921","ondisk_log_start":"325'2921","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313589760,"num_objects":4785,"num_object_clones":0,"num_object_copies":14355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9959,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1862},{"shards":"0,4,7","objects":4785}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":363,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.462127+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:40:02.462127+0000","last_peered":"2024-08-17T17:40:02.462127+0000","last_clean":"2024-08-17T17:40:02.462127+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:40:02.462127+0000","last_undegraded":"2024-08-17T17:40:02.462127+0000","last_fullsized":"2024-08-17T17:40:02.462127+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":313,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481321+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:39:51.481321+0000","last_peered":"2024-08-17T17:39:51.481321+0000","last_clean":"2024-08-17T17:39:51.481321+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:39:51.481321+0000","last_undegraded":"2024-08-17T17:39:51.481321+0000","last_fullsized":"2024-08-17T17:39:51.481321+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":289,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.477548+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:40:02.477548+0000","last_peered":"2024-08-17T17:40:02.477548+0000","last_clean":"2024-08-17T17:40:02.477548+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:40:02.477548+0000","last_undegraded":"2024-08-17T17:40:02.477548+0000","last_fullsized":"2024-08-17T17:40:02.477548+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"377'9964","reported_seq":10039,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:06.900550+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:40:06.900550+0000","last_peered":"2024-08-17T17:40:06.900550+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:40:06.900550+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'3417","ondisk_log_start":"325'3417","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6547,"log_dups_size":0,"ondisk_log_size":6547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316669952,"num_objects":4832,"num_object_clones":0,"num_object_copies":14496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9964,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4832},{"shards":"5,6","objects":-2464}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":455,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471391+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:40:02.471391+0000","last_peered":"2024-08-17T17:40:02.471391+0000","last_clean":"2024-08-17T17:40:02.471391+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:40:02.471391+0000","last_undegraded":"2024-08-17T17:40:02.471391+0000","last_fullsized":"2024-08-17T17:40:02.471391+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":113,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.610293+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:39:51.610293+0000","last_peered":"2024-08-17T17:39:51.610293+0000","last_clean":"2024-08-17T17:39:51.610293+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:39:51.610293+0000","last_undegraded":"2024-08-17T17:39:51.610293+0000","last_fullsized":"2024-08-17T17:39:51.610293+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":639,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.463884+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:40:02.463884+0000","last_peered":"2024-08-17T17:40:02.463884+0000","last_clean":"2024-08-17T17:40:02.463884+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:40:02.463884+0000","last_undegraded":"2024-08-17T17:40:02.463884+0000","last_fullsized":"2024-08-17T17:40:02.463884+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"377'9903","reported_seq":10763,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:09.724757+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:40:09.724757+0000","last_peered":"2024-08-17T17:40:09.724757+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:40:09.724757+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'2916","ondisk_log_start":"325'2916","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310837248,"num_objects":4743,"num_object_clones":0,"num_object_copies":14229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9903,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4743},{"shards":"0,5","objects":-1911}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":431,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.462921+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:40:02.462921+0000","last_peered":"2024-08-17T17:40:02.462921+0000","last_clean":"2024-08-17T17:40:02.462921+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:40:02.462921+0000","last_undegraded":"2024-08-17T17:40:02.462921+0000","last_fullsized":"2024-08-17T17:40:02.462921+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":352,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481982+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:39:51.481982+0000","last_peered":"2024-08-17T17:39:51.481982+0000","last_clean":"2024-08-17T17:39:51.481982+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:39:51.481982+0000","last_undegraded":"2024-08-17T17:39:51.481982+0000","last_fullsized":"2024-08-17T17:39:51.481982+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471344+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:40:02.471344+0000","last_peered":"2024-08-17T17:40:02.471344+0000","last_clean":"2024-08-17T17:40:02.471344+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:40:02.471344+0000","last_undegraded":"2024-08-17T17:40:02.471344+0000","last_fullsized":"2024-08-17T17:40:02.471344+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"377'10040","reported_seq":25020,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:05.379075+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:40:05.379075+0000","last_peered":"2024-08-17T17:40:05.379075+0000","last_clean":"2024-08-17T17:40:05.379075+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:40:05.379075+0000","last_undegraded":"2024-08-17T17:40:05.379075+0000","last_fullsized":"2024-08-17T17:40:05.379075+0000","mapping_epoch":355,"log_start":"326'3730","ondisk_log_start":"326'3730","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6310,"log_dups_size":0,"ondisk_log_size":6310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314310656,"num_objects":4796,"num_object_clones":0,"num_object_copies":14388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10040,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":291,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.645026+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:39:51.645026+0000","last_peered":"2024-08-17T17:39:51.645026+0000","last_clean":"2024-08-17T17:39:51.645026+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:39:51.645026+0000","last_undegraded":"2024-08-17T17:39:51.645026+0000","last_fullsized":"2024-08-17T17:39:51.645026+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":95,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471246+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:40:02.471246+0000","last_peered":"2024-08-17T17:40:02.471246+0000","last_clean":"2024-08-17T17:40:02.471246+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:40:02.471246+0000","last_undegraded":"2024-08-17T17:40:02.471246+0000","last_fullsized":"2024-08-17T17:40:02.471246+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":590,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.463065+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:40:02.463065+0000","last_peered":"2024-08-17T17:40:02.463065+0000","last_clean":"2024-08-17T17:40:02.463065+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:40:02.463065+0000","last_undegraded":"2024-08-17T17:40:02.463065+0000","last_fullsized":"2024-08-17T17:40:02.463065+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"377'9720","reported_seq":24267,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:05.586732+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:40:05.586732+0000","last_peered":"2024-08-17T17:40:05.586732+0000","last_clean":"2024-08-17T17:40:05.586732+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:40:05.586732+0000","last_undegraded":"2024-08-17T17:40:05.586732+0000","last_fullsized":"2024-08-17T17:40:05.586732+0000","mapping_epoch":345,"log_start":"325'3372","ondisk_log_start":"325'3372","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6348,"log_dups_size":0,"ondisk_log_size":6348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306970624,"num_objects":4684,"num_object_clones":0,"num_object_copies":14052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9720,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":287,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.644836+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:39:51.644836+0000","last_peered":"2024-08-17T17:39:51.644836+0000","last_clean":"2024-08-17T17:39:51.644836+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:39:51.644836+0000","last_undegraded":"2024-08-17T17:39:51.644836+0000","last_fullsized":"2024-08-17T17:39:51.644836+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":346,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.476813+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:40:02.476813+0000","last_peered":"2024-08-17T17:40:02.476813+0000","last_clean":"2024-08-17T17:40:02.476813+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:40:02.476813+0000","last_undegraded":"2024-08-17T17:40:02.476813+0000","last_fullsized":"2024-08-17T17:40:02.476813+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":429,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.644949+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:39:51.644949+0000","last_peered":"2024-08-17T17:39:51.644949+0000","last_clean":"2024-08-17T17:39:51.644949+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:39:51.644949+0000","last_undegraded":"2024-08-17T17:39:51.644949+0000","last_fullsized":"2024-08-17T17:39:51.644949+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"377'9989","reported_seq":10144,"reported_epoch":377,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:05.428828+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:40:05.428828+0000","last_peered":"2024-08-17T17:40:05.428828+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:40:05.428828+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'3687","ondisk_log_start":"326'3687","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6302,"log_dups_size":0,"ondisk_log_size":6302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314245120,"num_objects":4795,"num_object_clones":0,"num_object_copies":14385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9989,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4795},{"shards":"0,3","objects":-2453}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":620,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.472029+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:40:02.472029+0000","last_peered":"2024-08-17T17:40:02.472029+0000","last_clean":"2024-08-17T17:40:02.472029+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:40:02.472029+0000","last_undegraded":"2024-08-17T17:40:02.472029+0000","last_fullsized":"2024-08-17T17:40:02.472029+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":588,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.464060+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:40:02.464060+0000","last_peered":"2024-08-17T17:40:02.464060+0000","last_clean":"2024-08-17T17:40:02.464060+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:40:02.464060+0000","last_undegraded":"2024-08-17T17:40:02.464060+0000","last_fullsized":"2024-08-17T17:40:02.464060+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":606,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.480526+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:40:02.480526+0000","last_peered":"2024-08-17T17:40:02.480526+0000","last_clean":"2024-08-17T17:40:02.480526+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:40:02.480526+0000","last_undegraded":"2024-08-17T17:40:02.480526+0000","last_fullsized":"2024-08-17T17:40:02.480526+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"377'9877","reported_seq":24551,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:06.701595+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:40:06.701595+0000","last_peered":"2024-08-17T17:40:06.701595+0000","last_clean":"2024-08-17T17:40:06.701595+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:40:06.701595+0000","last_undegraded":"2024-08-17T17:40:06.701595+0000","last_fullsized":"2024-08-17T17:40:06.701595+0000","mapping_epoch":347,"log_start":"325'3272","ondisk_log_start":"325'3272","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307822592,"num_objects":4697,"num_object_clones":0,"num_object_copies":14091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9877,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":279,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.477461+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:40:02.477461+0000","last_peered":"2024-08-17T17:40:02.477461+0000","last_clean":"2024-08-17T17:40:02.477461+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:40:02.477461+0000","last_undegraded":"2024-08-17T17:40:02.477461+0000","last_fullsized":"2024-08-17T17:40:02.477461+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":381,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.611015+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:39:51.611015+0000","last_peered":"2024-08-17T17:39:51.611015+0000","last_clean":"2024-08-17T17:39:51.611015+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:39:51.611015+0000","last_undegraded":"2024-08-17T17:39:51.611015+0000","last_fullsized":"2024-08-17T17:39:51.611015+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":331,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481577+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:39:51.481577+0000","last_peered":"2024-08-17T17:39:51.481577+0000","last_clean":"2024-08-17T17:39:51.481577+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:39:51.481577+0000","last_undegraded":"2024-08-17T17:39:51.481577+0000","last_fullsized":"2024-08-17T17:39:51.481577+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":331,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481928+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:39:51.481928+0000","last_peered":"2024-08-17T17:39:51.481928+0000","last_clean":"2024-08-17T17:39:51.481928+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:39:51.481928+0000","last_undegraded":"2024-08-17T17:39:51.481928+0000","last_fullsized":"2024-08-17T17:39:51.481928+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":374,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481648+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:39:51.481648+0000","last_peered":"2024-08-17T17:39:51.481648+0000","last_clean":"2024-08-17T17:39:51.481648+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:39:51.481648+0000","last_undegraded":"2024-08-17T17:39:51.481648+0000","last_fullsized":"2024-08-17T17:39:51.481648+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":595,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.462960+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:40:02.462960+0000","last_peered":"2024-08-17T17:40:02.462960+0000","last_clean":"2024-08-17T17:40:02.462960+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:40:02.462960+0000","last_undegraded":"2024-08-17T17:40:02.462960+0000","last_fullsized":"2024-08-17T17:40:02.462960+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":135,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.472349+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:40:02.472349+0000","last_peered":"2024-08-17T17:40:02.472349+0000","last_clean":"2024-08-17T17:40:02.472349+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:40:02.472349+0000","last_undegraded":"2024-08-17T17:40:02.472349+0000","last_fullsized":"2024-08-17T17:40:02.472349+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":281,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.644724+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:39:51.644724+0000","last_peered":"2024-08-17T17:39:51.644724+0000","last_clean":"2024-08-17T17:39:51.644724+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:39:51.644724+0000","last_undegraded":"2024-08-17T17:39:51.644724+0000","last_fullsized":"2024-08-17T17:39:51.644724+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":553,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471026+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:40:02.471026+0000","last_peered":"2024-08-17T17:40:02.471026+0000","last_clean":"2024-08-17T17:40:02.471026+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:40:02.471026+0000","last_undegraded":"2024-08-17T17:40:02.471026+0000","last_fullsized":"2024-08-17T17:40:02.471026+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":425,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.645654+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:39:51.645654+0000","last_peered":"2024-08-17T17:39:51.645654+0000","last_clean":"2024-08-17T17:39:51.645654+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:39:51.645654+0000","last_undegraded":"2024-08-17T17:39:51.645654+0000","last_fullsized":"2024-08-17T17:39:51.645654+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":547,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.474082+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:40:02.474082+0000","last_peered":"2024-08-17T17:40:02.474082+0000","last_clean":"2024-08-17T17:40:02.474082+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:40:02.474082+0000","last_undegraded":"2024-08-17T17:40:02.474082+0000","last_fullsized":"2024-08-17T17:40:02.474082+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":348,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.610952+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:39:51.610952+0000","last_peered":"2024-08-17T17:39:51.610952+0000","last_clean":"2024-08-17T17:39:51.610952+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:39:51.610952+0000","last_undegraded":"2024-08-17T17:39:51.610952+0000","last_fullsized":"2024-08-17T17:39:51.610952+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":579,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.480641+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:40:02.480641+0000","last_peered":"2024-08-17T17:40:02.480641+0000","last_clean":"2024-08-17T17:40:02.480641+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:40:02.480641+0000","last_undegraded":"2024-08-17T17:40:02.480641+0000","last_fullsized":"2024-08-17T17:40:02.480641+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":295,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471695+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:40:02.471695+0000","last_peered":"2024-08-17T17:40:02.471695+0000","last_clean":"2024-08-17T17:40:02.471695+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:40:02.471695+0000","last_undegraded":"2024-08-17T17:40:02.471695+0000","last_fullsized":"2024-08-17T17:40:02.471695+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":587,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.462514+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:40:02.462514+0000","last_peered":"2024-08-17T17:40:02.462514+0000","last_clean":"2024-08-17T17:40:02.462514+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:40:02.462514+0000","last_undegraded":"2024-08-17T17:40:02.462514+0000","last_fullsized":"2024-08-17T17:40:02.462514+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":546,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.472171+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:40:02.472171+0000","last_peered":"2024-08-17T17:40:02.472171+0000","last_clean":"2024-08-17T17:40:02.472171+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:40:02.472171+0000","last_undegraded":"2024-08-17T17:40:02.472171+0000","last_fullsized":"2024-08-17T17:40:02.472171+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":360,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481622+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:39:51.481622+0000","last_peered":"2024-08-17T17:39:51.481622+0000","last_clean":"2024-08-17T17:39:51.481622+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:39:51.481622+0000","last_undegraded":"2024-08-17T17:39:51.481622+0000","last_fullsized":"2024-08-17T17:39:51.481622+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":235,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471498+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:40:02.471498+0000","last_peered":"2024-08-17T17:40:02.471498+0000","last_clean":"2024-08-17T17:40:02.471498+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:40:02.471498+0000","last_undegraded":"2024-08-17T17:40:02.471498+0000","last_fullsized":"2024-08-17T17:40:02.471498+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":292,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.480222+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:40:02.480222+0000","last_peered":"2024-08-17T17:40:02.480222+0000","last_clean":"2024-08-17T17:40:02.480222+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:40:02.480222+0000","last_undegraded":"2024-08-17T17:40:02.480222+0000","last_fullsized":"2024-08-17T17:40:02.480222+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":368,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481729+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:39:51.481729+0000","last_peered":"2024-08-17T17:39:51.481729+0000","last_clean":"2024-08-17T17:39:51.481729+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:39:51.481729+0000","last_undegraded":"2024-08-17T17:39:51.481729+0000","last_fullsized":"2024-08-17T17:39:51.481729+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":231,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481612+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:39:51.481612+0000","last_peered":"2024-08-17T17:39:51.481612+0000","last_clean":"2024-08-17T17:39:51.481612+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:39:51.481612+0000","last_undegraded":"2024-08-17T17:39:51.481612+0000","last_fullsized":"2024-08-17T17:39:51.481612+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":368,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481424+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:39:51.481424+0000","last_peered":"2024-08-17T17:39:51.481424+0000","last_clean":"2024-08-17T17:39:51.481424+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:39:51.481424+0000","last_undegraded":"2024-08-17T17:39:51.481424+0000","last_fullsized":"2024-08-17T17:39:51.481424+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":231,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481841+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:39:51.481841+0000","last_peered":"2024-08-17T17:39:51.481841+0000","last_clean":"2024-08-17T17:39:51.481841+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:39:51.481841+0000","last_undegraded":"2024-08-17T17:39:51.481841+0000","last_fullsized":"2024-08-17T17:39:51.481841+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":599,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471481+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:40:02.471481+0000","last_peered":"2024-08-17T17:40:02.471481+0000","last_clean":"2024-08-17T17:40:02.471481+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:40:02.471481+0000","last_undegraded":"2024-08-17T17:40:02.471481+0000","last_fullsized":"2024-08-17T17:40:02.471481+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":572,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.471860+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:40:02.471860+0000","last_peered":"2024-08-17T17:40:02.471860+0000","last_clean":"2024-08-17T17:40:02.471860+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:40:02.471860+0000","last_undegraded":"2024-08-17T17:40:02.471860+0000","last_fullsized":"2024-08-17T17:40:02.471860+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":331,"reported_epoch":376,"state":"active+clean","last_fresh":"2024-08-17T17:39:51.481490+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:39:51.481490+0000","last_peered":"2024-08-17T17:39:51.481490+0000","last_clean":"2024-08-17T17:39:51.481490+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:39:51.481490+0000","last_undegraded":"2024-08-17T17:39:51.481490+0000","last_fullsized":"2024-08-17T17:39:51.481490+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":294,"reported_epoch":377,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.477107+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:40:02.477107+0000","last_peered":"2024-08-17T17:40:02.477107+0000","last_clean":"2024-08-17T17:40:02.477107+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:40:02.477107+0000","last_undegraded":"2024-08-17T17:40:02.477107+0000","last_fullsized":"2024-08-17T17:40:02.477107+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5026086936,"num_objects":76693,"num_object_clones":0,"num_object_copies":230079,"num_objects_missing_on_primary":5460,"num_objects_missing":5460,"num_objects_degraded":53998,"num_objects_misplaced":30327,"num_objects_unfound":0,"num_objects_dirty":76693,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159697,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27898,"num_bytes_recovered":1705771008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16482840576,"data_stored":16482828360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104816,"ondisk_log_size":104816,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170084,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463454,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3043816,"kb_used_data":2154272,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90701336,"statfs":{"total":95995035648,"available":92878168064,"internally_reserved":0,"allocated":2205974528,"data_stored":2203936328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492129,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011980,"kb_used_data":616644,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92733172,"statfs":{"total":95995035648,"available":94958768128,"internally_reserved":0,"allocated":631443456,"data_stored":629427556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986720,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2418524,"kb_used_data":1225044,"kb_used_omap":1,"kb_used_meta":1193470,"kb_avail":91326628,"statfs":{"total":95995035648,"available":93518467072,"internally_reserved":0,"allocated":1254445056,"data_stored":1252417796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1222113624},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225547,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3389068,"kb_used_data":2459460,"kb_used_omap":4,"kb_used_meta":929595,"kb_avail":90356084,"statfs":{"total":95995035648,"available":92524630016,"internally_reserved":0,"allocated":2518487040,"data_stored":2516471438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":951905728},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075873,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533766,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3196752,"kb_used_data":2150600,"kb_used_omap":5,"kb_used_meta":1046138,"kb_avail":90548400,"statfs":{"total":95995035648,"available":92721561600,"internally_reserved":0,"allocated":2202214400,"data_stored":2200193476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1071245552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089743,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2852008,"kb_used_data":1842784,"kb_used_omap":5,"kb_used_meta":1009210,"kb_avail":90893144,"statfs":{"total":95995035648,"available":93074579456,"internally_reserved":0,"allocated":1887010816,"data_stored":1884972675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1033431099},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1878982656,"data_stored":1878982656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2195918848,"data_stored":2195914776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3359707136,"data_stored":3359703064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2509504512,"data_stored":2509504512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1248460800,"data_stored":1248460800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":625737728,"data_stored":625737728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2198667264,"data_stored":2198667264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2465861632,"data_stored":2465857560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:13.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:40:13.334 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:13.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:13.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:13.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:13.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:13.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:13.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:13.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:13 smithi142 ceph-mon[106729]: pgmap v1406: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 54001/229236 objects degraded (23.557%); 30285/229236 objects misplaced (13.211%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:13 smithi142 ceph-mon[103614]: pgmap v1406: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 54001/229236 objects degraded (23.557%); 30285/229236 objects misplaced (13.211%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:13.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:13.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:13 smithi152 ceph-mon[93456]: pgmap v1406: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 54001/229236 objects degraded (23.557%); 30285/229236 objects misplaced (13.211%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:13.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:13.853 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:13.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:13.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:14.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:14.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:14.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:14.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:14.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:14.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:14.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:14.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:14.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:14.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:14.603 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:14 smithi142 ceph-mon[106729]: pgmap v1407: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 53929/228738 objects degraded (23.577%); 30119/228738 objects misplaced (13.167%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:40:14.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:14 smithi142 ceph-mon[103614]: pgmap v1407: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 53929/228738 objects degraded (23.577%); 30119/228738 objects misplaced (13.167%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:40:14.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:14.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:14 smithi152 ceph-mon[93456]: pgmap v1407: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 53929/228738 objects degraded (23.577%); 30119/228738 objects misplaced (13.167%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:40:14.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:14.853 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:14.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:14.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:15.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:15.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:15.203 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:15.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:15.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:15.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:15.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:15.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:15.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:15.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:15.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:15.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:15.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:15.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:15.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:15.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:16.035 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53929/228738 objects degraded (23.577%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:16.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53929/228738 objects degraded (23.577%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:16.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53929/228738 objects degraded (23.577%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:16.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:16.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:16.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:16.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:16.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:16.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:16.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:16.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:16.471 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:16.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:16.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:16.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:16.724 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:16.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:16.742 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:40:16.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:16.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:16.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:16.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:16.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:17.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:16 smithi152 ceph-mon[93456]: pgmap v1408: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 53929/228201 objects degraded (23.632%); 30119/228201 objects misplaced (13.198%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:40:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2592643530' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:17.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:17.098 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:16 smithi142 ceph-mon[106729]: pgmap v1408: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 53929/228201 objects degraded (23.632%); 30119/228201 objects misplaced (13.198%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:40:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2592643530' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:16 smithi142 ceph-mon[103614]: pgmap v1408: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 53929/228201 objects degraded (23.632%); 30119/228201 objects misplaced (13.198%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:40:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2592643530' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:17.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:17.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:17.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:17.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:17.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:17.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:40:17.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:17.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:17.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:17.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:17.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:17.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:17.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:17.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:17 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2592643530' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:17 smithi142 ceph-mon[106729]: osdmap e378: 8 total, 8 up, 6 in 2024-08-17T17:40:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:17 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2592643530' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:17 smithi142 ceph-mon[103614]: osdmap e378: 8 total, 8 up, 6 in 2024-08-17T17:40:17.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:17.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:17.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:18.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:17 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2592643530' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:18.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:17 smithi152 ceph-mon[93456]: osdmap e378: 8 total, 8 up, 6 in 2024-08-17T17:40:18.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:18.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:18.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:18.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:18.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:18.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:18.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:18.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:18.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:18.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:18 smithi142 ceph-mon[106729]: pgmap v1410: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 53929/227598 objects degraded (23.695%); 30082/227598 objects misplaced (13.217%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:18 smithi142 ceph-mon[103614]: pgmap v1410: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 53929/227598 objects degraded (23.695%); 30082/227598 objects misplaced (13.217%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:18.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:18.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:19.067 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:19.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:18 smithi152 ceph-mon[93456]: pgmap v1410: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 53929/227598 objects degraded (23.695%); 30082/227598 objects misplaced (13.217%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:40:19.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:40:19.104 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:40:19.104 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:40:19.104 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:40:19.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:19.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:19.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:19.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:19.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:19.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:19.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:19.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:19.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:19.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:40:19.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:19.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:19.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:19.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:19.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:19.964 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:20.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:20.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:20.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:40:20.138 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:20.176 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:20.181 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:20.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:20.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:20.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:20.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:20.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:20.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:20.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:20.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:40:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:20.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:20.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:20.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:20.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:20.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:20.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:20.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:20.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:20.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:21.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:20 smithi152 ceph-mon[93456]: pgmap v1411: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 53845/226506 objects degraded (23.772%); 29846/226506 objects misplaced (13.177%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:21.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:21.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:21.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.083 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[106729]: pgmap v1411: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 53845/226506 objects degraded (23.772%); 29846/226506 objects misplaced (13.177%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:21.085 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.085 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.085 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:21.086 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:21.086 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:21.086 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:20 smithi142 ceph-mon[103614]: pgmap v1411: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 53845/226506 objects degraded (23.772%); 29846/226506 objects misplaced (13.177%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:21.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:21.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:21.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:21.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:21.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:21.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:21.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:21.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:21.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.759 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:21.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:21.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:22.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:22.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53929/227598 objects degraded (23.695%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:22.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:22.111 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53929/227598 objects degraded (23.695%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:22.111 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53929/227598 objects degraded (23.695%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:22.111 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:40:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:40:21] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:40:22.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:22.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:22.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:22.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:22.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:22.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:22.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:22.507 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:22.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:22.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:22.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:22.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:22.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:22.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:22.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:22.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:22.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:23.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:22 smithi152 ceph-mon[93456]: pgmap v1412: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 53845/225555 objects degraded (23.872%); 29807/225555 objects misplaced (13.215%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:23.080 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:23.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:22 smithi142 ceph-mon[106729]: pgmap v1412: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 53845/225555 objects degraded (23.872%); 29807/225555 objects misplaced (13.215%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:22 smithi142 ceph-mon[103614]: pgmap v1412: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 53845/225555 objects degraded (23.872%); 29807/225555 objects misplaced (13.215%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:23.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:23.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:23.353 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:23.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:23.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:23.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:23.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:23.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:23.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:23.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:23.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:23.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:24.099 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:40:24.100 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:24.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:24.108 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:40:24.108 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:24.123 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:24.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:24.132 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:40:24.133 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:24.149 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:40:24.149 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:24.164 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:40:24.164 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:24.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:40:24.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:24.196 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:40:24.197 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:24.212 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:40:24.212 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:24.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:24.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:24.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:24.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:24.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3123412830' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:24.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3123412830' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:24.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:24.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:24.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:24.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:24.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3123412830' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:24.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:24.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:24.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:24.745 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:24.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:24.847 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:24.868 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:24.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:24.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:25.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:25.117 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:25.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:25.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:25.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:25.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:25.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:25 smithi142 ceph-mon[106729]: pgmap v1413: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 53765/225000 objects degraded (23.896%); 29620/225000 objects misplaced (13.164%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:25.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:25 smithi142 ceph-mon[103614]: pgmap v1413: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 53765/225000 objects degraded (23.896%); 29620/225000 objects misplaced (13.164%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:25.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:25.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:25.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:25 smithi152 ceph-mon[93456]: pgmap v1413: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 53765/225000 objects degraded (23.896%); 29620/225000 objects misplaced (13.164%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:25.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:25.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:25.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:25.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:25.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:25.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:25.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:25.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:25.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:26.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:26.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:26.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:26.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:26.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:26.249 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53765/225000 objects degraded (23.896%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:26.249 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53765/225000 objects degraded (23.896%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:26.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:26.355 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:26.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:26.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:26.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53765/225000 objects degraded (23.896%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:26.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:26.599 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:40:26.599 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 53765/225000 objects degraded (23.896%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2107,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":378,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":75000,"data_bytes":4914026469,"bytes_used":16041365504,"bytes_avail":559928848384,"bytes_total":575970213888,"degraded_objects":53765,"degraded_total":225000,"degraded_ratio":0.23895555555555556,"misplaced_objects":29620,"misplaced_total":225000,"misplaced_ratio":0.13164444444444445,"recovering_objects_per_sec":23,"recovering_bytes_per_sec":1565846,"recovering_keys_per_sec":0,"num_objects_recovered":239,"num_bytes_recovered":15663104,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":124},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:40:26.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:26.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:26.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:26.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:26.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:26.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:26.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:27.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:27.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:27.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:27.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:27.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:27.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:27 smithi142 ceph-mon[106729]: pgmap v1414: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 53765/224334 objects degraded (23.966%); 29620/224334 objects misplaced (13.204%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:27.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/683629193' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:40:27.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:27 smithi142 ceph-mon[103614]: pgmap v1414: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 53765/224334 objects degraded (23.966%); 29620/224334 objects misplaced (13.204%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:27.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:27.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/683629193' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:40:27.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:27.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:27.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:27.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:27 smithi152 ceph-mon[93456]: pgmap v1414: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 53765/224334 objects degraded (23.966%); 29620/224334 objects misplaced (13.204%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:27.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:27.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/683629193' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:40:27.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:27.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:27.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:27.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:27.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:27.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:27.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:27.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:28.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:28.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:28.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:28.216 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:28.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:28.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:28.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:28.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:28.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:28.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:28.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:28.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:28.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:28.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:28.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:28.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:28.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:28.960 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:28.960 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:28.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:29.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:29.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:29.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:29.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:29.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:29.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:29.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:29.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:29.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:29.606 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:40:29.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:40:29.668 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:29 smithi142 ceph-mon[106729]: pgmap v1415: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 53765/223953 objects degraded (24.007%); 29596/223953 objects misplaced (13.215%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:29.668 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1411707' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:29.669 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:29.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:29 smithi142 ceph-mon[103614]: pgmap v1415: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 53765/223953 objects degraded (24.007%); 29596/223953 objects misplaced (13.215%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:29.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1411707' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:29.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:29.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:29.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:29.722 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:40:29.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:29 smithi152 ceph-mon[93456]: pgmap v1415: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 53765/223953 objects degraded (24.007%); 29596/223953 objects misplaced (13.215%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:40:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1411707' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:29.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:29.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:29.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:29.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:30.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:30.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:30.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:30.214 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:30.316 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:30.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:30.381 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:40:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:40:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:30.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:30.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:30.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:30 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:30.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:30.469 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:30 smithi142 ceph-mon[106729]: osdmap e379: 8 total, 8 up, 6 in 2024-08-17T17:40:30.469 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:30 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:30.469 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:30.469 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:30 smithi142 ceph-mon[103614]: osdmap e379: 8 total, 8 up, 6 in 2024-08-17T17:40:30.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:40:30.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:30.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:30.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:30.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:30.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:30.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:30 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:30 smithi152 ceph-mon[93456]: osdmap e379: 8 total, 8 up, 6 in 2024-08-17T17:40:30.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:30.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:30.961 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:30.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:30.969 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:31.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:31.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:31.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:31.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:31.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:31.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:31.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:31.476 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:31.510 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:31 smithi142 ceph-mon[106729]: pgmap v1417: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53701/223287 objects degraded (24.050%); 29449/223287 objects misplaced (13.189%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:31.511 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53701/223287 objects degraded (24.050%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:31.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:31 smithi142 ceph-mon[103614]: pgmap v1417: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53701/223287 objects degraded (24.050%); 29449/223287 objects misplaced (13.189%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:31.511 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53701/223287 objects degraded (24.050%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:31 smithi152 ceph-mon[93456]: pgmap v1417: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53701/223287 objects degraded (24.050%); 29449/223287 objects misplaced (13.189%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:31.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53701/223287 objects degraded (24.050%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:31.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:31.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:31.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:31.747 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:31.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:31.849 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:31.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:31.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:32.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:32.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:32.098 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:40:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:40:31] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:40:32.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:32.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:32.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:32.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:32.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:32.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:32.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:32.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:32.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:32.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:32.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:32.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:32.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:32.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:32.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:32.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:40:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:32.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:33.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:33.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:33.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:33.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:33.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:33.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:33.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:33.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:33.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.592 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:33 smithi142 ceph-mon[106729]: pgmap v1418: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 53701/222498 objects degraded (24.135%); 29404/222498 objects misplaced (13.215%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:33.592 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:33 smithi142 ceph-mon[103614]: pgmap v1418: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 53701/222498 objects degraded (24.135%); 29404/222498 objects misplaced (13.215%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:33.692 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:33.714 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:33.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:33 smithi152 ceph-mon[93456]: pgmap v1418: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 53701/222498 objects degraded (24.135%); 29404/222498 objects misplaced (13.215%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:33.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:33.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:33.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:34.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:34.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:34.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:34.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:34.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:34.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:34.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.594 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:34.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.673 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:40:34.674 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:34.694 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:40:34.695 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:34.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:34.717 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:40:34.717 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:34.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.735 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:40:34.735 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:34.755 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:40:34.756 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:34.775 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:40:34.775 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:34.792 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:40:34.792 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:34.815 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:40:34.815 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:34.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:34.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:34.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:35.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:35.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:35.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:35.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:35.214 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:35.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:35.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:35.366 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:35.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:35.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:35.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:35 smithi142 ceph-mon[106729]: pgmap v1419: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 53633/221979 objects degraded (24.161%); 29229/221979 objects misplaced (13.167%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:35.512 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/883297265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:35.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:35 smithi142 ceph-mon[103614]: pgmap v1419: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 53633/221979 objects degraded (24.161%); 29229/221979 objects misplaced (13.167%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:35.512 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/883297265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:35.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:35.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:35.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:35.719 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:35.820 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:35 smithi152 ceph-mon[93456]: pgmap v1419: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 53633/221979 objects degraded (24.161%); 29229/221979 objects misplaced (13.167%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/883297265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:35.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:35.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:35.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:35.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:36.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:36.085 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:40:36.086 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:40:36.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:36.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:36.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:36.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:36.340 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:36.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:36.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:36.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:36.590 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:36.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:36.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53633/221979 objects degraded (24.161%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:36.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53633/221979 objects degraded (24.161%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:36.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:36.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:36.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53633/221979 objects degraded (24.161%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:36.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:36.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:36.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:36.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:37.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:37.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:37.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:37.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:37.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:37.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:37.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:37.429 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:37 smithi142 ceph-mon[106729]: from='client.47399 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:37.429 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:37 smithi142 ceph-mon[106729]: pgmap v1420: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 53633/221241 objects degraded (24.242%); 29229/221241 objects misplaced (13.211%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:37.429 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:37 smithi142 ceph-mon[103614]: from='client.47399 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:37.430 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:37 smithi142 ceph-mon[103614]: pgmap v1420: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 53633/221241 objects degraded (24.242%); 29229/221241 objects misplaced (13.211%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:37.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:37.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:37.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:37.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:37.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:37.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:37.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:37.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:37.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:37.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:37 smithi152 ceph-mon[93456]: from='client.47399 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:37 smithi152 ceph-mon[93456]: pgmap v1420: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 53633/221241 objects degraded (24.242%); 29229/221241 objects misplaced (13.211%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:37.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:37.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:37.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:38.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:38.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:38.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:38.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:38.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:38.292 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1419,"stamp":"2024-08-17T17:40:34.328610+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4848031717,"num_objects":73993,"num_object_clones":0,"num_object_copies":221979,"num_objects_missing_on_primary":5279,"num_objects_missing":5279,"num_objects_degraded":53633,"num_objects_misplaced":29229,"num_objects_unfound":0,"num_objects_dirty":73993,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":162701,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28472,"num_bytes_recovered":1743069838,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104949,"ondisk_log_size":104949,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15544076,"kb_used_data":10080724,"kb_used_omap":27,"kb_used_meta":5463268,"kb_avail":546926836,"statfs":{"total":575970213888,"available":560053080064,"internally_reserved":0,"allocated":10322661376,"data_stored":10310419626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5594387176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-65994752,"num_objects":-1007,"num_object_clones":0,"num_object_copies":-3021,"num_objects_missing_on_primary":-74,"num_objects_missing":-74,"num_objects_degraded":-132,"num_objects_misplaced":-391,"num_objects_unfound":0,"num_objects_dirty":-1007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1007,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":193,"num_bytes_recovered":12648448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002528"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":600,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385163+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:40:29.385163+0000","last_peered":"2024-08-17T17:40:29.385163+0000","last_clean":"2024-08-17T17:40:29.385163+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:40:29.385163+0000","last_undegraded":"2024-08-17T17:40:29.385163+0000","last_fullsized":"2024-08-17T17:40:29.385163+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":88,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765232+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:40:16.765232+0000","last_peered":"2024-08-17T17:40:16.765232+0000","last_clean":"2024-08-17T17:40:16.765232+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:40:16.765232+0000","last_undegraded":"2024-08-17T17:40:16.765232+0000","last_fullsized":"2024-08-17T17:40:16.765232+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":282,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.801422+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:40:16.801422+0000","last_peered":"2024-08-17T17:40:16.801422+0000","last_clean":"2024-08-17T17:40:16.801422+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:40:16.801422+0000","last_undegraded":"2024-08-17T17:40:16.801422+0000","last_fullsized":"2024-08-17T17:40:16.801422+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":298,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.386348+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:40:29.386348+0000","last_peered":"2024-08-17T17:40:29.386348+0000","last_clean":"2024-08-17T17:40:29.386348+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:40:29.386348+0000","last_undegraded":"2024-08-17T17:40:29.386348+0000","last_fullsized":"2024-08-17T17:40:29.386348+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":475,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385351+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:40:29.385351+0000","last_peered":"2024-08-17T17:40:29.385351+0000","last_clean":"2024-08-17T17:40:29.385351+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:40:29.385351+0000","last_undegraded":"2024-08-17T17:40:29.385351+0000","last_fullsized":"2024-08-17T17:40:29.385351+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":231,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765096+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:40:16.765096+0000","last_peered":"2024-08-17T17:40:16.765096+0000","last_clean":"2024-08-17T17:40:16.765096+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:40:16.765096+0000","last_undegraded":"2024-08-17T17:40:16.765096+0000","last_fullsized":"2024-08-17T17:40:16.765096+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":569,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385622+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:40:29.385622+0000","last_peered":"2024-08-17T17:40:29.385622+0000","last_clean":"2024-08-17T17:40:29.385622+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:40:29.385622+0000","last_undegraded":"2024-08-17T17:40:29.385622+0000","last_fullsized":"2024-08-17T17:40:29.385622+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":399,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.750587+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:40:16.750587+0000","last_peered":"2024-08-17T17:40:16.750587+0000","last_clean":"2024-08-17T17:40:16.750587+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:40:16.750587+0000","last_undegraded":"2024-08-17T17:40:16.750587+0000","last_fullsized":"2024-08-17T17:40:16.750587+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"379'10100","reported_seq":14229,"reported_epoch":379,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:40:33.795338+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:40:33.795338+0000","last_peered":"2024-08-17T17:40:33.795338+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:40:33.795338+0000","last_undegraded":"2024-08-17T17:40:33.795338+0000","last_fullsized":"2024-08-17T17:40:33.795338+0000","mapping_epoch":337,"log_start":"325'3263","ondisk_log_start":"325'3263","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301727744,"num_objects":4604,"num_object_clones":0,"num_object_copies":13812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4604,"num_objects_unfound":0,"num_objects_dirty":4604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10100,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2019,"num_bytes_recovered":132317184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4604}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":84,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.810202+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:40:16.810202+0000","last_peered":"2024-08-17T17:40:16.810202+0000","last_clean":"2024-08-17T17:40:16.810202+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:40:16.810202+0000","last_undegraded":"2024-08-17T17:40:16.810202+0000","last_fullsized":"2024-08-17T17:40:16.810202+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":265,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.800960+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:40:16.800960+0000","last_peered":"2024-08-17T17:40:16.800960+0000","last_clean":"2024-08-17T17:40:16.800960+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:40:16.800960+0000","last_undegraded":"2024-08-17T17:40:16.800960+0000","last_fullsized":"2024-08-17T17:40:16.800960+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"379'10367","reported_seq":10444,"reported_epoch":379,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:31.829616+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:40:31.829616+0000","last_peered":"2024-08-17T17:40:31.829616+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:40:31.829616+0000","last_undegraded":"2024-08-17T17:40:31.829616+0000","last_fullsized":"2024-08-17T17:40:31.829616+0000","mapping_epoch":337,"log_start":"326'3782","ondisk_log_start":"326'3782","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308477952,"num_objects":4707,"num_object_clones":0,"num_object_copies":14121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4707,"num_objects_unfound":0,"num_objects_dirty":4707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10367,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4707}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":86,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765423+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:40:16.765423+0000","last_peered":"2024-08-17T17:40:16.765423+0000","last_clean":"2024-08-17T17:40:16.765423+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:40:16.765423+0000","last_undegraded":"2024-08-17T17:40:16.765423+0000","last_fullsized":"2024-08-17T17:40:16.765423+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":350,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765021+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:40:16.765021+0000","last_peered":"2024-08-17T17:40:16.765021+0000","last_clean":"2024-08-17T17:40:16.765021+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:40:16.765021+0000","last_undegraded":"2024-08-17T17:40:16.765021+0000","last_fullsized":"2024-08-17T17:40:16.765021+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"379'10127","reported_seq":10207,"reported_epoch":379,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:33.674432+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:40:33.674432+0000","last_peered":"2024-08-17T17:40:33.674432+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:40:33.674432+0000","last_undegraded":"2024-08-17T17:40:33.674432+0000","last_fullsized":"2024-08-17T17:40:33.674432+0000","mapping_epoch":337,"log_start":"325'3253","ondisk_log_start":"325'3253","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4607,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10127,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4607}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":82,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.801321+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:40:16.801321+0000","last_peered":"2024-08-17T17:40:16.801321+0000","last_clean":"2024-08-17T17:40:16.801321+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:40:16.801321+0000","last_undegraded":"2024-08-17T17:40:16.801321+0000","last_fullsized":"2024-08-17T17:40:16.801321+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":360,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.750760+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:40:16.750760+0000","last_peered":"2024-08-17T17:40:16.750760+0000","last_clean":"2024-08-17T17:40:16.750760+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:40:16.750760+0000","last_undegraded":"2024-08-17T17:40:16.750760+0000","last_fullsized":"2024-08-17T17:40:16.750760+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"378'10284","reported_seq":10422,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:29.508246+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:40:29.508246+0000","last_peered":"2024-08-17T17:40:29.508246+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:40:29.508246+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'3213","ondisk_log_start":"325'3213","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306315264,"num_objects":4674,"num_object_clones":0,"num_object_copies":14022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10284,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2095},{"shards":"0,3,7","objects":4674}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":512,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385314+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:40:29.385314+0000","last_peered":"2024-08-17T17:40:29.385314+0000","last_clean":"2024-08-17T17:40:29.385314+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:40:29.385314+0000","last_undegraded":"2024-08-17T17:40:29.385314+0000","last_fullsized":"2024-08-17T17:40:29.385314+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":279,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.508860+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:40:29.508860+0000","last_peered":"2024-08-17T17:40:29.508860+0000","last_clean":"2024-08-17T17:40:29.508860+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:40:29.508860+0000","last_undegraded":"2024-08-17T17:40:29.508860+0000","last_fullsized":"2024-08-17T17:40:29.508860+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"379'10023","reported_seq":10091,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:31.994052+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:40:31.994052+0000","last_peered":"2024-08-17T17:40:31.994052+0000","last_clean":"2024-08-17T17:40:31.994052+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:40:31.994052+0000","last_undegraded":"2024-08-17T17:40:31.994052+0000","last_fullsized":"2024-08-17T17:40:31.994052+0000","mapping_epoch":319,"log_start":"326'3474","ondisk_log_start":"326'3474","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565056,"num_objects":4571,"num_object_clones":0,"num_object_copies":13713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10023,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":602,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385988+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:40:29.385988+0000","last_peered":"2024-08-17T17:40:29.385988+0000","last_clean":"2024-08-17T17:40:29.385988+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:40:29.385988+0000","last_undegraded":"2024-08-17T17:40:29.385988+0000","last_fullsized":"2024-08-17T17:40:29.385988+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":333,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765403+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:40:16.765403+0000","last_peered":"2024-08-17T17:40:16.765403+0000","last_clean":"2024-08-17T17:40:16.765403+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:40:16.765403+0000","last_undegraded":"2024-08-17T17:40:16.765403+0000","last_fullsized":"2024-08-17T17:40:16.765403+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"379'10194","reported_seq":10272,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:31.917293+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:40:31.917293+0000","last_peered":"2024-08-17T17:40:31.917293+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:40:31.917293+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'3635","ondisk_log_start":"326'3635","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310509568,"num_objects":4738,"num_object_clones":0,"num_object_copies":14214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10194,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4738},{"shards":"1,6","objects":-2628}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":334,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.750916+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:40:16.750916+0000","last_peered":"2024-08-17T17:40:16.750916+0000","last_clean":"2024-08-17T17:40:16.750916+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:40:16.750916+0000","last_undegraded":"2024-08-17T17:40:16.750916+0000","last_fullsized":"2024-08-17T17:40:16.750916+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":360,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765917+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:40:16.765917+0000","last_peered":"2024-08-17T17:40:16.765917+0000","last_clean":"2024-08-17T17:40:16.765917+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:40:16.765917+0000","last_undegraded":"2024-08-17T17:40:16.765917+0000","last_fullsized":"2024-08-17T17:40:16.765917+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"379'10352","reported_seq":10491,"reported_epoch":379,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:33.157405+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:40:33.157405+0000","last_peered":"2024-08-17T17:40:33.157405+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:40:33.157405+0000","last_undegraded":"2024-08-17T17:40:33.157405+0000","last_fullsized":"2024-08-17T17:40:33.157405+0000","mapping_epoch":337,"log_start":"326'4761","ondisk_log_start":"326'4761","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305266688,"num_objects":4658,"num_object_clones":0,"num_object_copies":13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4658,"num_objects_unfound":0,"num_objects_dirty":4658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10352,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4658}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":371,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765276+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:40:16.765276+0000","last_peered":"2024-08-17T17:40:16.765276+0000","last_clean":"2024-08-17T17:40:16.765276+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:40:16.765276+0000","last_undegraded":"2024-08-17T17:40:16.765276+0000","last_fullsized":"2024-08-17T17:40:16.765276+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":333,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765601+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:40:16.765601+0000","last_peered":"2024-08-17T17:40:16.765601+0000","last_clean":"2024-08-17T17:40:16.765601+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:40:16.765601+0000","last_undegraded":"2024-08-17T17:40:16.765601+0000","last_fullsized":"2024-08-17T17:40:16.765601+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"379'10117","reported_seq":14334,"reported_epoch":379,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:40:33.788175+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:40:33.788175+0000","last_peered":"2024-08-17T17:40:33.788175+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:40:33.788175+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:40:33.788175+0000","mapping_epoch":337,"log_start":"325'3241","ondisk_log_start":"325'3241","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300613632,"num_objects":4587,"num_object_clones":0,"num_object_copies":13761,"num_objects_missing_on_primary":5279,"num_objects_missing":5279,"num_objects_degraded":4587,"num_objects_misplaced":5279,"num_objects_unfound":0,"num_objects_dirty":4587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10117,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2059,"num_bytes_recovered":134873088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":14},{"shards":"1,2,7","objects":4573}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":571,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385605+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:40:29.385605+0000","last_peered":"2024-08-17T17:40:29.385605+0000","last_clean":"2024-08-17T17:40:29.385605+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:40:29.385605+0000","last_undegraded":"2024-08-17T17:40:29.385605+0000","last_fullsized":"2024-08-17T17:40:29.385605+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":508,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385934+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:40:29.385934+0000","last_peered":"2024-08-17T17:40:29.385934+0000","last_clean":"2024-08-17T17:40:29.385934+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:40:29.385934+0000","last_undegraded":"2024-08-17T17:40:29.385934+0000","last_fullsized":"2024-08-17T17:40:29.385934+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"379'10272","reported_seq":14548,"reported_epoch":379,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:40:33.663667+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:40:33.663667+0000","last_peered":"2024-08-17T17:40:33.663667+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:40:33.663667+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:40:33.663667+0000","mapping_epoch":337,"log_start":"326'4644","ondisk_log_start":"326'4644","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5628,"log_dups_size":0,"ondisk_log_size":5628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302972952,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4624,"num_objects_misplaced":5374,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10272,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2074,"num_bytes_recovered":135856128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4624}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":702,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:29.507789+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:40:29.507789+0000","last_peered":"2024-08-17T17:40:29.507789+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:40:29.507789+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":308,"reported_epoch":378,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:40:16.801363+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:40:16.801363+0000","last_peered":"2024-08-17T17:40:16.801363+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:40:16.801363+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:40:16.801363+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":551,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.386262+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:40:29.386262+0000","last_peered":"2024-08-17T17:40:29.386262+0000","last_clean":"2024-08-17T17:40:29.386262+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:40:29.386262+0000","last_undegraded":"2024-08-17T17:40:29.386262+0000","last_fullsized":"2024-08-17T17:40:29.386262+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":343,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.751575+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:40:16.751575+0000","last_peered":"2024-08-17T17:40:16.751575+0000","last_clean":"2024-08-17T17:40:16.751575+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:40:16.751575+0000","last_undegraded":"2024-08-17T17:40:16.751575+0000","last_fullsized":"2024-08-17T17:40:16.751575+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"379'10092","reported_seq":10232,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:29.514740+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:40:29.514740+0000","last_peered":"2024-08-17T17:40:29.514740+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:40:29.514740+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'3021","ondisk_log_start":"325'3021","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304873472,"num_objects":4652,"num_object_clones":0,"num_object_copies":13956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10092,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-1995},{"shards":"0,4,7","objects":4652}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":367,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385235+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:40:29.385235+0000","last_peered":"2024-08-17T17:40:29.385235+0000","last_clean":"2024-08-17T17:40:29.385235+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:40:29.385235+0000","last_undegraded":"2024-08-17T17:40:29.385235+0000","last_fullsized":"2024-08-17T17:40:29.385235+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":315,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.764942+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:40:16.764942+0000","last_peered":"2024-08-17T17:40:16.764942+0000","last_clean":"2024-08-17T17:40:16.764942+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:40:16.764942+0000","last_undegraded":"2024-08-17T17:40:16.764942+0000","last_fullsized":"2024-08-17T17:40:16.764942+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":293,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.508475+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:40:29.508475+0000","last_peered":"2024-08-17T17:40:29.508475+0000","last_clean":"2024-08-17T17:40:29.508475+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:40:29.508475+0000","last_undegraded":"2024-08-17T17:40:29.508475+0000","last_fullsized":"2024-08-17T17:40:29.508475+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"379'10133","reported_seq":10211,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:31.538047+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:40:31.538047+0000","last_peered":"2024-08-17T17:40:31.538047+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:40:31.538047+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"325'3517","ondisk_log_start":"325'3517","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305594368,"num_objects":4663,"num_object_clones":0,"num_object_copies":13989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10133,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4663},{"shards":"5,6","objects":-2633}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":459,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385766+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:40:29.385766+0000","last_peered":"2024-08-17T17:40:29.385766+0000","last_clean":"2024-08-17T17:40:29.385766+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:40:29.385766+0000","last_undegraded":"2024-08-17T17:40:29.385766+0000","last_fullsized":"2024-08-17T17:40:29.385766+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":115,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.750463+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:40:16.750463+0000","last_peered":"2024-08-17T17:40:16.750463+0000","last_clean":"2024-08-17T17:40:16.750463+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:40:16.750463+0000","last_undegraded":"2024-08-17T17:40:16.750463+0000","last_fullsized":"2024-08-17T17:40:16.750463+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":644,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.387986+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:40:29.387986+0000","last_peered":"2024-08-17T17:40:29.387986+0000","last_clean":"2024-08-17T17:40:29.387986+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:40:29.387986+0000","last_undegraded":"2024-08-17T17:40:29.387986+0000","last_fullsized":"2024-08-17T17:40:29.387986+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"378'10034","reported_seq":10898,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:29.508613+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:40:29.508613+0000","last_peered":"2024-08-17T17:40:29.508613+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:40:29.508613+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'3016","ondisk_log_start":"325'3016","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302252032,"num_objects":4612,"num_object_clones":0,"num_object_copies":13836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10034,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4612},{"shards":"0,5","objects":-2042}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":435,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385723+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:40:29.385723+0000","last_peered":"2024-08-17T17:40:29.385723+0000","last_clean":"2024-08-17T17:40:29.385723+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:40:29.385723+0000","last_undegraded":"2024-08-17T17:40:29.385723+0000","last_fullsized":"2024-08-17T17:40:29.385723+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":354,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.771678+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:40:16.771678+0000","last_peered":"2024-08-17T17:40:16.771678+0000","last_clean":"2024-08-17T17:40:16.771678+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:40:16.771678+0000","last_undegraded":"2024-08-17T17:40:16.771678+0000","last_fullsized":"2024-08-17T17:40:16.771678+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":230,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385094+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:40:29.385094+0000","last_peered":"2024-08-17T17:40:29.385094+0000","last_clean":"2024-08-17T17:40:29.385094+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:40:29.385094+0000","last_undegraded":"2024-08-17T17:40:29.385094+0000","last_fullsized":"2024-08-17T17:40:29.385094+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"379'10247","reported_seq":25231,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:30.533080+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:40:30.533080+0000","last_peered":"2024-08-17T17:40:30.533080+0000","last_clean":"2024-08-17T17:40:30.533080+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:40:30.533080+0000","last_undegraded":"2024-08-17T17:40:30.533080+0000","last_fullsized":"2024-08-17T17:40:30.533080+0000","mapping_epoch":355,"log_start":"326'3930","ondisk_log_start":"326'3930","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6317,"log_dups_size":0,"ondisk_log_size":6317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300744704,"num_objects":4589,"num_object_clones":0,"num_object_copies":13767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10247,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":293,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.801241+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:40:16.801241+0000","last_peered":"2024-08-17T17:40:16.801241+0000","last_clean":"2024-08-17T17:40:16.801241+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:40:16.801241+0000","last_undegraded":"2024-08-17T17:40:16.801241+0000","last_fullsized":"2024-08-17T17:40:16.801241+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385017+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:40:29.385017+0000","last_peered":"2024-08-17T17:40:29.385017+0000","last_clean":"2024-08-17T17:40:29.385017+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:40:29.385017+0000","last_undegraded":"2024-08-17T17:40:29.385017+0000","last_fullsized":"2024-08-17T17:40:29.385017+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":594,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.386009+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:40:29.386009+0000","last_peered":"2024-08-17T17:40:29.386009+0000","last_clean":"2024-08-17T17:40:29.386009+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:40:29.386009+0000","last_undegraded":"2024-08-17T17:40:29.386009+0000","last_fullsized":"2024-08-17T17:40:29.386009+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"379'9886","reported_seq":24437,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:30.238411+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:40:30.238411+0000","last_peered":"2024-08-17T17:40:30.238411+0000","last_clean":"2024-08-17T17:40:30.238411+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:40:30.238411+0000","last_undegraded":"2024-08-17T17:40:30.238411+0000","last_fullsized":"2024-08-17T17:40:30.238411+0000","mapping_epoch":345,"log_start":"326'3572","ondisk_log_start":"326'3572","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6314,"log_dups_size":0,"ondisk_log_size":6314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296091648,"num_objects":4518,"num_object_clones":0,"num_object_copies":13554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9886,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":289,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.801783+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:40:16.801783+0000","last_peered":"2024-08-17T17:40:16.801783+0000","last_clean":"2024-08-17T17:40:16.801783+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:40:16.801783+0000","last_undegraded":"2024-08-17T17:40:16.801783+0000","last_fullsized":"2024-08-17T17:40:16.801783+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":350,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.508065+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:40:29.508065+0000","last_peered":"2024-08-17T17:40:29.508065+0000","last_clean":"2024-08-17T17:40:29.508065+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:40:29.508065+0000","last_undegraded":"2024-08-17T17:40:29.508065+0000","last_fullsized":"2024-08-17T17:40:29.508065+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":431,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.801239+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:40:16.801239+0000","last_peered":"2024-08-17T17:40:16.801239+0000","last_clean":"2024-08-17T17:40:16.801239+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:40:16.801239+0000","last_undegraded":"2024-08-17T17:40:16.801239+0000","last_fullsized":"2024-08-17T17:40:16.801239+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"379'10160","reported_seq":10320,"reported_epoch":379,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:30.083327+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:40:30.083327+0000","last_peered":"2024-08-17T17:40:30.083327+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:40:30.083327+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'3887","ondisk_log_start":"326'3887","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10160,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4624},{"shards":"0,3","objects":-2624}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":624,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.386010+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:40:29.386010+0000","last_peered":"2024-08-17T17:40:29.386010+0000","last_clean":"2024-08-17T17:40:29.386010+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:40:29.386010+0000","last_undegraded":"2024-08-17T17:40:29.386010+0000","last_fullsized":"2024-08-17T17:40:29.386010+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":593,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.387591+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:40:29.387591+0000","last_peered":"2024-08-17T17:40:29.387591+0000","last_clean":"2024-08-17T17:40:29.387591+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:40:29.387591+0000","last_undegraded":"2024-08-17T17:40:29.387591+0000","last_fullsized":"2024-08-17T17:40:29.387591+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":610,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.384875+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:40:29.384875+0000","last_peered":"2024-08-17T17:40:29.384875+0000","last_clean":"2024-08-17T17:40:29.384875+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:40:29.384875+0000","last_undegraded":"2024-08-17T17:40:29.384875+0000","last_fullsized":"2024-08-17T17:40:29.384875+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"379'10035","reported_seq":24711,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:31.598586+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:40:31.598586+0000","last_peered":"2024-08-17T17:40:31.598586+0000","last_clean":"2024-08-17T17:40:31.598586+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:40:31.598586+0000","last_undegraded":"2024-08-17T17:40:31.598586+0000","last_fullsized":"2024-08-17T17:40:31.598586+0000","mapping_epoch":347,"log_start":"325'3472","ondisk_log_start":"325'3472","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10035,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":283,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.508385+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:40:29.508385+0000","last_peered":"2024-08-17T17:40:29.508385+0000","last_clean":"2024-08-17T17:40:29.508385+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:40:29.508385+0000","last_undegraded":"2024-08-17T17:40:29.508385+0000","last_fullsized":"2024-08-17T17:40:29.508385+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":383,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.751758+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:40:16.751758+0000","last_peered":"2024-08-17T17:40:16.751758+0000","last_clean":"2024-08-17T17:40:16.751758+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:40:16.751758+0000","last_undegraded":"2024-08-17T17:40:16.751758+0000","last_fullsized":"2024-08-17T17:40:16.751758+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":333,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765195+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:40:16.765195+0000","last_peered":"2024-08-17T17:40:16.765195+0000","last_clean":"2024-08-17T17:40:16.765195+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:40:16.765195+0000","last_undegraded":"2024-08-17T17:40:16.765195+0000","last_fullsized":"2024-08-17T17:40:16.765195+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":333,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.771550+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:40:16.771550+0000","last_peered":"2024-08-17T17:40:16.771550+0000","last_clean":"2024-08-17T17:40:16.771550+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:40:16.771550+0000","last_undegraded":"2024-08-17T17:40:16.771550+0000","last_fullsized":"2024-08-17T17:40:16.771550+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":376,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765570+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:40:16.765570+0000","last_peered":"2024-08-17T17:40:16.765570+0000","last_clean":"2024-08-17T17:40:16.765570+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:40:16.765570+0000","last_undegraded":"2024-08-17T17:40:16.765570+0000","last_fullsized":"2024-08-17T17:40:16.765570+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":600,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385848+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:40:29.385848+0000","last_peered":"2024-08-17T17:40:29.385848+0000","last_clean":"2024-08-17T17:40:29.385848+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:40:29.385848+0000","last_undegraded":"2024-08-17T17:40:29.385848+0000","last_fullsized":"2024-08-17T17:40:29.385848+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":139,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.386024+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:40:29.386024+0000","last_peered":"2024-08-17T17:40:29.386024+0000","last_clean":"2024-08-17T17:40:29.386024+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:40:29.386024+0000","last_undegraded":"2024-08-17T17:40:29.386024+0000","last_fullsized":"2024-08-17T17:40:29.386024+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":283,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.801124+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:40:16.801124+0000","last_peered":"2024-08-17T17:40:16.801124+0000","last_clean":"2024-08-17T17:40:16.801124+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:40:16.801124+0000","last_undegraded":"2024-08-17T17:40:16.801124+0000","last_fullsized":"2024-08-17T17:40:16.801124+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":557,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385286+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:40:29.385286+0000","last_peered":"2024-08-17T17:40:29.385286+0000","last_clean":"2024-08-17T17:40:29.385286+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:40:29.385286+0000","last_undegraded":"2024-08-17T17:40:29.385286+0000","last_fullsized":"2024-08-17T17:40:29.385286+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":427,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.802740+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:40:16.802740+0000","last_peered":"2024-08-17T17:40:16.802740+0000","last_clean":"2024-08-17T17:40:16.802740+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:40:16.802740+0000","last_undegraded":"2024-08-17T17:40:16.802740+0000","last_fullsized":"2024-08-17T17:40:16.802740+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":551,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.384773+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:40:29.384773+0000","last_peered":"2024-08-17T17:40:29.384773+0000","last_clean":"2024-08-17T17:40:29.384773+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:40:29.384773+0000","last_undegraded":"2024-08-17T17:40:29.384773+0000","last_fullsized":"2024-08-17T17:40:29.384773+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":350,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.751686+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:40:16.751686+0000","last_peered":"2024-08-17T17:40:16.751686+0000","last_clean":"2024-08-17T17:40:16.751686+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:40:16.751686+0000","last_undegraded":"2024-08-17T17:40:16.751686+0000","last_fullsized":"2024-08-17T17:40:16.751686+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":583,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.384967+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:40:29.384967+0000","last_peered":"2024-08-17T17:40:29.384967+0000","last_clean":"2024-08-17T17:40:29.384967+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:40:29.384967+0000","last_undegraded":"2024-08-17T17:40:29.384967+0000","last_fullsized":"2024-08-17T17:40:29.384967+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":299,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385267+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:40:29.385267+0000","last_peered":"2024-08-17T17:40:29.385267+0000","last_clean":"2024-08-17T17:40:29.385267+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:40:29.385267+0000","last_undegraded":"2024-08-17T17:40:29.385267+0000","last_fullsized":"2024-08-17T17:40:29.385267+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":591,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.386996+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:40:29.386996+0000","last_peered":"2024-08-17T17:40:29.386996+0000","last_clean":"2024-08-17T17:40:29.386996+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:40:29.386996+0000","last_undegraded":"2024-08-17T17:40:29.386996+0000","last_fullsized":"2024-08-17T17:40:29.386996+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":551,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385830+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:40:29.385830+0000","last_peered":"2024-08-17T17:40:29.385830+0000","last_clean":"2024-08-17T17:40:29.385830+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:40:29.385830+0000","last_undegraded":"2024-08-17T17:40:29.385830+0000","last_fullsized":"2024-08-17T17:40:29.385830+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":362,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765386+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:40:16.765386+0000","last_peered":"2024-08-17T17:40:16.765386+0000","last_clean":"2024-08-17T17:40:16.765386+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:40:16.765386+0000","last_undegraded":"2024-08-17T17:40:16.765386+0000","last_fullsized":"2024-08-17T17:40:16.765386+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":239,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385237+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:40:29.385237+0000","last_peered":"2024-08-17T17:40:29.385237+0000","last_clean":"2024-08-17T17:40:29.385237+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:40:29.385237+0000","last_undegraded":"2024-08-17T17:40:29.385237+0000","last_fullsized":"2024-08-17T17:40:29.385237+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":296,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.508100+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:40:29.508100+0000","last_peered":"2024-08-17T17:40:29.508100+0000","last_clean":"2024-08-17T17:40:29.508100+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:40:29.508100+0000","last_undegraded":"2024-08-17T17:40:29.508100+0000","last_fullsized":"2024-08-17T17:40:29.508100+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":370,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.771551+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:40:16.771551+0000","last_peered":"2024-08-17T17:40:16.771551+0000","last_clean":"2024-08-17T17:40:16.771551+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:40:16.771551+0000","last_undegraded":"2024-08-17T17:40:16.771551+0000","last_fullsized":"2024-08-17T17:40:16.771551+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":233,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765478+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:40:16.765478+0000","last_peered":"2024-08-17T17:40:16.765478+0000","last_clean":"2024-08-17T17:40:16.765478+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:40:16.765478+0000","last_undegraded":"2024-08-17T17:40:16.765478+0000","last_fullsized":"2024-08-17T17:40:16.765478+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":370,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765265+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:40:16.765265+0000","last_peered":"2024-08-17T17:40:16.765265+0000","last_clean":"2024-08-17T17:40:16.765265+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:40:16.765265+0000","last_undegraded":"2024-08-17T17:40:16.765265+0000","last_fullsized":"2024-08-17T17:40:16.765265+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":233,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765817+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:40:16.765817+0000","last_peered":"2024-08-17T17:40:16.765817+0000","last_clean":"2024-08-17T17:40:16.765817+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:40:16.765817+0000","last_undegraded":"2024-08-17T17:40:16.765817+0000","last_fullsized":"2024-08-17T17:40:16.765817+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":603,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385843+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:40:29.385843+0000","last_peered":"2024-08-17T17:40:29.385843+0000","last_clean":"2024-08-17T17:40:29.385843+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:40:29.385843+0000","last_undegraded":"2024-08-17T17:40:29.385843+0000","last_fullsized":"2024-08-17T17:40:29.385843+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":576,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.385559+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:40:29.385559+0000","last_peered":"2024-08-17T17:40:29.385559+0000","last_clean":"2024-08-17T17:40:29.385559+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:40:29.385559+0000","last_undegraded":"2024-08-17T17:40:29.385559+0000","last_fullsized":"2024-08-17T17:40:29.385559+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":333,"reported_epoch":378,"state":"active+clean","last_fresh":"2024-08-17T17:40:16.765004+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:40:16.765004+0000","last_peered":"2024-08-17T17:40:16.765004+0000","last_clean":"2024-08-17T17:40:16.765004+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:40:16.765004+0000","last_undegraded":"2024-08-17T17:40:16.765004+0000","last_fullsized":"2024-08-17T17:40:16.765004+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":298,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.508271+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:40:29.508271+0000","last_peered":"2024-08-17T17:40:29.508271+0000","last_clean":"2024-08-17T17:40:29.508271+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:40:29.508271+0000","last_undegraded":"2024-08-17T17:40:29.508271+0000","last_fullsized":"2024-08-17T17:40:29.508271+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4847435800,"num_objects":73967,"num_object_clones":0,"num_object_copies":221901,"num_objects_missing_on_primary":5279,"num_objects_missing":5279,"num_objects_degraded":53630,"num_objects_misplaced":29229,"num_objects_unfound":0,"num_objects_dirty":73967,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162423,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28431,"num_bytes_recovered":1740701696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16030052352,"data_stored":16030040136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104742,"ondisk_log_size":104742,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170089,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463459,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2974664,"kb_used_data":2076800,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":90770488,"statfs":{"total":95995035648,"available":92948979712,"internally_reserved":0,"allocated":2126643200,"data_stored":2124589414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492133,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994908,"kb_used_data":599572,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92750244,"statfs":{"total":95995035648,"available":94976249856,"internally_reserved":0,"allocated":613961728,"data_stored":611938035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986725,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2360764,"kb_used_data":1179764,"kb_used_omap":1,"kb_used_meta":1180990,"kb_avail":91384388,"statfs":{"total":95995035648,"available":93577613312,"internally_reserved":0,"allocated":1208078336,"data_stored":1206035490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1209334104},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225552,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3305968,"kb_used_data":2368036,"kb_used_omap":4,"kb_used_meta":937915,"kb_avail":90439184,"statfs":{"total":95995035648,"available":92609724416,"internally_reserved":0,"allocated":2424868864,"data_stored":2422837676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":960425408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075877,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533771,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3106736,"kb_used_data":2068904,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90638416,"statfs":{"total":95995035648,"available":92813737984,"internally_reserved":0,"allocated":2118557696,"data_stored":2116521186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089747,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2801036,"kb_used_data":1787648,"kb_used_omap":5,"kb_used_meta":1013370,"kb_avail":90944116,"statfs":{"total":95995035648,"available":93126774784,"internally_reserved":0,"allocated":1830551552,"data_stored":1828497825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1037690939},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1824718848,"data_stored":1824718848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2112622592,"data_stored":2112618520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2417164288,"data_stored":2417164288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1202061312,"data_stored":1202061312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":609026048,"data_stored":609026048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2119827456,"data_stored":2119827456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2405371904,"data_stored":2405367832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:38.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:38.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:38.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:40:38.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:38.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:38.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e352/337 cost 1049576)", 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:35:46.513019+0000", 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 291.947011234, 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00078923399999999999, 2024-08-17T17:40:38.562 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513019+0000", 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513019+0000", 2024-08-17T17:40:38.563 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513020+0000", 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6700000000000002e-07 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513030+0000", 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0688e-05 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.564 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513040+0000", 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3929999999999997e-06 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513045+0000", 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.361e-06 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.565 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513099+0000", 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4277999999999998e-05 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:46.513808+0000", 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00070854700000000002 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.566 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e354/337 cost 1049576)", 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:14.850423+0000", 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 263.60960715300001, 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00094614500000000002, 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.567 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850423+0000", 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850423+0000", 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850424+0000", 2024-08-17T17:40:38.568 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1960000000000001e-06 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850435+0000", 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1375e-05 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850450+0000", 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4575e-05 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.569 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850458+0000", 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3899999999999993e-06 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850520+0000", 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1626999999999998e-05 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.851369+0000", 2024-08-17T17:40:38.570 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00084898199999999997 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e356/337 cost 1049576)", 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:25.354294+0000", 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 253.10573628099999, 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011246629999999999, 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.571 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354294+0000", 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354294+0000", 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.572 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354295+0000", 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1230000000000001e-06 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354307+0000", 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2517999999999999e-05 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.573 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354322+0000", 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4842e-05 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354330+0000", 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.422e-06 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354389+0000", 2024-08-17T17:40:38.574 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8996000000000002e-05 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.355418+0000", 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010287619999999999 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e356/337 cost 1049576)", 2024-08-17T17:40:38.575 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:26.497810+0000", 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 251.962219645, 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00079853300000000003, 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.497810+0000", 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.576 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.497810+0000", 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.497812+0000", 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.886e-06 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.497836+0000", 2024-08-17T17:40:38.577 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3456999999999999e-05 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.497864+0000", 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8779e-05 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.497900+0000", 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5889999999999997e-05 2024-08-17T17:40:38.578 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.498018+0000", 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117414 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:26.498609+0000", 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000591107 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.579 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e359/337 cost 1049576)", 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:53.957533+0000", 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 224.502497011, 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00080595299999999999, 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.580 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957533+0000", 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957533+0000", 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957535+0000", 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9489999999999999e-06 2024-08-17T17:40:38.581 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957557+0000", 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2520000000000001e-05 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957587+0000", 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9723e-05 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.582 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957601+0000", 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3825999999999999e-05 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.957741+0000", 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014032199999999999 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:53.958339+0000", 2024-08-17T17:40:38.583 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059761299999999999 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e359/337 cost 1049576)", 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:55.753828+0000", 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 222.70620169099999, 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00076366200000000002, 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.584 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.753828+0000", 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.753828+0000", 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.585 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.753830+0000", 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0200000000000001e-06 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.753856+0000", 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6288999999999999e-05 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.753900+0000", 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3263000000000002e-05 2024-08-17T17:40:38.586 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.753914+0000", 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4496e-05 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.754071+0000", 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015729700000000001 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.587 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:55.754592+0000", 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00052029699999999997 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e361/337 cost 1049576)", 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:14.509815+0000", 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 203.950214609, 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000773776, 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.588 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.509815+0000", 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.509815+0000", 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.589 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.509817+0000", 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.72e-06 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.509836+0000", 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9446999999999999e-05 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.509862+0000", 2024-08-17T17:40:38.590 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5698999999999999e-05 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.509874+0000", 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2367e-05 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.510061+0000", 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018697700000000001 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.591 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.510589+0000", 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00052756600000000004 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e361/337 cost 1049576)", 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:14.878928+0000", 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 203.581101722, 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00077525999999999995, 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.592 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.878928+0000", 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.878928+0000", 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.593 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.878930+0000", 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.295e-06 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.878949+0000", 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8793000000000001e-05 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.594 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.878971+0000", 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1900999999999999e-05 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.878982+0000", 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0691e-05 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.879062+0000", 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0615e-05 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.595 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:14.879703+0000", 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00064096499999999996 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e363/337 cost 1049576)", 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:34.815023+0000", 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.64500673200001, 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00075558799999999996, 2024-08-17T17:40:38.596 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815023+0000", 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815023+0000", 2024-08-17T17:40:38.597 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815025+0000", 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1229999999999998e-06 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815053+0000", 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7393e-05 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.598 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815081+0000", 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8353000000000001e-05 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815095+0000", 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4369000000000001e-05 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.599 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815252+0000", 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000156541 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:34.815779+0000", 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00052680900000000004 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.600 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e363/337 cost 1049576)", 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:42.250411+0000", 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 176.20961847800001, 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00091109999999999997, 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250411+0000", 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.601 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250411+0000", 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250413+0000", 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.477e-06 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250429+0000", 2024-08-17T17:40:38.602 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6166000000000001e-05 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250451+0000", 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2192e-05 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250461+0000", 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4280000000000002e-06 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.603 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250527+0000", 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6298999999999997e-05 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.251322+0000", 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00079553800000000004 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e363/337 cost 1049576)", 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:44.764754+0000", 2024-08-17T17:40:38.604 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 173.695275944, 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00077047300000000003, 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764754+0000", 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.605 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764754+0000", 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764756+0000", 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.826e-06 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764780+0000", 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4300000000000001e-05 2024-08-17T17:40:38.606 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764806+0000", 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5999999999999998e-05 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764819+0000", 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3295999999999999e-05 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.607 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.764964+0000", 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014471800000000001 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:44.765524+0000", 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00056033299999999999 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.608 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e364/337 cost 1049576)", 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:46.750065+0000", 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 171.70996476600001, 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014278940000000001, 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750065+0000", 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.609 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750065+0000", 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750066+0000", 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1629999999999999e-06 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.610 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750080+0000", 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.366e-05 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750095+0000", 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.504e-05 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750102+0000", 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8179999999999998e-06 2024-08-17T17:40:38.611 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750164+0000", 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2589000000000007e-05 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.751493+0000", 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001328624 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.612 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e365/337 cost 1049576)", 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:07.356751+0000", 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 151.10327905, 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00077691999999999995, 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356751+0000", 2024-08-17T17:40:38.613 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356751+0000", 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356753+0000", 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1399999999999998e-06 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.614 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356779+0000", 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5689000000000001e-05 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356808+0000", 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9879999999999999e-05 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356823+0000", 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.455e-05 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.615 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.356969+0000", 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000145604 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:07.357528+0000", 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055905699999999996 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.616 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e372/337 cost 1049576)", 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:10.396609+0000", 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 88.063421073000001, 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00083456000000000003, 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.617 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396609+0000", 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396609+0000", 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396611+0000", 2024-08-17T17:40:38.618 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.857e-06 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396632+0000", 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1756e-05 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396661+0000", 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8580000000000001e-05 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.619 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396674+0000", 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3249e-05 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396801+0000", 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012715200000000001 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.397443+0000", 2024-08-17T17:40:38.620 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000641966 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e372/337 cost 1049576)", 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:11.268479+0000", 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 87.191550415999998, 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00078389099999999999, 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.621 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268479+0000", 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268479+0000", 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.622 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268481+0000", 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9870000000000002e-06 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268505+0000", 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3737e-05 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268534+0000", 2024-08-17T17:40:38.623 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9238999999999999e-05 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268548+0000", 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.375e-05 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.268699+0000", 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015115800000000001 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.624 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:11.269263+0000", 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00056402000000000004 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.625 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e375/337 cost 1049576)", 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:49.278494+0000", 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 49.181535861999997, 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00080887400000000001, 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278494+0000", 2024-08-17T17:40:38.626 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278494+0000", 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278496+0000", 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2400000000000002e-06 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.627 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278531+0000", 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.506e-05 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278563+0000", 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1631999999999998e-05 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278582+0000", 2024-08-17T17:40:38.628 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9564000000000001e-05 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.278743+0000", 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016087999999999999 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:49.279303+0000", 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055949799999999996 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.629 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e376/337 cost 1049576)", 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.819130+0000", 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 43.640900277, 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00090844500000000002, 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.630 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819130+0000", 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819130+0000", 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.631 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819131+0000", 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.37e-06 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819147+0000", 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6504e-05 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819168+0000", 2024-08-17T17:40:38.632 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0712000000000001e-05 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819178+0000", 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8870000000000004e-06 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819267+0000", 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9237e-05 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.633 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.820038+0000", 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00077073500000000004 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e376/337 cost 1049576)", 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:57.908273+0000", 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 40.551756717000004, 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00079865800000000005, 2024-08-17T17:40:38.634 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908273+0000", 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908273+0000", 2024-08-17T17:40:38.635 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908275+0000", 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7430000000000001e-06 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908299+0000", 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.444e-05 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.636 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908328+0000", 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9026e-05 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908343+0000", 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4440000000000001e-05 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.908506+0000", 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000163401 2024-08-17T17:40:38.637 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:57.909072+0000", 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000565608 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e377/337 cost 1049576)", 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:13.580528+0000", 2024-08-17T17:40:38.638 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 24.879501524999998, 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00083345299999999995, 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580528+0000", 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.639 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580528+0000", 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580536+0000", 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0800000000000006e-06 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580556+0000", 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9553000000000001e-05 2024-08-17T17:40:38.640 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580581+0000", 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4936999999999999e-05 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580593+0000", 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2405999999999999e-05 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580731+0000", 2024-08-17T17:40:38.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013768400000000001 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.581362+0000", 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00063079300000000004 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e378/337 cost 1049576)", 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:18.512744+0000", 2024-08-17T17:40:38.642 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 19.947285609000001, 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013030660000000001, 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512744+0000", 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512744+0000", 2024-08-17T17:40:38.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512746+0000", 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.373e-06 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512761+0000", 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5372000000000001e-05 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512785+0000", 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3770000000000001e-05 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512794+0000", 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6139999999999998e-06 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512908+0000", 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000113936 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:40:38.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.514047+0000", 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011390009999999999 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:40:38.646 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:40:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.648 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:38.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:38.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:38.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:38.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:38.895 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:38.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:39.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:39.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:39.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:39.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:39.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:39.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:39.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:39.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:39.516 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:39.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:39.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:39 smithi142 ceph-mon[103614]: pgmap v1421: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 53633/220575 objects degraded (24.315%); 29192/220575 objects misplaced (13.235%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:39 smithi142 ceph-mon[106729]: pgmap v1421: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 53633/220575 objects degraded (24.315%); 29192/220575 objects misplaced (13.235%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:39.640 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:39.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:39.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:39 smithi152 ceph-mon[93456]: pgmap v1421: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 53633/220575 objects degraded (24.315%); 29192/220575 objects misplaced (13.235%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:40:39.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:39.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:40.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:40.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:40.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:40.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:40.260 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:40.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:40.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:40:40.485 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:40.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:40.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:40:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:40:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:40.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.682 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:40.741 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:40.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:40.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:40.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:40.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:40.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:41.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:41.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:41.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:41.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:41.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:41.297 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:40:41.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:41.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:41.404 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:40:41.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:41.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:41 smithi152 ceph-mon[93456]: pgmap v1422: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 53548/219570 objects degraded (24.388%); 28968/219570 objects misplaced (13.193%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:40:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53548/219570 objects degraded (24.388%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:41.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1814986969' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:41.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:41.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:41.680 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:41.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:41.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:41.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:41.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-mon[106729]: pgmap v1422: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 53548/219570 objects degraded (24.388%); 28968/219570 objects misplaced (13.193%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:40:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53548/219570 objects degraded (24.388%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1814986969' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-mon[103614]: pgmap v1422: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 53548/219570 objects degraded (24.388%); 28968/219570 objects misplaced (13.193%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:40:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53548/219570 objects degraded (24.388%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1814986969' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:41.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:41.891 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:41.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:42.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:42.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:42.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:40:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:40:41] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:40:42.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:42.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:42.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:42.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:42.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:42.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:42.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:42.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:42.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1814986969' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[106729]: osdmap e380: 8 total, 8 up, 6 in 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1814986969' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[103614]: osdmap e380: 8 total, 8 up, 6 in 2024-08-17T17:40:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:42.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:42.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:42.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:42.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:42.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:42 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1814986969' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:42 smithi152 ceph-mon[93456]: osdmap e380: 8 total, 8 up, 6 in 2024-08-17T17:40:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:42.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:42.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:42.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:43.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:43.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:43.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:43.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:43.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:43.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:43.353 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:43.377 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:43.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:43.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:43.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:43.523 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:40:43.523 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:40:43.523 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:43.523 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:43.541 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:40:43.541 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:43.559 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:40:43.559 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:43.577 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:40:43.577 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:43.578 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:43 smithi142 ceph-mon[103614]: pgmap v1424: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 53548/218610 objects degraded (24.495%); 28929/218610 objects misplaced (13.233%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:43.578 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:43 smithi142 ceph-mon[106729]: pgmap v1424: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 53548/218610 objects degraded (24.495%); 28929/218610 objects misplaced (13.233%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:43.598 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:40:43.598 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:43.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:43.615 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:40:43.616 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:43.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:43.634 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:40:43.635 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:43.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:43.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:43.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:43 smithi152 ceph-mon[93456]: pgmap v1424: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 53548/218610 objects degraded (24.495%); 28929/218610 objects misplaced (13.233%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:40:43.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:43.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:43.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:44.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:44.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:44.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:44.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:44.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:44.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:44.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:40:44.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:44.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:44.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:44.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:44.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:44.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2097483565' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:44.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2097483565' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:44.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:44.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:44.748 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2097483565' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:44.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:44.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:44.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:44.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:44.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:45.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:45.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:45.219 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:45.241 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:45.269 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:45.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:45.365 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:45.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:45.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:45.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:45.554 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:45.555 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:45 smithi142 ceph-mon[106729]: pgmap v1425: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 130 op/s; 53469/218049 objects degraded (24.522%); 28742/218049 objects misplaced (13.181%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:45.556 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:45 smithi142 ceph-mon[103614]: pgmap v1425: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 130 op/s; 53469/218049 objects degraded (24.522%); 28742/218049 objects misplaced (13.181%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:45.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:45.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:45.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:45.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:45.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:45.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:45 smithi152 ceph-mon[93456]: pgmap v1425: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 130 op/s; 53469/218049 objects degraded (24.522%); 28742/218049 objects misplaced (13.181%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:45.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:45.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:45.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:45.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:46.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:46.106 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:46.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:46.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:46.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:46.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:46.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:46.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:46.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:46.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:46.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:46.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:46.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53469/218049 objects degraded (24.522%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:46.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:46.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:46.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:46.856 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53469/218049 objects degraded (24.522%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:46.856 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53469/218049 objects degraded (24.522%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:46.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:46.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:46.974 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:47.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:47.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:47.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:47.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:47.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:47.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:47.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:47.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:47.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:47.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:47.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:47.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:47.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:47.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:47.746 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:47 smithi142 ceph-mon[106729]: pgmap v1426: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 53469/217494 objects degraded (24.584%); 28742/217494 objects misplaced (13.215%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:47.746 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:47 smithi142 ceph-mon[103614]: pgmap v1426: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 53469/217494 objects degraded (24.584%); 28742/217494 objects misplaced (13.215%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:47 smithi152 ceph-mon[93456]: pgmap v1426: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 53469/217494 objects degraded (24.584%); 28742/217494 objects misplaced (13.215%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:47.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:47.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:47.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:47.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:47.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:48.030 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:48.030 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:48.064 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:40:48.065 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:40:48.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:48.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:48.209 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:48.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:48.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:48.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:48.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:48.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:48.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:48.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:48.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:48.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:48.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:40:48.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.864 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:48.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:48.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:48.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:49.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:49.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:49.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:49.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:49.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:49.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:49 smithi152 ceph-mon[93456]: from='client.37554 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:49 smithi152 ceph-mon[93456]: pgmap v1427: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 53469/217116 objects degraded (24.627%); 28719/217116 objects misplaced (13.227%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:49.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:49.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:49 smithi142 ceph-mon[106729]: from='client.37554 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:49 smithi142 ceph-mon[106729]: pgmap v1427: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 53469/217116 objects degraded (24.627%); 28719/217116 objects misplaced (13.227%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:49 smithi142 ceph-mon[103614]: from='client.37554 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:49 smithi142 ceph-mon[103614]: pgmap v1427: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 53469/217116 objects degraded (24.627%); 28719/217116 objects misplaced (13.227%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:40:49.922 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1426,"stamp":"2024-08-17T17:40:46.331770+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4750055397,"num_objects":72498,"num_object_clones":0,"num_object_copies":217494,"num_objects_missing_on_primary":5209,"num_objects_missing":5209,"num_objects_degraded":53469,"num_objects_misplaced":28742,"num_objects_unfound":0,"num_objects_dirty":72498,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":164196,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28724,"num_bytes_recovered":1759584910,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104644,"ondisk_log_size":104644,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15349052,"kb_used_data":9873220,"kb_used_omap":27,"kb_used_meta":5475748,"kb_avail":547121860,"statfs":{"total":575970213888,"available":560252784640,"internally_reserved":0,"allocated":10110177280,"data_stored":10097880979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5607166696},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3,2,3,0,0,0,2,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-81854464,"num_objects":-1249,"num_object_clones":0,"num_object_copies":-3747,"num_objects_missing_on_primary":-70,"num_objects_missing":-70,"num_objects_degraded":-164,"num_objects_misplaced":-487,"num_objects_unfound":0,"num_objects_dirty":-1249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1249,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":252,"num_bytes_recovered":16515072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002581"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":603,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456179+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:40:41.456179+0000","last_peered":"2024-08-17T17:40:41.456179+0000","last_clean":"2024-08-17T17:40:41.456179+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:40:41.456179+0000","last_undegraded":"2024-08-17T17:40:41.456179+0000","last_fullsized":"2024-08-17T17:40:41.456179+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":89,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.402575+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:40:29.402575+0000","last_peered":"2024-08-17T17:40:29.402575+0000","last_clean":"2024-08-17T17:40:29.402575+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:40:29.402575+0000","last_undegraded":"2024-08-17T17:40:29.402575+0000","last_fullsized":"2024-08-17T17:40:29.402575+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528717+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:40:29.528717+0000","last_peered":"2024-08-17T17:40:29.528717+0000","last_clean":"2024-08-17T17:40:29.528717+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:40:29.528717+0000","last_undegraded":"2024-08-17T17:40:29.528717+0000","last_fullsized":"2024-08-17T17:40:29.528717+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":301,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455848+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:40:41.455848+0000","last_peered":"2024-08-17T17:40:41.455848+0000","last_clean":"2024-08-17T17:40:41.455848+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:40:41.455848+0000","last_undegraded":"2024-08-17T17:40:41.455848+0000","last_fullsized":"2024-08-17T17:40:41.455848+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":478,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456360+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:40:41.456360+0000","last_peered":"2024-08-17T17:40:41.456360+0000","last_clean":"2024-08-17T17:40:41.456360+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:40:41.456360+0000","last_undegraded":"2024-08-17T17:40:41.456360+0000","last_fullsized":"2024-08-17T17:40:41.456360+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":232,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400336+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:40:29.400336+0000","last_peered":"2024-08-17T17:40:29.400336+0000","last_clean":"2024-08-17T17:40:29.400336+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:40:29.400336+0000","last_undegraded":"2024-08-17T17:40:29.400336+0000","last_fullsized":"2024-08-17T17:40:29.400336+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":571,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456556+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:40:41.456556+0000","last_peered":"2024-08-17T17:40:41.456556+0000","last_clean":"2024-08-17T17:40:41.456556+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:40:41.456556+0000","last_undegraded":"2024-08-17T17:40:41.456556+0000","last_fullsized":"2024-08-17T17:40:41.456556+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":400,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.395444+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:40:29.395444+0000","last_peered":"2024-08-17T17:40:29.395444+0000","last_clean":"2024-08-17T17:40:29.395444+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:40:29.395444+0000","last_undegraded":"2024-08-17T17:40:29.395444+0000","last_fullsized":"2024-08-17T17:40:29.395444+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"380'10188","reported_seq":14494,"reported_epoch":380,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:40:43.736416+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:40:43.736416+0000","last_peered":"2024-08-17T17:40:43.736416+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:40:43.736416+0000","last_undegraded":"2024-08-17T17:40:43.736416+0000","last_fullsized":"2024-08-17T17:40:43.736416+0000","mapping_epoch":337,"log_start":"325'3363","ondisk_log_start":"325'3363","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6825,"log_dups_size":0,"ondisk_log_size":6825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4516,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10188,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2107,"num_bytes_recovered":138084352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4516}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":85,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.477385+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:40:29.477385+0000","last_peered":"2024-08-17T17:40:29.477385+0000","last_clean":"2024-08-17T17:40:29.477385+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:40:29.477385+0000","last_undegraded":"2024-08-17T17:40:29.477385+0000","last_fullsized":"2024-08-17T17:40:29.477385+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528092+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:40:29.528092+0000","last_peered":"2024-08-17T17:40:29.528092+0000","last_clean":"2024-08-17T17:40:29.528092+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:40:29.528092+0000","last_undegraded":"2024-08-17T17:40:29.528092+0000","last_fullsized":"2024-08-17T17:40:29.528092+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"380'10443","reported_seq":10521,"reported_epoch":380,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:41.624265+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:40:41.624265+0000","last_peered":"2024-08-17T17:40:41.624265+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:40:41.624265+0000","last_undegraded":"2024-08-17T17:40:41.624265+0000","last_fullsized":"2024-08-17T17:40:41.624265+0000","mapping_epoch":337,"log_start":"326'3882","ondisk_log_start":"326'3882","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303497216,"num_objects":4631,"num_object_clones":0,"num_object_copies":13893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4631,"num_objects_unfound":0,"num_objects_dirty":4631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10443,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4631}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":87,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.403758+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:40:29.403758+0000","last_peered":"2024-08-17T17:40:29.403758+0000","last_clean":"2024-08-17T17:40:29.403758+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:40:29.403758+0000","last_undegraded":"2024-08-17T17:40:29.403758+0000","last_fullsized":"2024-08-17T17:40:29.403758+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":351,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400262+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:40:29.400262+0000","last_peered":"2024-08-17T17:40:29.400262+0000","last_clean":"2024-08-17T17:40:29.400262+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:40:29.400262+0000","last_undegraded":"2024-08-17T17:40:29.400262+0000","last_fullsized":"2024-08-17T17:40:29.400262+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"380'10219","reported_seq":10300,"reported_epoch":380,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:43.724380+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:40:43.724380+0000","last_peered":"2024-08-17T17:40:43.724380+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:40:43.724380+0000","last_undegraded":"2024-08-17T17:40:43.724380+0000","last_fullsized":"2024-08-17T17:40:43.724380+0000","mapping_epoch":337,"log_start":"325'3353","ondisk_log_start":"325'3353","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295895040,"num_objects":4515,"num_object_clones":0,"num_object_copies":13545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4515,"num_objects_unfound":0,"num_objects_dirty":4515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10219,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4515}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":83,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528652+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:40:29.528652+0000","last_peered":"2024-08-17T17:40:29.528652+0000","last_clean":"2024-08-17T17:40:29.528652+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:40:29.528652+0000","last_undegraded":"2024-08-17T17:40:29.528652+0000","last_fullsized":"2024-08-17T17:40:29.528652+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":361,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.395188+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:40:29.395188+0000","last_peered":"2024-08-17T17:40:29.395188+0000","last_clean":"2024-08-17T17:40:29.395188+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:40:29.395188+0000","last_undegraded":"2024-08-17T17:40:29.395188+0000","last_fullsized":"2024-08-17T17:40:29.395188+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"380'10390","reported_seq":10530,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:44.740745+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:40:44.740745+0000","last_peered":"2024-08-17T17:40:44.740745+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:40:44.740745+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'3413","ondisk_log_start":"325'3413","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299368448,"num_objects":4568,"num_object_clones":0,"num_object_copies":13704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10390,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2201},{"shards":"0,3,7","objects":4568}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":514,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456386+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:40:41.456386+0000","last_peered":"2024-08-17T17:40:41.456386+0000","last_clean":"2024-08-17T17:40:41.456386+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:40:41.456386+0000","last_undegraded":"2024-08-17T17:40:41.456386+0000","last_fullsized":"2024-08-17T17:40:41.456386+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":281,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.430766+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:40:41.430766+0000","last_peered":"2024-08-17T17:40:41.430766+0000","last_clean":"2024-08-17T17:40:41.430766+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:40:41.430766+0000","last_undegraded":"2024-08-17T17:40:41.430766+0000","last_fullsized":"2024-08-17T17:40:41.430766+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"380'10106","reported_seq":10175,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:42.097704+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:40:42.097704+0000","last_peered":"2024-08-17T17:40:42.097704+0000","last_clean":"2024-08-17T17:40:42.097704+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:40:42.097704+0000","last_undegraded":"2024-08-17T17:40:42.097704+0000","last_fullsized":"2024-08-17T17:40:42.097704+0000","mapping_epoch":319,"log_start":"326'3574","ondisk_log_start":"326'3574","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10106,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":604,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456577+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:40:41.456577+0000","last_peered":"2024-08-17T17:40:41.456577+0000","last_clean":"2024-08-17T17:40:41.456577+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:40:41.456577+0000","last_undegraded":"2024-08-17T17:40:41.456577+0000","last_fullsized":"2024-08-17T17:40:41.456577+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":334,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400304+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:40:29.400304+0000","last_peered":"2024-08-17T17:40:29.400304+0000","last_clean":"2024-08-17T17:40:29.400304+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:40:29.400304+0000","last_undegraded":"2024-08-17T17:40:29.400304+0000","last_fullsized":"2024-08-17T17:40:29.400304+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"380'10293","reported_seq":10372,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:41.904998+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:40:41.904998+0000","last_peered":"2024-08-17T17:40:41.904998+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:40:41.904998+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'3735","ondisk_log_start":"326'3735","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10293,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4639},{"shards":"1,6","objects":-2727}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":335,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.395330+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:40:29.395330+0000","last_peered":"2024-08-17T17:40:29.395330+0000","last_clean":"2024-08-17T17:40:29.395330+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:40:29.395330+0000","last_undegraded":"2024-08-17T17:40:29.395330+0000","last_fullsized":"2024-08-17T17:40:29.395330+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":361,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.403976+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:40:29.403976+0000","last_peered":"2024-08-17T17:40:29.403976+0000","last_clean":"2024-08-17T17:40:29.403976+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:40:29.403976+0000","last_undegraded":"2024-08-17T17:40:29.403976+0000","last_fullsized":"2024-08-17T17:40:29.403976+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"380'10420","reported_seq":10561,"reported_epoch":380,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:40:43.815715+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:40:43.815715+0000","last_peered":"2024-08-17T17:40:43.815715+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:40:43.815715+0000","last_undegraded":"2024-08-17T17:40:43.815715+0000","last_fullsized":"2024-08-17T17:40:43.815715+0000","mapping_epoch":337,"log_start":"327'4861","ondisk_log_start":"327'4861","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300810240,"num_objects":4590,"num_object_clones":0,"num_object_copies":13770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4590,"num_objects_unfound":0,"num_objects_dirty":4590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10420,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4590}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":372,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400847+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:40:29.400847+0000","last_peered":"2024-08-17T17:40:29.400847+0000","last_clean":"2024-08-17T17:40:29.400847+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:40:29.400847+0000","last_undegraded":"2024-08-17T17:40:29.400847+0000","last_fullsized":"2024-08-17T17:40:29.400847+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":334,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.403829+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:40:29.403829+0000","last_peered":"2024-08-17T17:40:29.403829+0000","last_clean":"2024-08-17T17:40:29.403829+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:40:29.403829+0000","last_undegraded":"2024-08-17T17:40:29.403829+0000","last_fullsized":"2024-08-17T17:40:29.403829+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"380'10188","reported_seq":14547,"reported_epoch":380,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:40:43.834257+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:40:43.834257+0000","last_peered":"2024-08-17T17:40:43.834257+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:40:43.834257+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:40:43.834257+0000","mapping_epoch":337,"log_start":"325'3341","ondisk_log_start":"325'3341","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6847,"log_dups_size":0,"ondisk_log_size":6847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":5209,"num_objects_missing":5209,"num_objects_degraded":4516,"num_objects_misplaced":5209,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10188,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2130,"num_bytes_recovered":139526144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":4503}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":573,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455552+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:40:41.455552+0000","last_peered":"2024-08-17T17:40:41.455552+0000","last_clean":"2024-08-17T17:40:41.455552+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:40:41.455552+0000","last_undegraded":"2024-08-17T17:40:41.455552+0000","last_fullsized":"2024-08-17T17:40:41.455552+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":510,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456634+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:40:41.456634+0000","last_peered":"2024-08-17T17:40:41.456634+0000","last_clean":"2024-08-17T17:40:41.456634+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:40:41.456634+0000","last_undegraded":"2024-08-17T17:40:41.456634+0000","last_fullsized":"2024-08-17T17:40:41.456634+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"380'10365","reported_seq":14829,"reported_epoch":380,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:40:43.798030+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:40:43.798030+0000","last_peered":"2024-08-17T17:40:43.798030+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:40:43.798030+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:40:43.798030+0000","mapping_epoch":337,"log_start":"326'4744","ondisk_log_start":"326'4744","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5621,"log_dups_size":0,"ondisk_log_size":5621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296878104,"num_objects":4531,"num_object_clones":0,"num_object_copies":13593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4531,"num_objects_misplaced":5281,"num_objects_unfound":0,"num_objects_dirty":4531,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10365,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2167,"num_bytes_recovered":141950976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4531}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":704,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:41.430070+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:40:41.430070+0000","last_peered":"2024-08-17T17:40:41.430070+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:40:41.430070+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":309,"reported_epoch":379,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:40:29.528404+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:40:29.528404+0000","last_peered":"2024-08-17T17:40:29.528404+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:40:29.528404+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:40:29.528404+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":553,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455759+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:40:41.455759+0000","last_peered":"2024-08-17T17:40:41.455759+0000","last_clean":"2024-08-17T17:40:41.455759+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:40:41.455759+0000","last_undegraded":"2024-08-17T17:40:41.455759+0000","last_fullsized":"2024-08-17T17:40:41.455759+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":344,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.396005+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:40:29.396005+0000","last_peered":"2024-08-17T17:40:29.396005+0000","last_clean":"2024-08-17T17:40:29.396005+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:40:29.396005+0000","last_undegraded":"2024-08-17T17:40:29.396005+0000","last_fullsized":"2024-08-17T17:40:29.396005+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"380'10211","reported_seq":10354,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:44.739414+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:40:44.739414+0000","last_peered":"2024-08-17T17:40:44.739414+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:40:44.739414+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'3221","ondisk_log_start":"325'3221","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297074688,"num_objects":4533,"num_object_clones":0,"num_object_copies":13599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10211,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2114},{"shards":"0,4,7","objects":4533}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":369,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.408700+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:40:41.408700+0000","last_peered":"2024-08-17T17:40:41.408700+0000","last_clean":"2024-08-17T17:40:41.408700+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:40:41.408700+0000","last_undegraded":"2024-08-17T17:40:41.408700+0000","last_fullsized":"2024-08-17T17:40:41.408700+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":316,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400143+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:40:29.400143+0000","last_peered":"2024-08-17T17:40:29.400143+0000","last_clean":"2024-08-17T17:40:29.400143+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:40:29.400143+0000","last_undegraded":"2024-08-17T17:40:29.400143+0000","last_fullsized":"2024-08-17T17:40:29.400143+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":295,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.431282+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:40:41.431282+0000","last_peered":"2024-08-17T17:40:41.431282+0000","last_clean":"2024-08-17T17:40:41.431282+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:40:41.431282+0000","last_undegraded":"2024-08-17T17:40:41.431282+0000","last_fullsized":"2024-08-17T17:40:41.431282+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"380'10221","reported_seq":10300,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:41.857347+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:40:41.857347+0000","last_peered":"2024-08-17T17:40:41.857347+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:40:41.857347+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'3617","ondisk_log_start":"326'3617","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299827200,"num_objects":4575,"num_object_clones":0,"num_object_copies":13725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10221,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4575},{"shards":"5,6","objects":-2721}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":461,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.458948+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:40:41.458948+0000","last_peered":"2024-08-17T17:40:41.458948+0000","last_clean":"2024-08-17T17:40:41.458948+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:40:41.458948+0000","last_undegraded":"2024-08-17T17:40:41.458948+0000","last_fullsized":"2024-08-17T17:40:41.458948+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":116,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.395121+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:40:29.395121+0000","last_peered":"2024-08-17T17:40:29.395121+0000","last_clean":"2024-08-17T17:40:29.395121+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:40:29.395121+0000","last_undegraded":"2024-08-17T17:40:29.395121+0000","last_fullsized":"2024-08-17T17:40:29.395121+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":646,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.409492+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:40:41.409492+0000","last_peered":"2024-08-17T17:40:41.409492+0000","last_clean":"2024-08-17T17:40:41.409492+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:40:41.409492+0000","last_undegraded":"2024-08-17T17:40:41.409492+0000","last_fullsized":"2024-08-17T17:40:41.409492+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"380'10144","reported_seq":11011,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:44.795118+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:40:44.795118+0000","last_peered":"2024-08-17T17:40:44.795118+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:40:44.795118+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'3116","ondisk_log_start":"325'3116","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295043072,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10144,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4502},{"shards":"0,5","objects":-2152}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":437,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.410357+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:40:41.410357+0000","last_peered":"2024-08-17T17:40:41.410357+0000","last_clean":"2024-08-17T17:40:41.410357+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:40:41.410357+0000","last_undegraded":"2024-08-17T17:40:41.410357+0000","last_fullsized":"2024-08-17T17:40:41.410357+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":355,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.402770+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:40:29.402770+0000","last_peered":"2024-08-17T17:40:29.402770+0000","last_clean":"2024-08-17T17:40:29.402770+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:40:29.402770+0000","last_undegraded":"2024-08-17T17:40:29.402770+0000","last_fullsized":"2024-08-17T17:40:29.402770+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":232,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456085+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:40:41.456085+0000","last_peered":"2024-08-17T17:40:41.456085+0000","last_clean":"2024-08-17T17:40:41.456085+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:40:41.456085+0000","last_undegraded":"2024-08-17T17:40:41.456085+0000","last_fullsized":"2024-08-17T17:40:41.456085+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"380'10370","reported_seq":25356,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:44.944873+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:40:44.944873+0000","last_peered":"2024-08-17T17:40:44.944873+0000","last_clean":"2024-08-17T17:40:44.944873+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:40:44.944873+0000","last_undegraded":"2024-08-17T17:40:44.944873+0000","last_fullsized":"2024-08-17T17:40:44.944873+0000","mapping_epoch":355,"log_start":"326'4030","ondisk_log_start":"326'4030","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6340,"log_dups_size":0,"ondisk_log_size":6340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292683776,"num_objects":4466,"num_object_clones":0,"num_object_copies":13398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":294,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528576+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:40:29.528576+0000","last_peered":"2024-08-17T17:40:29.528576+0000","last_clean":"2024-08-17T17:40:29.528576+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:40:29.528576+0000","last_undegraded":"2024-08-17T17:40:29.528576+0000","last_fullsized":"2024-08-17T17:40:29.528576+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":101,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455956+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:40:41.455956+0000","last_peered":"2024-08-17T17:40:41.455956+0000","last_clean":"2024-08-17T17:40:41.455956+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:40:41.455956+0000","last_undegraded":"2024-08-17T17:40:41.455956+0000","last_fullsized":"2024-08-17T17:40:41.455956+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":596,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.410654+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:40:41.410654+0000","last_peered":"2024-08-17T17:40:41.410654+0000","last_clean":"2024-08-17T17:40:41.410654+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:40:41.410654+0000","last_undegraded":"2024-08-17T17:40:41.410654+0000","last_fullsized":"2024-08-17T17:40:41.410654+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"380'9999","reported_seq":24553,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:45.417240+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:40:45.417240+0000","last_peered":"2024-08-17T17:40:45.417240+0000","last_clean":"2024-08-17T17:40:45.417240+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:40:45.417240+0000","last_undegraded":"2024-08-17T17:40:45.417240+0000","last_fullsized":"2024-08-17T17:40:45.417240+0000","mapping_epoch":345,"log_start":"326'3672","ondisk_log_start":"326'3672","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288686080,"num_objects":4405,"num_object_clones":0,"num_object_copies":13215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9999,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":290,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528667+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:40:29.528667+0000","last_peered":"2024-08-17T17:40:29.528667+0000","last_clean":"2024-08-17T17:40:29.528667+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:40:29.528667+0000","last_undegraded":"2024-08-17T17:40:29.528667+0000","last_fullsized":"2024-08-17T17:40:29.528667+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":352,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.430275+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:40:41.430275+0000","last_peered":"2024-08-17T17:40:41.430275+0000","last_clean":"2024-08-17T17:40:41.430275+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:40:41.430275+0000","last_undegraded":"2024-08-17T17:40:41.430275+0000","last_fullsized":"2024-08-17T17:40:41.430275+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":432,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528833+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:40:29.528833+0000","last_peered":"2024-08-17T17:40:29.528833+0000","last_clean":"2024-08-17T17:40:29.528833+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:40:29.528833+0000","last_undegraded":"2024-08-17T17:40:29.528833+0000","last_fullsized":"2024-08-17T17:40:29.528833+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"380'10252","reported_seq":10414,"reported_epoch":380,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:45.132844+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:40:45.132844+0000","last_peered":"2024-08-17T17:40:45.132844+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:40:45.132844+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'3987","ondisk_log_start":"326'3987","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6265,"log_dups_size":0,"ondisk_log_size":6265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297009152,"num_objects":4532,"num_object_clones":0,"num_object_copies":13596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10252,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4532},{"shards":"0,3","objects":-2716}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":626,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.457913+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:40:41.457913+0000","last_peered":"2024-08-17T17:40:41.457913+0000","last_clean":"2024-08-17T17:40:41.457913+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:40:41.457913+0000","last_undegraded":"2024-08-17T17:40:41.457913+0000","last_fullsized":"2024-08-17T17:40:41.457913+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":595,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.409667+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:40:41.409667+0000","last_peered":"2024-08-17T17:40:41.409667+0000","last_clean":"2024-08-17T17:40:41.409667+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:40:41.409667+0000","last_undegraded":"2024-08-17T17:40:41.409667+0000","last_fullsized":"2024-08-17T17:40:41.409667+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":613,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455395+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:40:41.455395+0000","last_peered":"2024-08-17T17:40:41.455395+0000","last_clean":"2024-08-17T17:40:41.455395+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:40:41.455395+0000","last_undegraded":"2024-08-17T17:40:41.455395+0000","last_fullsized":"2024-08-17T17:40:41.455395+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"380'10109","reported_seq":24786,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.886754+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:40:41.886754+0000","last_peered":"2024-08-17T17:40:41.886754+0000","last_clean":"2024-08-17T17:40:41.886754+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:40:41.886754+0000","last_undegraded":"2024-08-17T17:40:41.886754+0000","last_fullsized":"2024-08-17T17:40:41.886754+0000","mapping_epoch":347,"log_start":"326'3572","ondisk_log_start":"326'3572","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292618240,"num_objects":4465,"num_object_clones":0,"num_object_copies":13395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10109,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":285,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.431186+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:40:41.431186+0000","last_peered":"2024-08-17T17:40:41.431186+0000","last_clean":"2024-08-17T17:40:41.431186+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:40:41.431186+0000","last_undegraded":"2024-08-17T17:40:41.431186+0000","last_fullsized":"2024-08-17T17:40:41.431186+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":384,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.396197+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:40:29.396197+0000","last_peered":"2024-08-17T17:40:29.396197+0000","last_clean":"2024-08-17T17:40:29.396197+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:40:29.396197+0000","last_undegraded":"2024-08-17T17:40:29.396197+0000","last_fullsized":"2024-08-17T17:40:29.396197+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":334,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400540+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:40:29.400540+0000","last_peered":"2024-08-17T17:40:29.400540+0000","last_clean":"2024-08-17T17:40:29.400540+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:40:29.400540+0000","last_undegraded":"2024-08-17T17:40:29.400540+0000","last_fullsized":"2024-08-17T17:40:29.400540+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":334,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.402692+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:40:29.402692+0000","last_peered":"2024-08-17T17:40:29.402692+0000","last_clean":"2024-08-17T17:40:29.402692+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:40:29.402692+0000","last_undegraded":"2024-08-17T17:40:29.402692+0000","last_fullsized":"2024-08-17T17:40:29.402692+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":377,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.403651+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:40:29.403651+0000","last_peered":"2024-08-17T17:40:29.403651+0000","last_clean":"2024-08-17T17:40:29.403651+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:40:29.403651+0000","last_undegraded":"2024-08-17T17:40:29.403651+0000","last_fullsized":"2024-08-17T17:40:29.403651+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":602,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.410448+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:40:41.410448+0000","last_peered":"2024-08-17T17:40:41.410448+0000","last_clean":"2024-08-17T17:40:41.410448+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:40:41.410448+0000","last_undegraded":"2024-08-17T17:40:41.410448+0000","last_fullsized":"2024-08-17T17:40:41.410448+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":141,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.459406+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:40:41.459406+0000","last_peered":"2024-08-17T17:40:41.459406+0000","last_clean":"2024-08-17T17:40:41.459406+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:40:41.459406+0000","last_undegraded":"2024-08-17T17:40:41.459406+0000","last_fullsized":"2024-08-17T17:40:41.459406+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":284,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.528849+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:40:29.528849+0000","last_peered":"2024-08-17T17:40:29.528849+0000","last_clean":"2024-08-17T17:40:29.528849+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:40:29.528849+0000","last_undegraded":"2024-08-17T17:40:29.528849+0000","last_fullsized":"2024-08-17T17:40:29.528849+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":560,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.458599+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:40:41.458599+0000","last_peered":"2024-08-17T17:40:41.458599+0000","last_clean":"2024-08-17T17:40:41.458599+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:40:41.458599+0000","last_undegraded":"2024-08-17T17:40:41.458599+0000","last_fullsized":"2024-08-17T17:40:41.458599+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":428,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.529236+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:40:29.529236+0000","last_peered":"2024-08-17T17:40:29.529236+0000","last_clean":"2024-08-17T17:40:29.529236+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:40:29.529236+0000","last_undegraded":"2024-08-17T17:40:29.529236+0000","last_fullsized":"2024-08-17T17:40:29.529236+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":554,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455295+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:40:41.455295+0000","last_peered":"2024-08-17T17:40:41.455295+0000","last_clean":"2024-08-17T17:40:41.455295+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:40:41.455295+0000","last_undegraded":"2024-08-17T17:40:41.455295+0000","last_fullsized":"2024-08-17T17:40:41.455295+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":351,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.396155+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:40:29.396155+0000","last_peered":"2024-08-17T17:40:29.396155+0000","last_clean":"2024-08-17T17:40:29.396155+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:40:29.396155+0000","last_undegraded":"2024-08-17T17:40:29.396155+0000","last_fullsized":"2024-08-17T17:40:29.396155+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":585,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.455467+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:40:41.455467+0000","last_peered":"2024-08-17T17:40:41.455467+0000","last_clean":"2024-08-17T17:40:41.455467+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:40:41.455467+0000","last_undegraded":"2024-08-17T17:40:41.455467+0000","last_fullsized":"2024-08-17T17:40:41.455467+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":301,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456243+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:40:41.456243+0000","last_peered":"2024-08-17T17:40:41.456243+0000","last_clean":"2024-08-17T17:40:41.456243+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:40:41.456243+0000","last_undegraded":"2024-08-17T17:40:41.456243+0000","last_fullsized":"2024-08-17T17:40:41.456243+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":593,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.409737+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:40:41.409737+0000","last_peered":"2024-08-17T17:40:41.409737+0000","last_clean":"2024-08-17T17:40:41.409737+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:40:41.409737+0000","last_undegraded":"2024-08-17T17:40:41.409737+0000","last_fullsized":"2024-08-17T17:40:41.409737+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":553,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.459215+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:40:41.459215+0000","last_peered":"2024-08-17T17:40:41.459215+0000","last_clean":"2024-08-17T17:40:41.459215+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:40:41.459215+0000","last_undegraded":"2024-08-17T17:40:41.459215+0000","last_fullsized":"2024-08-17T17:40:41.459215+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":363,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400742+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:40:29.400742+0000","last_peered":"2024-08-17T17:40:29.400742+0000","last_clean":"2024-08-17T17:40:29.400742+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:40:29.400742+0000","last_undegraded":"2024-08-17T17:40:29.400742+0000","last_fullsized":"2024-08-17T17:40:29.400742+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":241,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456280+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:40:41.456280+0000","last_peered":"2024-08-17T17:40:41.456280+0000","last_clean":"2024-08-17T17:40:41.456280+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:40:41.456280+0000","last_undegraded":"2024-08-17T17:40:41.456280+0000","last_fullsized":"2024-08-17T17:40:41.456280+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":298,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.430637+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:40:41.430637+0000","last_peered":"2024-08-17T17:40:41.430637+0000","last_clean":"2024-08-17T17:40:41.430637+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:40:41.430637+0000","last_undegraded":"2024-08-17T17:40:41.430637+0000","last_fullsized":"2024-08-17T17:40:41.430637+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.402692+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:40:29.402692+0000","last_peered":"2024-08-17T17:40:29.402692+0000","last_clean":"2024-08-17T17:40:29.402692+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:40:29.402692+0000","last_undegraded":"2024-08-17T17:40:29.402692+0000","last_fullsized":"2024-08-17T17:40:29.402692+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":234,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.403638+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:40:29.403638+0000","last_peered":"2024-08-17T17:40:29.403638+0000","last_clean":"2024-08-17T17:40:29.403638+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:40:29.403638+0000","last_undegraded":"2024-08-17T17:40:29.403638+0000","last_fullsized":"2024-08-17T17:40:29.403638+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":371,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.400580+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:40:29.400580+0000","last_peered":"2024-08-17T17:40:29.400580+0000","last_clean":"2024-08-17T17:40:29.400580+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:40:29.400580+0000","last_undegraded":"2024-08-17T17:40:29.400580+0000","last_fullsized":"2024-08-17T17:40:29.400580+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":234,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.403909+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:40:29.403909+0000","last_peered":"2024-08-17T17:40:29.403909+0000","last_clean":"2024-08-17T17:40:29.403909+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:40:29.403909+0000","last_undegraded":"2024-08-17T17:40:29.403909+0000","last_fullsized":"2024-08-17T17:40:29.403909+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":605,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.459089+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:40:41.459089+0000","last_peered":"2024-08-17T17:40:41.459089+0000","last_clean":"2024-08-17T17:40:41.459089+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:40:41.459089+0000","last_undegraded":"2024-08-17T17:40:41.459089+0000","last_fullsized":"2024-08-17T17:40:41.459089+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":579,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.456414+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:40:41.456414+0000","last_peered":"2024-08-17T17:40:41.456414+0000","last_clean":"2024-08-17T17:40:41.456414+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:40:41.456414+0000","last_undegraded":"2024-08-17T17:40:41.456414+0000","last_fullsized":"2024-08-17T17:40:41.456414+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":334,"reported_epoch":379,"state":"active+clean","last_fresh":"2024-08-17T17:40:29.402465+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:40:29.402465+0000","last_peered":"2024-08-17T17:40:29.402465+0000","last_clean":"2024-08-17T17:40:29.402465+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:40:29.402465+0000","last_undegraded":"2024-08-17T17:40:29.402465+0000","last_fullsized":"2024-08-17T17:40:29.402465+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":300,"reported_epoch":380,"state":"active+clean","last_fresh":"2024-08-17T17:40:41.430596+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:40:41.430596+0000","last_peered":"2024-08-17T17:40:41.430596+0000","last_clean":"2024-08-17T17:40:41.430596+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:40:41.430596+0000","last_undegraded":"2024-08-17T17:40:41.430596+0000","last_fullsized":"2024-08-17T17:40:41.430596+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4749459480,"num_objects":72472,"num_object_clones":0,"num_object_copies":217416,"num_objects_missing_on_primary":5209,"num_objects_missing":5209,"num_objects_degraded":53466,"num_objects_misplaced":28742,"num_objects_unfound":0,"num_objects_dirty":72472,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163918,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28683,"num_bytes_recovered":1757216768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15778787328,"data_stored":15778775112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104437,"ondisk_log_size":104437,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170092,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463461,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2936540,"kb_used_data":2038672,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":90808612,"statfs":{"total":95995035648,"available":92988018688,"internally_reserved":0,"allocated":2087600128,"data_stored":2085538549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492136,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":984444,"kb_used_data":584948,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92760708,"statfs":{"total":95995035648,"available":94986964992,"internally_reserved":0,"allocated":598986752,"data_stored":596947473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986727,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2343308,"kb_used_data":1158148,"kb_used_omap":1,"kb_used_meta":1185150,"kb_avail":91401844,"statfs":{"total":95995035648,"available":93595488256,"internally_reserved":0,"allocated":1185943552,"data_stored":1183892913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213593944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,3,0,0,0,2,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225555,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3252732,"kb_used_data":2310644,"kb_used_omap":4,"kb_used_meta":942075,"kb_avail":90492420,"statfs":{"total":95995035648,"available":92664238080,"internally_reserved":0,"allocated":2366099456,"data_stored":2364060475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":964685248},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075880,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533773,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3071684,"kb_used_data":2033848,"kb_used_omap":5,"kb_used_meta":1037818,"kb_avail":90673468,"statfs":{"total":95995035648,"available":92849631232,"internally_reserved":0,"allocated":2082660352,"data_stored":2080616049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1062725872},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089750,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2760344,"kb_used_data":1746960,"kb_used_omap":5,"kb_used_meta":1013370,"kb_avail":90984808,"statfs":{"total":95995035648,"available":93168443392,"internally_reserved":0,"allocated":1788887040,"data_stored":1786825520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1037690939},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1781792768,"data_stored":1781792768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2075660288,"data_stored":2075656216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2359885824,"data_stored":2359885824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1179385856,"data_stored":1179385856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593362944,"data_stored":593362944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2080505856,"data_stored":2080505856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2368933888,"data_stored":2368929816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:49.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:40:49.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:49.980 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:49.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:50.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:50.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:50.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:50.230 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:50.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:50.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:50.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:50.355 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:50.355 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:50.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:50.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:40:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:40:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:50.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:50.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:50.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:50.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:50.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:50.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:50.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:50.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:51.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:51.098 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:51.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:51.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:51.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.351 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:51.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:51.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:51.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:51.560 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:51 smithi142 ceph-mon[106729]: pgmap v1428: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 53400/216327 objects degraded (24.685%); 28544/216327 objects misplaced (13.195%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:40:51.560 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53400/216327 objects degraded (24.685%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:51.560 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:51 smithi142 ceph-mon[103614]: pgmap v1428: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 53400/216327 objects degraded (24.685%); 28544/216327 objects misplaced (13.195%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:40:51.561 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53400/216327 objects degraded (24.685%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:51.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:51.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:51.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:51.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:51.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:51 smithi152 ceph-mon[93456]: pgmap v1428: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 53400/216327 objects degraded (24.685%); 28544/216327 objects misplaced (13.195%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:40:51.861 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53400/216327 objects degraded (24.685%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:51.862 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:51.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:40:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:40:51] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:40:51.883 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:51.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:51.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:52.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:52.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:52.131 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:52.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:52.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:52.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:52.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:52.262 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:52.356 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:52.378 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:52.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:52.488 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:40:52.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:52.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/570330804' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:52 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/570330804' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:52 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:52.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:52.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:52.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/570330804' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:52 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:52.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:52.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:52.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:52.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:52.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:53.101 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:53.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:53.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:53.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:53.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:53.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:53.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:53.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:53.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:53.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:53.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:53.616 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:53.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[106729]: pgmap v1429: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 53400/215553 objects degraded (24.773%); 28522/215553 objects misplaced (13.232%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[106729]: osdmap e381: 8 total, 8 up, 6 in 2024-08-17T17:40:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[103614]: pgmap v1429: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 53400/215553 objects degraded (24.773%); 28522/215553 objects misplaced (13.232%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:53.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:53 smithi142 ceph-mon[103614]: osdmap e381: 8 total, 8 up, 6 in 2024-08-17T17:40:53.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:53.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:53 smithi152 ceph-mon[93456]: pgmap v1429: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 53400/215553 objects degraded (24.773%); 28522/215553 objects misplaced (13.232%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:40:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:53 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:53 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:53 smithi152 ceph-mon[93456]: osdmap e381: 8 total, 8 up, 6 in 2024-08-17T17:40:53.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:53.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:53.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:53.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:54.013 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:54.013 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:54.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:54.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:54.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:54.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:54.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:54.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:54.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:54.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:54.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:54.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:54.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:54.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:54.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:40:54.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:54.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:54.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:54.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.984 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:54.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:55.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:55.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:55.233 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:55.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:55.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:55.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:55.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:55.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:55.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:55 smithi152 ceph-mon[93456]: pgmap v1431: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 53344/215178 objects degraded (24.791%); 28406/215178 objects misplaced (13.201%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:55.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.869 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:55 smithi142 ceph-mon[106729]: pgmap v1431: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 53344/215178 objects degraded (24.791%); 28406/215178 objects misplaced (13.201%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:55.870 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:55 smithi142 ceph-mon[103614]: pgmap v1431: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 53344/215178 objects degraded (24.791%); 28406/215178 objects misplaced (13.201%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:55.925 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:55.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:55.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:56.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:56.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:56.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:56.229 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:56.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:56.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:56.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:56.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:56.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:56.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:56.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:56.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:56.745 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53344/215178 objects degraded (24.791%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:56.745 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:56.745 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:56.745 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53344/215178 objects degraded (24.791%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:56.745 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:56.745 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53344/215178 objects degraded (24.791%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:40:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:56.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:56.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:56.861 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:40:56.861 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:56.865 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:40:56.865 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:56.883 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:40:56.883 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:56.901 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:40:56.901 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:56.918 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:40:56.918 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:56.935 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:40:56.935 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:56.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:56.952 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:40:56.953 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:56.967 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:40:56.967 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:56.973 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:57.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:57.098 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:57.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:57.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:57.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:57.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:57.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:57.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:57.471 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:57.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:40:57.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:57.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:57.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:57 smithi142 ceph-mon[103614]: pgmap v1432: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 53344/214716 objects degraded (24.844%); 28406/214716 objects misplaced (13.230%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:57.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3786378045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:57 smithi142 ceph-mon[106729]: pgmap v1432: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 53344/214716 objects degraded (24.844%); 28406/214716 objects misplaced (13.230%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3786378045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:57.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:57.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:57.821 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:57 smithi152 ceph-mon[93456]: pgmap v1432: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 53344/214716 objects degraded (24.844%); 28406/214716 objects misplaced (13.230%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3786378045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:57.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:57.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:57.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:58.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:58.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:58.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:58.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:58.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:58.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:58.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:58.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:58.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:58.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:58.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:58.688 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:58.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:58.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:58.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:58.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:58.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:58.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:59.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:59.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:59.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:59.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:59.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:59.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:59.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:59.451 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:59.517 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:59.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:59.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:59.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:59.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:40:59.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:59.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:59.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:40:59 smithi152 ceph-mon[93456]: pgmap v1433: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 53344/214404 objects degraded (24.880%); 28391/214404 objects misplaced (13.242%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:40:59 smithi142 ceph-mon[106729]: pgmap v1433: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 53344/214404 objects degraded (24.880%); 28391/214404 objects misplaced (13.242%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:40:59 smithi142 ceph-mon[103614]: pgmap v1433: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 53344/214404 objects degraded (24.880%); 28391/214404 objects misplaced (13.242%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:40:59.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:59.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:40:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:00.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:00.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:00.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:00.175 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:00.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:00.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:00.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:00.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:00.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:00.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:00.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:00.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:41:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:41:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:00.641 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:41:00.641 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:41:00.641 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:122681 12.a 12:56545667:::benchmark_data_smithi142_261410_object4484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:26.570381+0000", 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 454.06887494, 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.30745330999999998, 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.642 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 122681 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570381+0000", 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570381+0000", 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.643 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570383+0000", 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570389+0000", 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1299999999999998e-06 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.644 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570403+0000", 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3514e-05 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.570410+0000", 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3989999999999998e-06 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.574337+0000", 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039269960000000003 2024-08-17T17:41:00.645 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.574511+0000", 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017386800000000001 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.655496+0000", 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.080985594999999994 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.646 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.655606+0000", 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010913600000000001 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.655706+0000", 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000100478 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.647 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.674413+0000", 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018706901000000001 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.674422+0000", 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7150000000000004e-06 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.871241+0000", 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.196819094 2024-08-17T17:41:00.648 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.871281+0000", 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0666e-05 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.877835+0000", 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0065531230000000001 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.649 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:122686 12.a 12:52048b4e:::benchmark_data_smithi142_261410_object4489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 454.06393458399998, 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.277351352, 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.650 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 122686 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.651 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575322+0000", 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.09e-07 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575326+0000", 2024-08-17T17:41:00.652 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9639999999999997e-06 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575337+0000", 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1129e-05 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.575344+0000", 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.81e-06 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.653 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.578351+0000", 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030065280000000001 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.578523+0000", 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017180599999999999 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.578690+0000", 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016701800000000001 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.654 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.582865+0000", 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041752320000000001 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.582901+0000", 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5935000000000001e-05 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.655 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.847981+0000", 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.26508042399999998 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.848017+0000", 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5571999999999998e-05 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:26.852673+0000", 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046561249999999997 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.656 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.657 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.657 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124366 12.a 12:540a96b7:::benchmark_data_smithi142_261410_object6169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.839486+0000", 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 450.79977052800001, 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.376478849, 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124366 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.658 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839486+0000", 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839486+0000", 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.659 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839487+0000", 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0419999999999999e-06 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839491+0000", 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2169999999999996e-06 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839502+0000", 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0606e-05 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.660 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839507+0000", 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2660000000000001e-06 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839565+0000", 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8396999999999997e-05 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:00.661 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839679+0000", 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011374099999999999 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192319+0000", 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35264041299999999 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192415+0000", 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5812999999999999e-05 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.662 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192566+0000", 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000151161 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.210989+0000", 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018422581 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.211004+0000", 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4922e-05 2024-08-17T17:41:00.663 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.211038+0000", 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4196999999999998e-05 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.211230+0000", 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019243299999999999 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.215964+0000", 2024-08-17T17:41:00.664 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00473406 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131382 12.b 12:d72dad81:::benchmark_data_smithi142_261410_object13185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 361.15682706600001, 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.96633504400000003, 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.665 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131382 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.666 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482430+0000", 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5700000000000002e-07 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482433+0000", 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.692e-06 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482445+0000", 2024-08-17T17:41:00.667 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1564e-05 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482450+0000", 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5159999999999996e-06 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482507+0000", 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7800000000000002e-05 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.668 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482519+0000", 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1567e-05 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.436998+0000", 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95447953500000005 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.437123+0000", 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012464299999999999 2024-08-17T17:41:00.669 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.437248+0000", 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012493899999999999 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448394+0000", 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011145649000000001 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448408+0000", 2024-08-17T17:41:00.670 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4669e-05 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448665+0000", 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000256531 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448725+0000", 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0028999999999997e-05 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.671 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448764+0000", 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9252999999999998e-05 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131388 12.b 12:d3e07cb9:::benchmark_data_smithi142_261410_object13191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 361.08800154699998, 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.89768371400000002, 2024-08-17T17:41:00.672 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131388 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.673 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551256+0000", 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5230000000000001e-06 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.674 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551262+0000", 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7860000000000002e-06 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551283+0000", 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.084e-05 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551292+0000", 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4839999999999993e-06 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.675 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551356+0000", 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4068999999999999e-05 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551365+0000", 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8510000000000005e-06 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441263+0000", 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88989788199999997 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.676 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441385+0000", 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121975 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441494+0000", 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108819 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448506+0000", 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0070124380000000002 2024-08-17T17:41:00.677 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448539+0000", 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3089000000000001e-05 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448827+0000", 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00028744399999999998 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.678 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448896+0000", 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8782000000000007e-05 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448938+0000", 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2731999999999997e-05 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.679 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131408 12.b 12:df15f4a1:::benchmark_data_smithi142_261410_object13211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 360.64579990599998, 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.45568646400000001, 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.680 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131408 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:41:00.681 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993458+0000", 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0509999999999999e-06 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993466+0000", 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5270000000000001e-06 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.682 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993479+0000", 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3621000000000001e-05 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993488+0000", 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2369999999999992e-06 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993547+0000", 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9639000000000003e-05 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.683 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993556+0000", 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9940000000000006e-06 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444651+0000", 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.45109483099999997 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444777+0000", 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012631799999999999 2024-08-17T17:41:00.684 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444902+0000", 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000124721 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448979+0000", 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040765480000000002 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.685 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449036+0000", 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7623e-05 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449058+0000", 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1386000000000002e-05 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449104+0000", 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6388999999999999e-05 2024-08-17T17:41:00.686 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449143+0000", 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8578999999999998e-05 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:137199 12.a 12:52e19bf6:::benchmark_data_smithi142_261410_object19002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 285.16054753999998, 2024-08-17T17:41:00.687 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.056001591, 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 137199 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.688 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478710+0000", 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4600000000000003e-07 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.689 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478714+0000", 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7330000000000003e-06 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478724+0000", 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478731+0000", 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1169999999999998e-06 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.690 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478795+0000", 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4641999999999995e-05 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478804+0000", 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531212+0000", 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.052407855 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.691 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531320+0000", 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000107692 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531454+0000", 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134055 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534306+0000", 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028519069999999999 2024-08-17T17:41:00.692 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534367+0000", 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1266000000000006e-05 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534598+0000", 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000231373 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.693 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534651+0000", 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2369e-05 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534710+0000", 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9517e-05 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.694 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:142322 12.8 12:1e02f276:::benchmark_data_smithi142_261410_object24125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:16.716575+0000", 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 223.92268126900001, 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.021135414000000002, 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 142322 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.695 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716575+0000", 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716575+0000", 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716576+0000", 2024-08-17T17:41:00.696 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7699999999999992e-07 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716580+0000", 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0219999999999998e-06 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716589+0000", 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3130000000000004e-06 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.697 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716594+0000", 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0159999999999999e-06 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716647+0000", 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2979999999999998e-05 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716769+0000", 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121487 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.698 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.716874+0000", 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105548 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.720439+0000", 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035646580000000001 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.720788+0000", 2024-08-17T17:41:00.699 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00034901099999999998 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.733838+0000", 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013050526 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.733908+0000", 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9896999999999998e-05 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.737710+0000", 2024-08-17T17:41:00.700 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003801979 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:142323 12.8 12:19697d1d:::benchmark_data_smithi142_261410_object24126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:16.721792+0000", 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 223.91746385499999, 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019865588, 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.701 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 142323 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721792+0000", 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721792+0000", 2024-08-17T17:41:00.702 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721794+0000", 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9400000000000001e-06 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721801+0000", 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0709999999999997e-06 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.703 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721812+0000", 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1171e-05 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721820+0000", 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2830000000000004e-06 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.721871+0000", 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0908000000000001e-05 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.704 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.722043+0000", 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000172545 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.722157+0000", 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011375900000000001 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.725697+0000", 2024-08-17T17:41:00.705 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035404310000000001 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.725808+0000", 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000110827 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.737802+0000", 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011994207999999999 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.706 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.737881+0000", 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8807000000000006e-05 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:16.741658+0000", 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037766380000000001 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.707 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146690 12.8 12:1be560a4:::benchmark_data_smithi142_261410_object28493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.626791+0000", 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 178.01246560300001, 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.026113173999999999, 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.708 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146690 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626791+0000", 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626791+0000", 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.709 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626792+0000", 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6099999999999999e-07 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626795+0000", 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8340000000000001e-06 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.710 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626808+0000", 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2320999999999999e-05 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626815+0000", 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9229999999999996e-06 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626860+0000", 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5343999999999998e-05 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.711 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626973+0000", 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011325299999999999 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.627073+0000", 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9480000000000003e-05 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633055+0000", 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059819809999999999 2024-08-17T17:41:00.712 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633121+0000", 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6036000000000003e-05 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.650236+0000", 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017115253 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.650277+0000", 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0666e-05 2024-08-17T17:41:00.713 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.652904+0000", 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002627122 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146692 12.8 12:11913193:::benchmark_data_smithi142_261410_object28495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 178.01009865399999, 2024-08-17T17:41:00.714 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025921260000000002, 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146692 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.715 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.716 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:00.717 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.717 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:41:00.717 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.717 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.717 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.717 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1099999999999995e-07 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629162+0000", 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7390000000000001e-06 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629171+0000", 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9160000000000007e-06 2024-08-17T17:41:00.718 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629177+0000", 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5339999999999996e-06 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629761+0000", 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058357700000000005 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.719 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629901+0000", 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014035100000000001 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.630040+0000", 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000139147 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633107+0000", 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00306668 2024-08-17T17:41:00.720 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633129+0000", 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1738000000000001e-05 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.652958+0000", 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019828983000000001 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.652998+0000", 2024-08-17T17:41:00.721 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0197999999999997e-05 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.655079+0000", 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002080686 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:153271 12.8 12:142b9aba:::benchmark_data_smithi142_261410_object35074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:09.832566+0000", 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 110.80668978200001, 2024-08-17T17:41:00.722 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.020760174999999999, 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 153271 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.832566+0000", 2024-08-17T17:41:00.723 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.832566+0000", 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.832568+0000", 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3859999999999999e-06 2024-08-17T17:41:00.724 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.832572+0000", 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6609999999999999e-06 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.832584+0000", 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1114999999999999e-05 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.832591+0000", 2024-08-17T17:41:00.725 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1640000000000004e-06 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.835570+0000", 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002979027 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.835696+0000", 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012633699999999999 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.726 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.835798+0000", 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000102235 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.840681+0000", 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048831279999999996 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.841286+0000", 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00060483200000000003 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.727 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.850808+0000", 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0095218509999999996 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.850847+0000", 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8399999999999998e-05 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:09.853327+0000", 2024-08-17T17:41:00.728 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024800389999999999 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:154582 12.8 12:1f77bd10:::benchmark_data_smithi142_261410_object36385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:22.260308+0000", 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 98.378948167999994, 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.021346014, 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.729 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 154582 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260308+0000", 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.730 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260308+0000", 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260309+0000", 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2410000000000001e-06 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260314+0000", 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7480000000000001e-06 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260326+0000", 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1620000000000001e-05 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260332+0000", 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9410000000000004e-06 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.732 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260381+0000", 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9429999999999999e-05 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260498+0000", 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116826 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.260604+0000", 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000106339 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.733 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.263156+0000", 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025519539999999999 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.263340+0000", 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018348799999999999 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.277725+0000", 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014385409 2024-08-17T17:41:00.734 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.277767+0000", 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1891000000000003e-05 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.281654+0000", 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038871270000000002 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.735 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:154583 12.8 12:197bac67:::benchmark_data_smithi142_261410_object36386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:22.265021+0000", 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 98.374235365999994, 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.02024929, 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 154583 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.736 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265021+0000", 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265021+0000", 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.737 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265022+0000", 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1429999999999999e-06 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265026+0000", 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5070000000000002e-06 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265038+0000", 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1858000000000001e-05 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.738 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265045+0000", 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4829999999999996e-06 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265087+0000", 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2030999999999998e-05 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265195+0000", 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108247 2024-08-17T17:41:00.739 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.265296+0000", 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010076700000000001 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.268640+0000", 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033443219999999998 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.740 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.268844+0000", 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020384299999999999 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.281709+0000", 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01286459 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.281749+0000", 2024-08-17T17:41:00.741 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0339000000000001e-05 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:22.285270+0000", 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035211600000000002 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.742 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:155716 12.8 12:10430180:::benchmark_data_smithi142_261410_object37519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:32.278743+0000", 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 88.360513162999993, 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.023586887000000001, 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 155716 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.743 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.278743+0000", 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.278743+0000", 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.278744+0000", 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.006e-06 2024-08-17T17:41:00.744 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.278749+0000", 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.566e-06 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.278767+0000", 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8148999999999999e-05 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.745 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.278775+0000", 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1470000000000001e-06 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.280507+0000", 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0017322450000000001 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.746 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.280668+0000", 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000161084 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.280816+0000", 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014824599999999999 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.283763+0000", 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029466240000000001 2024-08-17T17:41:00.747 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.283957+0000", 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00019367000000000001 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.299258+0000", 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015300879999999999 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.748 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.299298+0000", 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0334999999999999e-05 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:32.302330+0000", 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030319349999999999 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.749 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:157304 12.8 12:1c13b566:::benchmark_data_smithi142_261410_object39107:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:46.533667+0000", 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 74.105588875999999, 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019132639999999999, 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 157304 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.750 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533667+0000", 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533667+0000", 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.751 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533669+0000", 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.793e-06 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533673+0000", 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3069999999999996e-06 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533684+0000", 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0956e-05 2024-08-17T17:41:00.752 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533690+0000", 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533740+0000", 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0142999999999997e-05 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.753 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533856+0000", 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115024 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.533974+0000", 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011852 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.536635+0000", 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026611920000000002 2024-08-17T17:41:00.754 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.536999+0000", 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036383699999999999 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.550263+0000", 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013263637999999999 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.755 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.550320+0000", 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6906e-05 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:46.552800+0000", 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024803239999999999 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.756 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPush(12.d 376/337 [PushOp(12:b17145dc:::benchmark_data_smithi142_261410_object40071:head, version: 324'2525, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:b17145dc:::benchmark_data_smithi142_261410_object40071:head@324'2525, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.829676+0000", 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 65.809579866999997, 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.033488366999999998, 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829676+0000", 2024-08-17T17:41:00.757 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829676+0000", 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829677+0000", 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7400000000000002e-07 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.758 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829705+0000", 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7934999999999999e-05 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829721+0000", 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5410999999999999e-05 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.759 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829732+0000", 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1357e-05 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829804+0000", 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2395000000000001e-05 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829813+0000", 2024-08-17T17:41:00.760 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4640000000000006e-06 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.863165+0000", 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.033351931000000001 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.761 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:158269 12.e e376/337 12:7a28666e:::benchmark_data_smithi142_261410_object40072:head v 376'9843, mlcod=324'725)", 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 65.808261373999997, 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.037591274000000001, 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:41:00.762 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830996+0000", 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9500000000000001e-07 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.763 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.831003+0000", 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9709999999999998e-06 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861303+0000", 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030299947000000001 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.764 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861329+0000", 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6420000000000001e-05 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861402+0000", 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2539999999999993e-05 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861430+0000", 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8707000000000001e-05 2024-08-17T17:41:00.765 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.868547+0000", 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0071166140000000003 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.868586+0000", 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9180000000000001e-05 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.766 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:161157 12.8 12:1760b7fe:::benchmark_data_smithi142_261410_object42960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:19.737796+0000", 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 40.901460165000003, 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.023894114000000001, 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.767 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 161157 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.737796+0000", 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.737796+0000", 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.768 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.737797+0000", 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5899999999999995e-07 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.737801+0000", 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0890000000000002e-06 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.737810+0000", 2024-08-17T17:41:00.769 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1789999999999997e-06 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.737817+0000", 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.579e-06 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.741360+0000", 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003543488 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.770 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.741484+0000", 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000124225 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.741597+0000", 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112697 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.748035+0000", 2024-08-17T17:41:00.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064376090000000004 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.748097+0000", 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2149000000000004e-05 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.757292+0000", 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0091948819999999997 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.757365+0000", 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2837999999999996e-05 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.761690+0000", 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043255200000000002 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.773 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:161158 12.8 12:1a81df31:::benchmark_data_smithi142_261410_object42961:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:19.738953+0000", 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 40.900303129999998, 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027048839000000002, 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 161158 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:41:00.774 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738953+0000", 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738953+0000", 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738954+0000", 2024-08-17T17:41:00.775 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1289999999999999e-06 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738958+0000", 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0600000000000001e-06 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738970+0000", 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1385e-05 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:41:00.776 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738976+0000", 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5649999999999998e-06 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.744444+0000", 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054678749999999996 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.744628+0000", 2024-08-17T17:41:00.777 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018376199999999999 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.744801+0000", 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017283500000000001 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.748151+0000", 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003350319 2024-08-17T17:41:00.778 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.748172+0000", 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1104999999999999e-05 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.761762+0000", 2024-08-17T17:41:00.779 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013589752 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.761832+0000", 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0297999999999997e-05 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.766002+0000", 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041697540000000003 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.780 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:41:00.781 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:41:00.781 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:41:00.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:00.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:00.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:00.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:00.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:01.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:01.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:01.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.238 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:01.238 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 53274/213693 objects degraded (24.930%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2141,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":381,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":71231,"data_bytes":4667021285,"bytes_used":15532253184,"bytes_avail":560437960704,"bytes_total":575970213888,"degraded_objects":53274,"degraded_total":213693,"degraded_ratio":0.24930156813746823,"misplaced_objects":28234,"misplaced_total":213693,"misplaced_ratio":0.13212412198808571,"recovering_objects_per_sec":17,"recovering_bytes_per_sec":1146600,"recovering_keys_per_sec":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":87},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:41:01.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:01.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:01.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:01.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:01.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:01.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:01.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:01.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:01.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:01.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:01.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:01.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-mon[106729]: pgmap v1434: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 53274/213693 objects degraded (24.930%); 28234/213693 objects misplaced (13.212%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:41:01.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53274/213693 objects degraded (24.930%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:01.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3278991798' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:01.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-mon[103614]: pgmap v1434: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 53274/213693 objects degraded (24.930%); 28234/213693 objects misplaced (13.212%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:41:01.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53274/213693 objects degraded (24.930%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:01.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3278991798' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:01.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:01.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:01.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:01 smithi152 ceph-mon[93456]: pgmap v1434: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 53274/213693 objects degraded (24.930%); 28234/213693 objects misplaced (13.212%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:41:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53274/213693 objects degraded (24.930%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3278991798' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:01.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:01.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:02.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:02.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:41:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:41:01] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-08-17T17:41:02.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:02.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:02.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:02.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:02.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:41:02.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:02.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:02.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:02.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:02.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:02.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:02.712 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:02.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:02.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:02.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:02.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:02.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:02.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:03.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:03.096 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:03.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:03.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:03.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:03.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:03.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:03.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:03.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:03.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:03.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:03.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:03 smithi152 ceph-mon[93456]: pgmap v1435: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 53274/213015 objects degraded (25.010%); 28205/213015 objects misplaced (13.241%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:41:03.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:03.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:03 smithi142 ceph-mon[106729]: pgmap v1435: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 53274/213015 objects degraded (25.010%); 28205/213015 objects misplaced (13.241%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:41:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:03 smithi142 ceph-mon[103614]: pgmap v1435: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 53274/213015 objects degraded (25.010%); 28205/213015 objects misplaced (13.241%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:41:03.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:03.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:04.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:04.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:04.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:04.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:04.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:04.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:04.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:04.396 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:41:04.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:04.425 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:04.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:04.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:04.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:04.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:04.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:04.673 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:04 smithi142 ceph-mon[106729]: pgmap v1436: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 53195/212454 objects degraded (25.038%); 28021/212454 objects misplaced (13.189%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:41:04.673 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:04 smithi142 ceph-mon[103614]: pgmap v1436: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 53195/212454 objects degraded (25.038%); 28021/212454 objects misplaced (13.189%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:41:04.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:04 smithi152 ceph-mon[93456]: pgmap v1436: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 53195/212454 objects degraded (25.038%); 28021/212454 objects misplaced (13.189%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:41:04.675 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165754 12.7 12:e26ce91a:::benchmark_data_smithi142_261410_object47557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.026721+0000", 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.6469896149999999, 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6470165030000001, 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.676 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165754 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.026721+0000", 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.026721+0000", 2024-08-17T17:41:04.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.026719+0000", 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.026727+0000", 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8979999999999996e-06 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.026739+0000", 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2293000000000001e-05 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.026745+0000", 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6570000000000002e-06 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.071815+0000", 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.045070230000000003 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.081166+0000", 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093509189999999992 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.127434+0000", 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046267363999999998 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.134771+0000", 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0073372749999999999 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165902 12.7 12:e6670d3d:::benchmark_data_smithi142_261410_object47705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.681 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.334870+0000", 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.33884029900000001, 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33892540700000001, 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165902 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.334870+0000", 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.334870+0000", 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.334869+0000", 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.334875+0000", 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6309999999999997e-06 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.334891+0000", 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5829e-05 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.334898+0000", 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9519999999999997e-06 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.451645+0000", 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.116747082 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.460251+0000", 2024-08-17T17:41:04.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086057259999999993 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.527849+0000", 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.067597870000000004 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.535846+0000", 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079968590000000003 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165822 12.7 12:e512d275:::benchmark_data_smithi142_261410_object47625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.554742+0000", 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.118968505, 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.119103008, 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165822 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.554742+0000", 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.554742+0000", 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.554740+0000", 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:41:04.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.554753+0000", 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3074999999999999e-05 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.554770+0000", 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7320999999999999e-05 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.554779+0000", 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7690000000000002e-06 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.686437+0000", 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13165787500000001 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.692873+0000", 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064366500000000004 2024-08-17T17:41:04.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.745575+0000", 2024-08-17T17:41:04.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.052701949999999997 2024-08-17T17:41:04.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.753049+0000", 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074735360000000002 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165825 12.7 12:e23a67b1:::benchmark_data_smithi142_261410_object47628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.566153+0000", 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.107557119, 2024-08-17T17:41:04.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.107744984, 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165825 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.566153+0000", 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.566153+0000", 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.566151+0000", 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:04.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.566163+0000", 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1765e-05 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.566178+0000", 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5279000000000001e-05 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.566185+0000", 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4710000000000004e-06 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.693946+0000", 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.127761338 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.701710+0000", 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077636839999999999 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.814694+0000", 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.112984211 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.821758+0000", 2024-08-17T17:41:04.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.007064018 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:165934 12.f e381/332 12:fab9f8c7:::benchmark_data_smithi142_261410_object47737:head v 381'10365, mlcod=322'102)", 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.534812+0000", 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.13889849800000001, 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13912892399999999, 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.534812+0000", 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.534812+0000", 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.534811+0000", 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:04.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.534819+0000", 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6219999999999998e-06 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.534845+0000", 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5531999999999999e-05 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.534851+0000", 2024-08-17T17:41:04.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2899999999999999e-06 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.588204+0000", 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.053352916 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.588225+0000", 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0568999999999999e-05 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.702 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165759 12.7 12:eaa010b3:::benchmark_data_smithi142_261410_object47562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.036301+0000", 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.6374088599999999, 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.637671184, 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165759 2024-08-17T17:41:04.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.036301+0000", 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.036301+0000", 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.036300+0000", 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.036307+0000", 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8279999999999997e-06 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.036319+0000", 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2153999999999999e-05 2024-08-17T17:41:04.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.036324+0000", 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.079e-06 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.082372+0000", 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046048527999999998 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.092493+0000", 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010120931999999999 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.163313+0000", 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.070819358999999998 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.171131+0000", 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078185449999999997 2024-08-17T17:41:04.707 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165688 12.7 12:e43a4cdb:::benchmark_data_smithi142_261410_object47491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:02.577453+0000", 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.0962570829999998, 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.096561844, 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.708 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165688 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.577453+0000", 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.577453+0000", 2024-08-17T17:41:04.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.577451+0000", 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.577460+0000", 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9940000000000006e-06 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.577475+0000", 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5071e-05 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.577482+0000", 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.815e-06 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.716593+0000", 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13911129899999999 2024-08-17T17:41:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.727380+0000", 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010786666 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.884150+0000", 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15677018000000001 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.891957+0000", 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078066480000000002 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165828 12.7 12:e9b3fe2a:::benchmark_data_smithi142_261410_object47631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.679636+0000", 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.994074385, 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99441952099999997, 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.713 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165828 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.679636+0000", 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.679636+0000", 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.679635+0000", 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.679641+0000", 2024-08-17T17:41:04.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2999999999999998e-06 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.679653+0000", 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1725999999999999e-05 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.679660+0000", 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4440000000000002e-06 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.703036+0000", 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023376312999999999 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.712881+0000", 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098443750000000007 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.779066+0000", 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.066185401000000005 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.786189+0000", 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071229459999999998 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.718 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165737 12.7 12:ef54b645:::benchmark_data_smithi142_261410_object47540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:02.925326+0000", 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.7483842329999999, 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7487698250000001, 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165737 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.925326+0000", 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.925326+0000", 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.925324+0000", 2024-08-17T17:41:04.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.925332+0000", 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0069999999999997e-06 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.925345+0000", 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2418999999999999e-05 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.925352+0000", 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3830000000000003e-06 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.986071+0000", 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.060718794999999999 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:02.992780+0000", 2024-08-17T17:41:04.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067089949999999997 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.061221+0000", 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.068441020000000005 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.068657+0000", 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074359539999999998 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.723 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165808 12.7 12:e4996a6d:::benchmark_data_smithi142_261410_object47611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.422792+0000", 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.2509183850000001, 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2513445860000001, 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165808 2024-08-17T17:41:04.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.422792+0000", 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.422792+0000", 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.422790+0000", 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.422799+0000", 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3690000000000007e-06 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.422811+0000", 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1950000000000001e-05 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.422816+0000", 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.801e-06 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.510316+0000", 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.087499936 2024-08-17T17:41:04.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.519758+0000", 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094415969999999995 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.651987+0000", 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13222898199999999 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.659608+0000", 2024-08-17T17:41:04.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.007621142 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165909 12.7 12:e6dd3217:::benchmark_data_smithi142_261410_object47712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.376585+0000", 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.29712494299999997, 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29759120500000003, 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.729 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165909 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.376585+0000", 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.376585+0000", 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.376584+0000", 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.376592+0000", 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4100000000000008e-06 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.376606+0000", 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3764000000000001e-05 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.376615+0000", 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8999999999999995e-06 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.461276+0000", 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.084661316 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.473729+0000", 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012452555000000001 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.618391+0000", 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14466267299999999 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.625947+0000", 2024-08-17T17:41:04.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0075557970000000004 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165810 12.7 12:e63064c3:::benchmark_data_smithi142_261410_object47613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.481414+0000", 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.1922959660000001, 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1928044280000001, 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.734 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165810 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.481414+0000", 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.481414+0000", 2024-08-17T17:41:04.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.481412+0000", 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.481422+0000", 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4870000000000008e-06 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.481435+0000", 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3723999999999999e-05 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.481441+0000", 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8690000000000001e-06 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.520523+0000", 2024-08-17T17:41:04.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039081892 2024-08-17T17:41:04.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.527717+0000", 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071940149999999998 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.614497+0000", 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.086779427000000006 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.621680+0000", 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071836640000000002 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165843 12.7 12:ef1c2dd1:::benchmark_data_smithi142_261410_object47646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.122061+0000", 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.55164946199999998, 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.55221108500000005, 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.740 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165843 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.122061+0000", 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.122061+0000", 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.122059+0000", 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.122065+0000", 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3090000000000001e-06 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.122077+0000", 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1764e-05 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.122082+0000", 2024-08-17T17:41:04.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7310000000000001e-06 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.211876+0000", 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089793778000000005 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.230119+0000", 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018242857000000001 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.440184+0000", 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.210065007 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.450533+0000", 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010349168000000001 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.746 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165944 12.7 12:ededf808:::benchmark_data_smithi142_261410_object47747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.746 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.610780+0000", 2024-08-17T17:41:04.746 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.062929726000000005, 2024-08-17T17:41:04.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.063531894000000005, 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for missing object", 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165944 2024-08-17T17:41:04.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.610780+0000", 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.610780+0000", 2024-08-17T17:41:04.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.610778+0000", 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.610786+0000", 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5599999999999996e-06 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.610800+0000", 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3711e-05 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.610806+0000", 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3389999999999998e-06 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.626644+0000", 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015837674999999999 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.633503+0000", 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068590999999999999 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.751 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(12.7 381/337 [PushOp(12:ededf808:::benchmark_data_smithi142_261410_object47747:head, version: 325'3010, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:ededf808:::benchmark_data_smithi142_261410_object47747:head@325'3010, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.634362+0000", 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.039347739, 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039997999999999999, 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.634362+0000", 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.634362+0000", 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.634361+0000", 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.634417+0000", 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6041000000000001e-05 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.634434+0000", 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6966999999999999e-05 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.634446+0000", 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2224e-05 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.654517+0000", 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020070937 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.655220+0000", 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070294599999999997 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.755 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165853 12.7 12:e3a97a45:::benchmark_data_smithi142_261410_object47656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.172015+0000", 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.50169496000000002, 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.50239257000000004, 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165853 2024-08-17T17:41:04.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.172015+0000", 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.172015+0000", 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.172014+0000", 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.172021+0000", 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2010000000000002e-06 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.172034+0000", 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3227000000000001e-05 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.172040+0000", 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7300000000000002e-06 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.231910+0000", 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.059870564000000001 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.243315+0000", 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011404625999999999 2024-08-17T17:41:04.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.354666+0000", 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11135104799999999 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.362716+0000", 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0080501319999999998 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165919 12.7 12:e1a499c3:::benchmark_data_smithi142_261410_object47722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:04.449414+0000", 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.22429656000000001, 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22503224399999999, 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165919 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.449414+0000", 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.449414+0000", 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.449412+0000", 2024-08-17T17:41:04.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.449419+0000", 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1420000000000004e-06 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.449432+0000", 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.294e-05 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.449438+0000", 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4530000000000001e-06 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.475061+0000", 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025622712999999998 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.484222+0000", 2024-08-17T17:41:04.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091613779999999995 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.572824+0000", 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.088601646000000006 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.585701+0000", 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012877579 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:04.765 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.766 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:04.766 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:41:04.766 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 17 2024-08-17T17:41:04.766 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:41:04.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:04.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:04.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:05.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:05.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:05.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:05.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:05.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:05.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:05.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:05.282 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:05.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:05.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:05.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:05.492 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:41:05.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:05.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:41:05.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:05.613 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2517309582' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:05.613 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:05 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:05.613 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2517309582' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:05.613 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:05 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:05.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:05.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:05.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:05.785 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:05.818 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2517309582' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:05 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:05.887 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:05.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:06.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:06.034 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:06.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:06.159 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:06.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:06.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:06.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:06.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:06.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:06.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:06.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:06.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:06.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:06.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:06 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:06.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:06 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:06.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:06 smithi152 ceph-mon[93456]: osdmap e382: 8 total, 8 up, 6 in 2024-08-17T17:41:06.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53195/212454 objects degraded (25.038%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:06.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:06 smithi152 ceph-mon[93456]: pgmap v1438: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 53195/211716 objects degraded (25.126%); 28021/211716 objects misplaced (13.235%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:06.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:06.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:06.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:06.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[106729]: osdmap e382: 8 total, 8 up, 6 in 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53195/212454 objects degraded (25.038%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[106729]: pgmap v1438: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 53195/211716 objects degraded (25.126%); 28021/211716 objects misplaced (13.235%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[103614]: osdmap e382: 8 total, 8 up, 6 in 2024-08-17T17:41:06.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53195/212454 objects degraded (25.038%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:06.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:06 smithi142 ceph-mon[103614]: pgmap v1438: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 53195/211716 objects degraded (25.126%); 28021/211716 objects misplaced (13.235%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:06.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:06.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:07.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:07.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:07.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:07.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:07.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:07.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:07.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:07.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:07.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:07.443 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:07.446 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:41:07.446 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:07.477 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:41:07.477 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:07.495 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:41:07.495 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:07.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:07.514 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:41:07.514 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:07.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:07.532 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:41:07.532 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:07.548 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:41:07.548 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:07.563 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:41:07.564 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:07.579 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:41:07.579 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:07.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:07.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:07.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:07.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:07.795 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2122753375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:07.810 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:07.810 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2122753375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2122753375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:07.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:07.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:08.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:08.030 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:08.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:08.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:08.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:08.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:08.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:08.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:08.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:08.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:41:08.503 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:08.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:41:08.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:08.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:08 smithi142 ceph-mon[106729]: pgmap v1439: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 53195/211242 objects degraded (25.182%); 27988/211242 objects misplaced (13.249%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:08.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:08 smithi142 ceph-mon[103614]: pgmap v1439: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 53195/211242 objects degraded (25.182%); 27988/211242 objects misplaced (13.249%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:08.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:08.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:08.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:08.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:08.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:08.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:08 smithi152 ceph-mon[93456]: pgmap v1439: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 53195/211242 objects degraded (25.182%); 27988/211242 objects misplaced (13.249%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:08.823 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:08.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:08.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:09.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.152 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.254 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:09.274 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:09.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:09.376 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:09.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:09.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:09.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:09.644 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:09.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:09.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:09.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:09.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:09.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:10.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:10.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:10.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:10.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:10.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:10.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:10.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:10.354 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:10.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:10.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:10.501 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:41:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:41:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:10.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:10.532 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:10.533 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:41:10.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:10.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:10.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:10.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:10.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:10.774 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:10.774 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:10.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:11.010 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:11.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:11.135 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:11.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:11.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:11.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:11.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:11.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:11.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:11.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:11.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:11.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:11.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:11.745 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:11 smithi152 ceph-mon[93456]: pgmap v1440: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53124/210456 objects degraded (25.242%); 27818/210456 objects misplaced (13.218%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:11.746 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53124/210456 objects degraded (25.242%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:11.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:11.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:11.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:11 smithi142 ceph-mon[106729]: pgmap v1440: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53124/210456 objects degraded (25.242%); 27818/210456 objects misplaced (13.218%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53124/210456 objects degraded (25.242%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:11 smithi142 ceph-mon[103614]: pgmap v1440: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53124/210456 objects degraded (25.242%); 27818/210456 objects misplaced (13.218%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53124/210456 objects degraded (25.242%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:11.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:41:11.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:41:11.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:11.873 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1440,"stamp":"2024-08-17T17:41:10.338324+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4596307941,"num_objects":70152,"num_object_clones":0,"num_object_copies":210456,"num_objects_missing_on_primary":5016,"num_objects_missing":5016,"num_objects_degraded":53124,"num_objects_misplaced":27818,"num_objects_unfound":0,"num_objects_dirty":70152,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":166542,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29238,"num_bytes_recovered":1793270414,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104890,"ondisk_log_size":104890,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15001892,"kb_used_data":9559348,"kb_used_omap":27,"kb_used_meta":5442468,"kb_avail":547469020,"statfs":{"total":575970213888,"available":560608276480,"internally_reserved":0,"allocated":9788772352,"data_stored":9776390328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5573087976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,1,0,0,0,3,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-70713344,"num_objects":-1079,"num_object_clones":0,"num_object_copies":-3237,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-150,"num_objects_misplaced":-416,"num_objects_unfound":0,"num_objects_dirty":-1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1079,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":217,"num_bytes_recovered":14221312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002750"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":607,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498344+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:41:05.498344+0000","last_peered":"2024-08-17T17:41:05.498344+0000","last_clean":"2024-08-17T17:41:05.498344+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:41:05.498344+0000","last_undegraded":"2024-08-17T17:41:05.498344+0000","last_fullsized":"2024-08-17T17:41:05.498344+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":91,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473720+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:40:52.473720+0000","last_peered":"2024-08-17T17:40:52.473720+0000","last_clean":"2024-08-17T17:40:52.473720+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:40:52.473720+0000","last_undegraded":"2024-08-17T17:40:52.473720+0000","last_fullsized":"2024-08-17T17:40:52.473720+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":285,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482791+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:40:52.482791+0000","last_peered":"2024-08-17T17:40:52.482791+0000","last_clean":"2024-08-17T17:40:52.482791+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:40:52.482791+0000","last_undegraded":"2024-08-17T17:40:52.482791+0000","last_fullsized":"2024-08-17T17:40:52.482791+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":305,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498648+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:41:05.498648+0000","last_peered":"2024-08-17T17:41:05.498648+0000","last_clean":"2024-08-17T17:41:05.498648+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:41:05.498648+0000","last_undegraded":"2024-08-17T17:41:05.498648+0000","last_fullsized":"2024-08-17T17:41:05.498648+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":482,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498505+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:41:05.498505+0000","last_peered":"2024-08-17T17:41:05.498505+0000","last_clean":"2024-08-17T17:41:05.498505+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:41:05.498505+0000","last_undegraded":"2024-08-17T17:41:05.498505+0000","last_fullsized":"2024-08-17T17:41:05.498505+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":234,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473058+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:40:52.473058+0000","last_peered":"2024-08-17T17:40:52.473058+0000","last_clean":"2024-08-17T17:40:52.473058+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:40:52.473058+0000","last_undegraded":"2024-08-17T17:40:52.473058+0000","last_fullsized":"2024-08-17T17:40:52.473058+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":575,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498704+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:41:05.498704+0000","last_peered":"2024-08-17T17:41:05.498704+0000","last_clean":"2024-08-17T17:41:05.498704+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:41:05.498704+0000","last_undegraded":"2024-08-17T17:41:05.498704+0000","last_fullsized":"2024-08-17T17:41:05.498704+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":402,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483143+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:40:52.483143+0000","last_peered":"2024-08-17T17:40:52.483143+0000","last_clean":"2024-08-17T17:40:52.483143+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:40:52.483143+0000","last_undegraded":"2024-08-17T17:40:52.483143+0000","last_fullsized":"2024-08-17T17:40:52.483143+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"382'10357","reported_seq":15003,"reported_epoch":382,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:41:08.603194+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:41:08.603194+0000","last_peered":"2024-08-17T17:41:08.603194+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:41:08.603194+0000","last_undegraded":"2024-08-17T17:41:08.603194+0000","last_fullsized":"2024-08-17T17:41:08.603194+0000","mapping_epoch":337,"log_start":"325'3463","ondisk_log_start":"325'3463","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6894,"log_dups_size":0,"ondisk_log_size":6894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284884992,"num_objects":4347,"num_object_clones":0,"num_object_copies":13041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4347,"num_objects_unfound":0,"num_objects_dirty":4347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10357,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2276,"num_bytes_recovered":149159936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4347}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":87,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.498858+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:40:52.498858+0000","last_peered":"2024-08-17T17:40:52.498858+0000","last_clean":"2024-08-17T17:40:52.498858+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:40:52.498858+0000","last_undegraded":"2024-08-17T17:40:52.498858+0000","last_fullsized":"2024-08-17T17:40:52.498858+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482304+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:40:52.482304+0000","last_peered":"2024-08-17T17:40:52.482304+0000","last_clean":"2024-08-17T17:40:52.482304+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:40:52.482304+0000","last_undegraded":"2024-08-17T17:40:52.482304+0000","last_fullsized":"2024-08-17T17:40:52.482304+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"382'10565","reported_seq":10645,"reported_epoch":382,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:06.815944+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:41:06.815944+0000","last_peered":"2024-08-17T17:41:06.815944+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:41:06.815944+0000","last_undegraded":"2024-08-17T17:41:06.815944+0000","last_fullsized":"2024-08-17T17:41:06.815944+0000","mapping_epoch":337,"log_start":"326'3982","ondisk_log_start":"326'3982","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295501824,"num_objects":4509,"num_object_clones":0,"num_object_copies":13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4509,"num_objects_unfound":0,"num_objects_dirty":4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10565,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4509}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":89,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.472939+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:40:52.472939+0000","last_peered":"2024-08-17T17:40:52.472939+0000","last_clean":"2024-08-17T17:40:52.472939+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:40:52.472939+0000","last_undegraded":"2024-08-17T17:40:52.472939+0000","last_fullsized":"2024-08-17T17:40:52.472939+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":353,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.472992+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:40:52.472992+0000","last_peered":"2024-08-17T17:40:52.472992+0000","last_clean":"2024-08-17T17:40:52.472992+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:40:52.472992+0000","last_undegraded":"2024-08-17T17:40:52.472992+0000","last_fullsized":"2024-08-17T17:40:52.472992+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"382'10362","reported_seq":10445,"reported_epoch":382,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:08.689619+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:41:08.689619+0000","last_peered":"2024-08-17T17:41:08.689619+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:41:08.689619+0000","last_undegraded":"2024-08-17T17:41:08.689619+0000","last_fullsized":"2024-08-17T17:41:08.689619+0000","mapping_epoch":337,"log_start":"325'3453","ondisk_log_start":"325'3453","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286523392,"num_objects":4372,"num_object_clones":0,"num_object_copies":13116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4372,"num_objects_unfound":0,"num_objects_dirty":4372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10362,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4372}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":85,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482765+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:40:52.482765+0000","last_peered":"2024-08-17T17:40:52.482765+0000","last_clean":"2024-08-17T17:40:52.482765+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:40:52.482765+0000","last_undegraded":"2024-08-17T17:40:52.482765+0000","last_fullsized":"2024-08-17T17:40:52.482765+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":363,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482868+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:40:52.482868+0000","last_peered":"2024-08-17T17:40:52.482868+0000","last_clean":"2024-08-17T17:40:52.482868+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:40:52.482868+0000","last_undegraded":"2024-08-17T17:40:52.482868+0000","last_fullsized":"2024-08-17T17:40:52.482868+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"382'10545","reported_seq":10689,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:09.477182+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:41:09.477182+0000","last_peered":"2024-08-17T17:41:09.477182+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:41:09.477182+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'3513","ondisk_log_start":"325'3513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289210368,"num_objects":4413,"num_object_clones":0,"num_object_copies":13239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10545,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2356},{"shards":"0,3,7","objects":4413}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":519,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498282+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:41:05.498282+0000","last_peered":"2024-08-17T17:41:05.498282+0000","last_clean":"2024-08-17T17:41:05.498282+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:41:05.498282+0000","last_undegraded":"2024-08-17T17:41:05.498282+0000","last_fullsized":"2024-08-17T17:41:05.498282+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":285,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.506014+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:41:05.506014+0000","last_peered":"2024-08-17T17:41:05.506014+0000","last_clean":"2024-08-17T17:41:05.506014+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:41:05.506014+0000","last_undegraded":"2024-08-17T17:41:05.506014+0000","last_fullsized":"2024-08-17T17:41:05.506014+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"382'10245","reported_seq":10316,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:06.789465+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:41:06.789465+0000","last_peered":"2024-08-17T17:41:06.789465+0000","last_clean":"2024-08-17T17:41:06.789465+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:41:06.789465+0000","last_undegraded":"2024-08-17T17:41:06.789465+0000","last_fullsized":"2024-08-17T17:41:06.789465+0000","mapping_epoch":319,"log_start":"326'3674","ondisk_log_start":"326'3674","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285016064,"num_objects":4349,"num_object_clones":0,"num_object_copies":13047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10245,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":608,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.500482+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:41:05.500482+0000","last_peered":"2024-08-17T17:41:05.500482+0000","last_clean":"2024-08-17T17:41:05.500482+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:41:05.500482+0000","last_undegraded":"2024-08-17T17:41:05.500482+0000","last_fullsized":"2024-08-17T17:41:05.500482+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":336,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473098+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:40:52.473098+0000","last_peered":"2024-08-17T17:40:52.473098+0000","last_clean":"2024-08-17T17:40:52.473098+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:40:52.473098+0000","last_undegraded":"2024-08-17T17:40:52.473098+0000","last_fullsized":"2024-08-17T17:40:52.473098+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"382'10441","reported_seq":10522,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:06.824249+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:41:06.824249+0000","last_peered":"2024-08-17T17:41:06.824249+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:41:06.824249+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'3835","ondisk_log_start":"326'3835","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6606,"log_dups_size":0,"ondisk_log_size":6606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10441,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4491},{"shards":"1,6","objects":-2875}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":337,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482995+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:40:52.482995+0000","last_peered":"2024-08-17T17:40:52.482995+0000","last_clean":"2024-08-17T17:40:52.482995+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:40:52.482995+0000","last_undegraded":"2024-08-17T17:40:52.482995+0000","last_fullsized":"2024-08-17T17:40:52.482995+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":363,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473760+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:40:52.473760+0000","last_peered":"2024-08-17T17:40:52.473760+0000","last_clean":"2024-08-17T17:40:52.473760+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:40:52.473760+0000","last_undegraded":"2024-08-17T17:40:52.473760+0000","last_fullsized":"2024-08-17T17:40:52.473760+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"382'10564","reported_seq":10709,"reported_epoch":382,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:08.673933+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:41:08.673933+0000","last_peered":"2024-08-17T17:41:08.673933+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:41:08.673933+0000","last_undegraded":"2024-08-17T17:41:08.673933+0000","last_fullsized":"2024-08-17T17:41:08.673933+0000","mapping_epoch":337,"log_start":"327'4961","ondisk_log_start":"327'4961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5603,"log_dups_size":0,"ondisk_log_size":5603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291373056,"num_objects":4446,"num_object_clones":0,"num_object_copies":13338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4446,"num_objects_unfound":0,"num_objects_dirty":4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10564,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4446}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":374,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473188+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:40:52.473188+0000","last_peered":"2024-08-17T17:40:52.473188+0000","last_clean":"2024-08-17T17:40:52.473188+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:40:52.473188+0000","last_undegraded":"2024-08-17T17:40:52.473188+0000","last_fullsized":"2024-08-17T17:40:52.473188+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":336,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.474007+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:40:52.474007+0000","last_peered":"2024-08-17T17:40:52.474007+0000","last_clean":"2024-08-17T17:40:52.474007+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:40:52.474007+0000","last_undegraded":"2024-08-17T17:40:52.474007+0000","last_fullsized":"2024-08-17T17:40:52.474007+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"382'10380","reported_seq":15125,"reported_epoch":382,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:09.069935+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:41:09.069935+0000","last_peered":"2024-08-17T17:41:09.069935+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:41:09.069935+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:41:09.069935+0000","mapping_epoch":337,"log_start":"325'3541","ondisk_log_start":"325'3541","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283377664,"num_objects":4324,"num_object_clones":0,"num_object_copies":12972,"num_objects_missing_on_primary":5016,"num_objects_missing":5016,"num_objects_degraded":4324,"num_objects_misplaced":5016,"num_objects_unfound":0,"num_objects_dirty":4324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10380,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2322,"num_bytes_recovered":152109056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":14},{"shards":"1,2,7","objects":4310}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":578,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498112+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:41:05.498112+0000","last_peered":"2024-08-17T17:41:05.498112+0000","last_clean":"2024-08-17T17:41:05.498112+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:41:05.498112+0000","last_undegraded":"2024-08-17T17:41:05.498112+0000","last_fullsized":"2024-08-17T17:41:05.498112+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":515,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498729+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:41:05.498729+0000","last_peered":"2024-08-17T17:41:05.498729+0000","last_clean":"2024-08-17T17:41:05.498729+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:41:05.498729+0000","last_undegraded":"2024-08-17T17:41:05.498729+0000","last_fullsized":"2024-08-17T17:41:05.498729+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"382'10518","reported_seq":15292,"reported_epoch":382,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:08.731907+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:41:08.731907+0000","last_peered":"2024-08-17T17:41:08.731907+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:41:08.731907+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:41:08.731907+0000","mapping_epoch":337,"log_start":"327'4944","ondisk_log_start":"327'4944","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5574,"log_dups_size":0,"ondisk_log_size":5574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286851096,"num_objects":4378,"num_object_clones":0,"num_object_copies":13134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4378,"num_objects_misplaced":5128,"num_objects_unfound":0,"num_objects_dirty":4378,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10518,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2320,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4378}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":708,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:05.503424+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:41:05.503424+0000","last_peered":"2024-08-17T17:41:05.503424+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:41:05.503424+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":311,"reported_epoch":381,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:40:52.482539+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:40:52.482539+0000","last_peered":"2024-08-17T17:40:52.482539+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:40:52.482539+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:40:52.482539+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":557,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498569+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:41:05.498569+0000","last_peered":"2024-08-17T17:41:05.498569+0000","last_clean":"2024-08-17T17:41:05.498569+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:41:05.498569+0000","last_undegraded":"2024-08-17T17:41:05.498569+0000","last_fullsized":"2024-08-17T17:41:05.498569+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":346,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483556+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:40:52.483556+0000","last_peered":"2024-08-17T17:40:52.483556+0000","last_clean":"2024-08-17T17:40:52.483556+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:40:52.483556+0000","last_undegraded":"2024-08-17T17:40:52.483556+0000","last_fullsized":"2024-08-17T17:40:52.483556+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"382'10389","reported_seq":10536,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:08.714545+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:41:08.714545+0000","last_peered":"2024-08-17T17:41:08.714545+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:41:08.714545+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'3321","ondisk_log_start":"325'3321","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285409280,"num_objects":4355,"num_object_clones":0,"num_object_copies":13065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10389,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2292},{"shards":"0,4,7","objects":4355}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":373,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.490856+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:41:05.490856+0000","last_peered":"2024-08-17T17:41:05.490856+0000","last_clean":"2024-08-17T17:41:05.490856+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:41:05.490856+0000","last_undegraded":"2024-08-17T17:41:05.490856+0000","last_fullsized":"2024-08-17T17:41:05.490856+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":318,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.472873+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:40:52.472873+0000","last_peered":"2024-08-17T17:40:52.472873+0000","last_clean":"2024-08-17T17:40:52.472873+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:40:52.472873+0000","last_undegraded":"2024-08-17T17:40:52.472873+0000","last_fullsized":"2024-08-17T17:40:52.472873+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":299,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.506188+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:41:05.506188+0000","last_peered":"2024-08-17T17:41:05.506188+0000","last_clean":"2024-08-17T17:41:05.506188+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:41:05.506188+0000","last_undegraded":"2024-08-17T17:41:05.506188+0000","last_fullsized":"2024-08-17T17:41:05.506188+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"382'10382","reported_seq":10463,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:06.799157+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:41:06.799157+0000","last_peered":"2024-08-17T17:41:06.799157+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:41:06.799157+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'3817","ondisk_log_start":"326'3817","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289275904,"num_objects":4414,"num_object_clones":0,"num_object_copies":13242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10382,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4414},{"shards":"5,6","objects":-2882}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":465,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498112+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:41:05.498112+0000","last_peered":"2024-08-17T17:41:05.498112+0000","last_clean":"2024-08-17T17:41:05.498112+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:41:05.498112+0000","last_undegraded":"2024-08-17T17:41:05.498112+0000","last_fullsized":"2024-08-17T17:41:05.498112+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":118,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482992+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:40:52.482992+0000","last_peered":"2024-08-17T17:40:52.482992+0000","last_clean":"2024-08-17T17:40:52.482992+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:40:52.482992+0000","last_undegraded":"2024-08-17T17:40:52.482992+0000","last_fullsized":"2024-08-17T17:40:52.482992+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":650,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491753+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:41:05.491753+0000","last_peered":"2024-08-17T17:41:05.491753+0000","last_clean":"2024-08-17T17:41:05.491753+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:41:05.491753+0000","last_undegraded":"2024-08-17T17:41:05.491753+0000","last_fullsized":"2024-08-17T17:41:05.491753+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"382'10310","reported_seq":11181,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:09.753967+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:41:09.753967+0000","last_peered":"2024-08-17T17:41:09.753967+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:41:09.753967+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'3316","ondisk_log_start":"325'3316","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284164096,"num_objects":4336,"num_object_clones":0,"num_object_copies":13008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10310,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4336},{"shards":"0,5","objects":-2318}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":441,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.490997+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:41:05.490997+0000","last_peered":"2024-08-17T17:41:05.490997+0000","last_clean":"2024-08-17T17:41:05.490997+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:41:05.490997+0000","last_undegraded":"2024-08-17T17:41:05.490997+0000","last_fullsized":"2024-08-17T17:41:05.490997+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":357,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.474115+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:40:52.474115+0000","last_peered":"2024-08-17T17:40:52.474115+0000","last_clean":"2024-08-17T17:40:52.474115+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:40:52.474115+0000","last_undegraded":"2024-08-17T17:40:52.474115+0000","last_fullsized":"2024-08-17T17:40:52.474115+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":236,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498091+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:41:05.498091+0000","last_peered":"2024-08-17T17:41:05.498091+0000","last_clean":"2024-08-17T17:41:05.498091+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:41:05.498091+0000","last_undegraded":"2024-08-17T17:41:05.498091+0000","last_fullsized":"2024-08-17T17:41:05.498091+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"381'10479","reported_seq":25469,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498414+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:41:05.498414+0000","last_peered":"2024-08-17T17:41:05.498414+0000","last_clean":"2024-08-17T17:41:05.498414+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:41:05.498414+0000","last_undegraded":"2024-08-17T17:41:05.498414+0000","last_fullsized":"2024-08-17T17:41:05.498414+0000","mapping_epoch":355,"log_start":"326'4130","ondisk_log_start":"326'4130","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6349,"log_dups_size":0,"ondisk_log_size":6349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285540352,"num_objects":4357,"num_object_clones":0,"num_object_copies":13071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10479,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":296,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482725+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:40:52.482725+0000","last_peered":"2024-08-17T17:40:52.482725+0000","last_clean":"2024-08-17T17:40:52.482725+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:40:52.482725+0000","last_undegraded":"2024-08-17T17:40:52.482725+0000","last_fullsized":"2024-08-17T17:40:52.482725+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":105,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498039+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:41:05.498039+0000","last_peered":"2024-08-17T17:41:05.498039+0000","last_clean":"2024-08-17T17:41:05.498039+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:41:05.498039+0000","last_undegraded":"2024-08-17T17:41:05.498039+0000","last_fullsized":"2024-08-17T17:41:05.498039+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":600,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491194+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:41:05.491194+0000","last_peered":"2024-08-17T17:41:05.491194+0000","last_clean":"2024-08-17T17:41:05.491194+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:41:05.491194+0000","last_undegraded":"2024-08-17T17:41:05.491194+0000","last_fullsized":"2024-08-17T17:41:05.491194+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"382'10123","reported_seq":24681,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.501412+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:41:05.501412+0000","last_peered":"2024-08-17T17:41:05.501412+0000","last_clean":"2024-08-17T17:41:05.501412+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:41:05.501412+0000","last_undegraded":"2024-08-17T17:41:05.501412+0000","last_fullsized":"2024-08-17T17:41:05.501412+0000","mapping_epoch":345,"log_start":"326'3872","ondisk_log_start":"326'3872","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280559616,"num_objects":4281,"num_object_clones":0,"num_object_copies":12843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10123,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":292,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482543+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:40:52.482543+0000","last_peered":"2024-08-17T17:40:52.482543+0000","last_clean":"2024-08-17T17:40:52.482543+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:40:52.482543+0000","last_undegraded":"2024-08-17T17:40:52.482543+0000","last_fullsized":"2024-08-17T17:40:52.482543+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":356,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.503942+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:41:05.503942+0000","last_peered":"2024-08-17T17:41:05.503942+0000","last_clean":"2024-08-17T17:41:05.503942+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:41:05.503942+0000","last_undegraded":"2024-08-17T17:41:05.503942+0000","last_fullsized":"2024-08-17T17:41:05.503942+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":434,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482430+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:40:52.482430+0000","last_peered":"2024-08-17T17:40:52.482430+0000","last_clean":"2024-08-17T17:40:52.482430+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:40:52.482430+0000","last_undegraded":"2024-08-17T17:40:52.482430+0000","last_fullsized":"2024-08-17T17:40:52.482430+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"381'10373","reported_seq":10539,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:05.497818+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:41:05.497818+0000","last_peered":"2024-08-17T17:41:05.497818+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:41:05.497818+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4087","ondisk_log_start":"326'4087","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6286,"log_dups_size":0,"ondisk_log_size":6286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289079296,"num_objects":4411,"num_object_clones":0,"num_object_copies":13233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10373,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4411},{"shards":"0,3","objects":-2837}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":631,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498803+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:41:05.498803+0000","last_peered":"2024-08-17T17:41:05.498803+0000","last_clean":"2024-08-17T17:41:05.498803+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:41:05.498803+0000","last_undegraded":"2024-08-17T17:41:05.498803+0000","last_fullsized":"2024-08-17T17:41:05.498803+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":599,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491775+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:41:05.491775+0000","last_peered":"2024-08-17T17:41:05.491775+0000","last_clean":"2024-08-17T17:41:05.491775+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:41:05.491775+0000","last_undegraded":"2024-08-17T17:41:05.491775+0000","last_fullsized":"2024-08-17T17:41:05.491775+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":617,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.497888+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:41:05.497888+0000","last_peered":"2024-08-17T17:41:05.497888+0000","last_clean":"2024-08-17T17:41:05.497888+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:41:05.497888+0000","last_undegraded":"2024-08-17T17:41:05.497888+0000","last_fullsized":"2024-08-17T17:41:05.497888+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"382'10231","reported_seq":24910,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:06.778849+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:41:06.778849+0000","last_peered":"2024-08-17T17:41:06.778849+0000","last_clean":"2024-08-17T17:41:06.778849+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:41:06.778849+0000","last_undegraded":"2024-08-17T17:41:06.778849+0000","last_fullsized":"2024-08-17T17:41:06.778849+0000","mapping_epoch":347,"log_start":"326'3672","ondisk_log_start":"326'3672","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284622848,"num_objects":4343,"num_object_clones":0,"num_object_copies":13029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10231,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":289,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.504447+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:41:05.504447+0000","last_peered":"2024-08-17T17:41:05.504447+0000","last_clean":"2024-08-17T17:41:05.504447+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:41:05.504447+0000","last_undegraded":"2024-08-17T17:41:05.504447+0000","last_fullsized":"2024-08-17T17:41:05.504447+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":386,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483622+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:40:52.483622+0000","last_peered":"2024-08-17T17:40:52.483622+0000","last_clean":"2024-08-17T17:40:52.483622+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:40:52.483622+0000","last_undegraded":"2024-08-17T17:40:52.483622+0000","last_fullsized":"2024-08-17T17:40:52.483622+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":336,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473124+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:40:52.473124+0000","last_peered":"2024-08-17T17:40:52.473124+0000","last_clean":"2024-08-17T17:40:52.473124+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:40:52.473124+0000","last_undegraded":"2024-08-17T17:40:52.473124+0000","last_fullsized":"2024-08-17T17:40:52.473124+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":336,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473984+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:40:52.473984+0000","last_peered":"2024-08-17T17:40:52.473984+0000","last_clean":"2024-08-17T17:40:52.473984+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:40:52.473984+0000","last_undegraded":"2024-08-17T17:40:52.473984+0000","last_fullsized":"2024-08-17T17:40:52.473984+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":379,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473263+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:40:52.473263+0000","last_peered":"2024-08-17T17:40:52.473263+0000","last_clean":"2024-08-17T17:40:52.473263+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:40:52.473263+0000","last_undegraded":"2024-08-17T17:40:52.473263+0000","last_fullsized":"2024-08-17T17:40:52.473263+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":606,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491114+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:41:05.491114+0000","last_peered":"2024-08-17T17:41:05.491114+0000","last_clean":"2024-08-17T17:41:05.491114+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:41:05.491114+0000","last_undegraded":"2024-08-17T17:41:05.491114+0000","last_fullsized":"2024-08-17T17:41:05.491114+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":146,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498918+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:41:05.498918+0000","last_peered":"2024-08-17T17:41:05.498918+0000","last_clean":"2024-08-17T17:41:05.498918+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:41:05.498918+0000","last_undegraded":"2024-08-17T17:41:05.498918+0000","last_fullsized":"2024-08-17T17:41:05.498918+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":286,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482926+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:40:52.482926+0000","last_peered":"2024-08-17T17:40:52.482926+0000","last_clean":"2024-08-17T17:40:52.482926+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:40:52.482926+0000","last_undegraded":"2024-08-17T17:40:52.482926+0000","last_fullsized":"2024-08-17T17:40:52.482926+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":564,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.497848+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:41:05.497848+0000","last_peered":"2024-08-17T17:41:05.497848+0000","last_clean":"2024-08-17T17:41:05.497848+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:41:05.497848+0000","last_undegraded":"2024-08-17T17:41:05.497848+0000","last_fullsized":"2024-08-17T17:41:05.497848+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":430,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483041+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:40:52.483041+0000","last_peered":"2024-08-17T17:40:52.483041+0000","last_clean":"2024-08-17T17:40:52.483041+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:40:52.483041+0000","last_undegraded":"2024-08-17T17:40:52.483041+0000","last_fullsized":"2024-08-17T17:40:52.483041+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":558,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.497771+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:41:05.497771+0000","last_peered":"2024-08-17T17:41:05.497771+0000","last_clean":"2024-08-17T17:41:05.497771+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:41:05.497771+0000","last_undegraded":"2024-08-17T17:41:05.497771+0000","last_fullsized":"2024-08-17T17:41:05.497771+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":353,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483561+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:40:52.483561+0000","last_peered":"2024-08-17T17:40:52.483561+0000","last_clean":"2024-08-17T17:40:52.483561+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:40:52.483561+0000","last_undegraded":"2024-08-17T17:40:52.483561+0000","last_fullsized":"2024-08-17T17:40:52.483561+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":589,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.497981+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:41:05.497981+0000","last_peered":"2024-08-17T17:41:05.497981+0000","last_clean":"2024-08-17T17:41:05.497981+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:41:05.497981+0000","last_undegraded":"2024-08-17T17:41:05.497981+0000","last_fullsized":"2024-08-17T17:41:05.497981+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":305,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498432+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:41:05.498432+0000","last_peered":"2024-08-17T17:41:05.498432+0000","last_clean":"2024-08-17T17:41:05.498432+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:41:05.498432+0000","last_undegraded":"2024-08-17T17:41:05.498432+0000","last_fullsized":"2024-08-17T17:41:05.498432+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":597,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491102+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:41:05.491102+0000","last_peered":"2024-08-17T17:41:05.491102+0000","last_clean":"2024-08-17T17:41:05.491102+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:41:05.491102+0000","last_undegraded":"2024-08-17T17:41:05.491102+0000","last_fullsized":"2024-08-17T17:41:05.491102+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":557,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498322+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:41:05.498322+0000","last_peered":"2024-08-17T17:41:05.498322+0000","last_clean":"2024-08-17T17:41:05.498322+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:41:05.498322+0000","last_undegraded":"2024-08-17T17:41:05.498322+0000","last_fullsized":"2024-08-17T17:41:05.498322+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":365,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473491+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:40:52.473491+0000","last_peered":"2024-08-17T17:40:52.473491+0000","last_clean":"2024-08-17T17:40:52.473491+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:40:52.473491+0000","last_undegraded":"2024-08-17T17:40:52.473491+0000","last_fullsized":"2024-08-17T17:40:52.473491+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":246,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498203+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:41:05.498203+0000","last_peered":"2024-08-17T17:41:05.498203+0000","last_clean":"2024-08-17T17:41:05.498203+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:41:05.498203+0000","last_undegraded":"2024-08-17T17:41:05.498203+0000","last_fullsized":"2024-08-17T17:41:05.498203+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":302,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.504044+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:41:05.504044+0000","last_peered":"2024-08-17T17:41:05.504044+0000","last_clean":"2024-08-17T17:41:05.504044+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:41:05.504044+0000","last_undegraded":"2024-08-17T17:41:05.504044+0000","last_fullsized":"2024-08-17T17:41:05.504044+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":373,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473914+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:40:52.473914+0000","last_peered":"2024-08-17T17:40:52.473914+0000","last_clean":"2024-08-17T17:40:52.473914+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:40:52.473914+0000","last_undegraded":"2024-08-17T17:40:52.473914+0000","last_fullsized":"2024-08-17T17:40:52.473914+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":236,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473177+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:40:52.473177+0000","last_peered":"2024-08-17T17:40:52.473177+0000","last_clean":"2024-08-17T17:40:52.473177+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:40:52.473177+0000","last_undegraded":"2024-08-17T17:40:52.473177+0000","last_fullsized":"2024-08-17T17:40:52.473177+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":373,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473405+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:40:52.473405+0000","last_peered":"2024-08-17T17:40:52.473405+0000","last_clean":"2024-08-17T17:40:52.473405+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:40:52.473405+0000","last_undegraded":"2024-08-17T17:40:52.473405+0000","last_fullsized":"2024-08-17T17:40:52.473405+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":236,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.473638+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:40:52.473638+0000","last_peered":"2024-08-17T17:40:52.473638+0000","last_clean":"2024-08-17T17:40:52.473638+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:40:52.473638+0000","last_undegraded":"2024-08-17T17:40:52.473638+0000","last_fullsized":"2024-08-17T17:40:52.473638+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":609,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498146+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:41:05.498146+0000","last_peered":"2024-08-17T17:41:05.498146+0000","last_clean":"2024-08-17T17:41:05.498146+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:41:05.498146+0000","last_undegraded":"2024-08-17T17:41:05.498146+0000","last_fullsized":"2024-08-17T17:41:05.498146+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":583,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.498574+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:41:05.498574+0000","last_peered":"2024-08-17T17:41:05.498574+0000","last_clean":"2024-08-17T17:41:05.498574+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:41:05.498574+0000","last_undegraded":"2024-08-17T17:41:05.498574+0000","last_fullsized":"2024-08-17T17:41:05.498574+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":336,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.474974+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:40:52.474974+0000","last_peered":"2024-08-17T17:40:52.474974+0000","last_clean":"2024-08-17T17:40:52.474974+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:40:52.474974+0000","last_undegraded":"2024-08-17T17:40:52.474974+0000","last_fullsized":"2024-08-17T17:40:52.474974+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":304,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.504148+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:41:05.504148+0000","last_peered":"2024-08-17T17:41:05.504148+0000","last_clean":"2024-08-17T17:41:05.504148+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:41:05.504148+0000","last_undegraded":"2024-08-17T17:41:05.504148+0000","last_fullsized":"2024-08-17T17:41:05.504148+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4595712024,"num_objects":70126,"num_object_clones":0,"num_object_copies":210378,"num_objects_missing_on_primary":5016,"num_objects_missing":5016,"num_objects_degraded":53121,"num_objects_misplaced":27818,"num_objects_unfound":0,"num_objects_dirty":70126,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":166264,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29197,"num_bytes_recovered":1790902272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15421616128,"data_stored":15421603912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104683,"ondisk_log_size":104683,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170096,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463466,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2861816,"kb_used_data":1972272,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90883336,"statfs":{"total":95995035648,"available":93064536064,"internally_reserved":0,"allocated":2019606528,"data_stored":2017529363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492141,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":963612,"kb_used_data":564116,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92781540,"statfs":{"total":95995035648,"available":95008296960,"internally_reserved":0,"allocated":577654784,"data_stored":575599919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986732,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2292524,"kb_used_data":1115684,"kb_used_omap":1,"kb_used_meta":1176830,"kb_avail":91452628,"statfs":{"total":95995035648,"available":93647491072,"internally_reserved":0,"allocated":1142460416,"data_stored":1140394191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1205074264},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,3,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225559,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3187404,"kb_used_data":2253636,"kb_used_omap":4,"kb_used_meta":933755,"kb_avail":90557748,"statfs":{"total":95995035648,"available":92731133952,"internally_reserved":0,"allocated":2307723264,"data_stored":2305676490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":956165568},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075885,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533778,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3008800,"kb_used_data":1966808,"kb_used_omap":5,"kb_used_meta":1041978,"kb_avail":90736352,"statfs":{"total":95995035648,"available":92914024448,"internally_reserved":0,"allocated":2014011392,"data_stored":2011951503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1066985712},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089755,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2687736,"kb_used_data":1686832,"kb_used_omap":5,"kb_used_meta":1000890,"kb_avail":91057416,"statfs":{"total":95995035648,"available":93242793984,"internally_reserved":0,"allocated":1727315968,"data_stored":1725238862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1024911419},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1720778752,"data_stored":1720778752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2007502848,"data_stored":2007498776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2301165568,"data_stored":2301165568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1136590848,"data_stored":1136590848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":572456960,"data_stored":572456960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2013593600,"data_stored":2013593600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2330267648,"data_stored":2330263576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:11.880 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:11.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:11.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:12.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:12.063 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:12.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:41:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:41:11] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:41:12.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:12.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:12.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:12.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:12.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:12.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:12.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:12.525 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:12 smithi152 ceph-mon[93456]: from='client.37590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:12.525 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:12.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:12 smithi142 ceph-mon[103614]: from='client.37590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:12 smithi142 ceph-mon[106729]: from='client.37590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:12.623 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:12.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:12.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:12.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:12.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:12.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:12.900 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:12.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:12.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:13.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.220 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:13.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:13.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:13.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:13.400 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:41:13.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:13.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:13 smithi152 ceph-mon[93456]: pgmap v1441: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53124/209784 objects degraded (25.323%); 27797/209784 objects misplaced (13.250%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:13.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2286051231' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:13.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:13 smithi142 ceph-mon[106729]: pgmap v1441: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53124/209784 objects degraded (25.323%); 27797/209784 objects misplaced (13.250%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2286051231' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:13 smithi142 ceph-mon[103614]: pgmap v1441: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 53124/209784 objects degraded (25.323%); 27797/209784 objects misplaced (13.250%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2286051231' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:13.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:13.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:13.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.913 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:13.913 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:13.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:13.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:14.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:14.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:14.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:14.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:14.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:14.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:14.358 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:14.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:14.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:14.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:14.548 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:14 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:14.548 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2286051231' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:14.548 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:14 smithi142 ceph-mon[106729]: osdmap e383: 8 total, 8 up, 6 in 2024-08-17T17:41:14.548 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:14 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:14.549 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2286051231' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:14.549 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:14 smithi142 ceph-mon[103614]: osdmap e383: 8 total, 8 up, 6 in 2024-08-17T17:41:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:14 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2286051231' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:14.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:14 smithi152 ceph-mon[93456]: osdmap e383: 8 total, 8 up, 6 in 2024-08-17T17:41:14.583 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:14.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:14.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:41:14.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:14.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:14.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:14.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:14.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:14.919 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:14.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:14.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:14.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:15.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:15.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:15.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:15.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:15.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:15.332 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:15.358 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:15.459 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:15.460 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:15 smithi142 ceph-mon[106729]: pgmap v1443: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 53063/209379 objects degraded (25.343%); 27662/209379 objects misplaced (13.211%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:15.461 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:15 smithi142 ceph-mon[103614]: pgmap v1443: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 53063/209379 objects degraded (25.343%); 27662/209379 objects misplaced (13.211%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.482 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:15.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:15.568 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:41:15.568 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:15.579 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:41:15.580 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:15.583 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:15.597 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:41:15.597 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:15.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:15.609 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:15.619 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:41:15.619 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:15.636 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:41:15.636 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:15.652 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:41:15.652 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:15.669 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:41:15.669 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:15.687 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:41:15.687 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:15.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:15.730 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:15 smithi152 ceph-mon[93456]: pgmap v1443: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 53063/209379 objects degraded (25.343%); 27662/209379 objects misplaced (13.211%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:15.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:15.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:15.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:15.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:15.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:15.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:16.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:16.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:16.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:16.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:16.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:16.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:16.350 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:16.452 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:16.474 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:16.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:16.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/439918574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:16.576 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 53063/209379 objects degraded (25.343%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:16.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/439918574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:16.577 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 53063/209379 objects degraded (25.343%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:16.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:16.628 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/439918574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:16.628 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 53063/209379 objects degraded (25.343%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:16.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:16.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:16.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:16.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:16.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:16.872 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:41:16.872 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:41:16.872 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:41:16.872 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:41:16.872 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:41:16.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:16.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:16.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:17.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:41:17.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:17.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:17.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:17.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:17.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:17.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:17.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:17.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:17.497 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:17 smithi142 ceph-mon[106729]: pgmap v1444: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 53063/208875 objects degraded (25.404%); 27662/208875 objects misplaced (13.243%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:41:17.497 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:17 smithi142 ceph-mon[103614]: pgmap v1444: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 53063/208875 objects degraded (25.404%); 27662/208875 objects misplaced (13.243%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:41:17.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:17.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:17 smithi152 ceph-mon[93456]: pgmap v1444: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 53063/208875 objects degraded (25.404%); 27662/208875 objects misplaced (13.243%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:41:17.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:41:17.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:17.675 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:17.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:17.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:17.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:17.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:17.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:17.935 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:18.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:18.055 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:18.108 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:18.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:18.179 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:18.192 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:18.194 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:41:18.281 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:18.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:18.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:18.427 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:18.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:18.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:18.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:18.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:18.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:18.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:18.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:18.923 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:18.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:19.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:19.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:19.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:19.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:19.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:19.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:19.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:19.426 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:19.433 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1444,"stamp":"2024-08-17T17:41:16.339920+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4561770469,"num_objects":69625,"num_object_clones":0,"num_object_copies":208875,"num_objects_missing_on_primary":4976,"num_objects_missing":4976,"num_objects_degraded":53063,"num_objects_misplaced":27662,"num_objects_unfound":0,"num_objects_dirty":69625,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":167069,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29320,"num_bytes_recovered":1798644366,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105017,"ondisk_log_size":105017,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14939732,"kb_used_data":9488852,"kb_used_omap":27,"kb_used_meta":5450788,"kb_avail":547531180,"statfs":{"total":575970213888,"available":560671928320,"internally_reserved":0,"allocated":9716584448,"data_stored":9704155666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5581607656},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-62062592,"num_objects":-947,"num_object_clones":0,"num_object_copies":-2841,"num_objects_missing_on_primary":-78,"num_objects_missing":-78,"num_objects_degraded":-132,"num_objects_misplaced":-359,"num_objects_unfound":0,"num_objects_dirty":-947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":947,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":11862016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002506"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":609,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403358+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:41:13.403358+0000","last_peered":"2024-08-17T17:41:13.403358+0000","last_clean":"2024-08-17T17:41:13.403358+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:41:13.403358+0000","last_undegraded":"2024-08-17T17:41:13.403358+0000","last_fullsized":"2024-08-17T17:41:13.403358+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":92,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492438+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:41:05.492438+0000","last_peered":"2024-08-17T17:41:05.492438+0000","last_clean":"2024-08-17T17:41:05.492438+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:41:05.492438+0000","last_undegraded":"2024-08-17T17:41:05.492438+0000","last_fullsized":"2024-08-17T17:41:05.492438+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":286,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.522342+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:41:05.522342+0000","last_peered":"2024-08-17T17:41:05.522342+0000","last_clean":"2024-08-17T17:41:05.522342+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:41:05.522342+0000","last_undegraded":"2024-08-17T17:41:05.522342+0000","last_fullsized":"2024-08-17T17:41:05.522342+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.404713+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:41:13.404713+0000","last_peered":"2024-08-17T17:41:13.404713+0000","last_clean":"2024-08-17T17:41:13.404713+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:41:13.404713+0000","last_undegraded":"2024-08-17T17:41:13.404713+0000","last_fullsized":"2024-08-17T17:41:13.404713+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":484,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403864+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:41:13.403864+0000","last_peered":"2024-08-17T17:41:13.403864+0000","last_clean":"2024-08-17T17:41:13.403864+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:41:13.403864+0000","last_undegraded":"2024-08-17T17:41:13.403864+0000","last_fullsized":"2024-08-17T17:41:13.403864+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":235,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491784+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:41:05.491784+0000","last_peered":"2024-08-17T17:41:05.491784+0000","last_clean":"2024-08-17T17:41:05.491784+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:41:05.491784+0000","last_undegraded":"2024-08-17T17:41:05.491784+0000","last_fullsized":"2024-08-17T17:41:05.491784+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":577,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403014+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:41:13.403014+0000","last_peered":"2024-08-17T17:41:13.403014+0000","last_clean":"2024-08-17T17:41:13.403014+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:41:13.403014+0000","last_undegraded":"2024-08-17T17:41:13.403014+0000","last_fullsized":"2024-08-17T17:41:13.403014+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":402,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483143+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:40:52.483143+0000","last_peered":"2024-08-17T17:40:52.483143+0000","last_clean":"2024-08-17T17:40:52.483143+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:40:52.483143+0000","last_undegraded":"2024-08-17T17:40:52.483143+0000","last_fullsized":"2024-08-17T17:40:52.483143+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"383'10377","reported_seq":15064,"reported_epoch":383,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:41:14.027693+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:41:14.027693+0000","last_peered":"2024-08-17T17:41:14.027693+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:41:14.027693+0000","last_undegraded":"2024-08-17T17:41:14.027693+0000","last_fullsized":"2024-08-17T17:41:14.027693+0000","mapping_epoch":337,"log_start":"325'3463","ondisk_log_start":"325'3463","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283574272,"num_objects":4327,"num_object_clones":0,"num_object_copies":12981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4327,"num_objects_unfound":0,"num_objects_dirty":4327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10377,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2296,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4327}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":88,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.521053+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:41:05.521053+0000","last_peered":"2024-08-17T17:41:05.521053+0000","last_clean":"2024-08-17T17:41:05.521053+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:41:05.521053+0000","last_undegraded":"2024-08-17T17:41:05.521053+0000","last_fullsized":"2024-08-17T17:41:05.521053+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.521817+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:41:05.521817+0000","last_peered":"2024-08-17T17:41:05.521817+0000","last_clean":"2024-08-17T17:41:05.521817+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:41:05.521817+0000","last_undegraded":"2024-08-17T17:41:05.521817+0000","last_fullsized":"2024-08-17T17:41:05.521817+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"382'10586","reported_seq":10666,"reported_epoch":382,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:11.385691+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:41:11.385691+0000","last_peered":"2024-08-17T17:41:11.385691+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:41:11.385691+0000","last_undegraded":"2024-08-17T17:41:11.385691+0000","last_fullsized":"2024-08-17T17:41:11.385691+0000","mapping_epoch":337,"log_start":"326'3982","ondisk_log_start":"326'3982","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4488,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10586,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4488}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":90,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492971+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:41:05.492971+0000","last_peered":"2024-08-17T17:41:05.492971+0000","last_clean":"2024-08-17T17:41:05.492971+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:41:05.492971+0000","last_undegraded":"2024-08-17T17:41:05.492971+0000","last_fullsized":"2024-08-17T17:41:05.492971+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":354,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491689+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:41:05.491689+0000","last_peered":"2024-08-17T17:41:05.491689+0000","last_clean":"2024-08-17T17:41:05.491689+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:41:05.491689+0000","last_undegraded":"2024-08-17T17:41:05.491689+0000","last_fullsized":"2024-08-17T17:41:05.491689+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"383'10388","reported_seq":10472,"reported_epoch":383,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:13.610140+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:41:13.610140+0000","last_peered":"2024-08-17T17:41:13.610140+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:41:13.610140+0000","last_undegraded":"2024-08-17T17:41:13.610140+0000","last_fullsized":"2024-08-17T17:41:13.610140+0000","mapping_epoch":337,"log_start":"325'3553","ondisk_log_start":"325'3553","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284819456,"num_objects":4346,"num_object_clones":0,"num_object_copies":13038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4346,"num_objects_unfound":0,"num_objects_dirty":4346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10388,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4346}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":86,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.522273+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:41:05.522273+0000","last_peered":"2024-08-17T17:41:05.522273+0000","last_clean":"2024-08-17T17:41:05.522273+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:41:05.522273+0000","last_undegraded":"2024-08-17T17:41:05.522273+0000","last_fullsized":"2024-08-17T17:41:05.522273+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":363,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482868+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:40:52.482868+0000","last_peered":"2024-08-17T17:40:52.482868+0000","last_clean":"2024-08-17T17:40:52.482868+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:40:52.482868+0000","last_undegraded":"2024-08-17T17:40:52.482868+0000","last_fullsized":"2024-08-17T17:40:52.482868+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"383'10584","reported_seq":10730,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:14.817729+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:41:14.817729+0000","last_peered":"2024-08-17T17:41:14.817729+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:41:14.817729+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"325'3513","ondisk_log_start":"325'3513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286654464,"num_objects":4374,"num_object_clones":0,"num_object_copies":13122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10584,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2395},{"shards":"0,3,7","objects":4374}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":521,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403434+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:41:13.403434+0000","last_peered":"2024-08-17T17:41:13.403434+0000","last_clean":"2024-08-17T17:41:13.403434+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:41:13.403434+0000","last_undegraded":"2024-08-17T17:41:13.403434+0000","last_fullsized":"2024-08-17T17:41:13.403434+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":287,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.440931+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:41:13.440931+0000","last_peered":"2024-08-17T17:41:13.440931+0000","last_clean":"2024-08-17T17:41:13.440931+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:41:13.440931+0000","last_undegraded":"2024-08-17T17:41:13.440931+0000","last_fullsized":"2024-08-17T17:41:13.440931+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"382'10270","reported_seq":10341,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:11.780459+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:41:11.780459+0000","last_peered":"2024-08-17T17:41:11.780459+0000","last_clean":"2024-08-17T17:41:11.780459+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:41:11.780459+0000","last_undegraded":"2024-08-17T17:41:11.780459+0000","last_fullsized":"2024-08-17T17:41:11.780459+0000","mapping_epoch":319,"log_start":"326'3674","ondisk_log_start":"326'3674","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6596,"log_dups_size":0,"ondisk_log_size":6596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283377664,"num_objects":4324,"num_object_clones":0,"num_object_copies":12972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10270,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":610,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.404321+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:41:13.404321+0000","last_peered":"2024-08-17T17:41:13.404321+0000","last_clean":"2024-08-17T17:41:13.404321+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:41:13.404321+0000","last_undegraded":"2024-08-17T17:41:13.404321+0000","last_fullsized":"2024-08-17T17:41:13.404321+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492194+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:41:05.492194+0000","last_peered":"2024-08-17T17:41:05.492194+0000","last_clean":"2024-08-17T17:41:05.492194+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:41:05.492194+0000","last_undegraded":"2024-08-17T17:41:05.492194+0000","last_fullsized":"2024-08-17T17:41:05.492194+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"382'10465","reported_seq":10546,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:11.799899+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:41:11.799899+0000","last_peered":"2024-08-17T17:41:11.799899+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:41:11.799899+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'3935","ondisk_log_start":"326'3935","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292749312,"num_objects":4467,"num_object_clones":0,"num_object_copies":13401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10465,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4467},{"shards":"1,6","objects":-2899}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":337,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482995+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:40:52.482995+0000","last_peered":"2024-08-17T17:40:52.482995+0000","last_clean":"2024-08-17T17:40:52.482995+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:40:52.482995+0000","last_undegraded":"2024-08-17T17:40:52.482995+0000","last_fullsized":"2024-08-17T17:40:52.482995+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":364,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492466+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:41:05.492466+0000","last_peered":"2024-08-17T17:41:05.492466+0000","last_clean":"2024-08-17T17:41:05.492466+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:41:05.492466+0000","last_undegraded":"2024-08-17T17:41:05.492466+0000","last_fullsized":"2024-08-17T17:41:05.492466+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"383'10592","reported_seq":10739,"reported_epoch":383,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:13.402851+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:41:13.402851+0000","last_peered":"2024-08-17T17:41:13.402851+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:41:13.402851+0000","last_undegraded":"2024-08-17T17:41:13.402851+0000","last_fullsized":"2024-08-17T17:41:13.402851+0000","mapping_epoch":337,"log_start":"327'4961","ondisk_log_start":"327'4961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289538048,"num_objects":4418,"num_object_clones":0,"num_object_copies":13254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4418,"num_objects_unfound":0,"num_objects_dirty":4418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10592,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4418}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":375,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491949+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:41:05.491949+0000","last_peered":"2024-08-17T17:41:05.491949+0000","last_clean":"2024-08-17T17:41:05.491949+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:41:05.491949+0000","last_undegraded":"2024-08-17T17:41:05.491949+0000","last_fullsized":"2024-08-17T17:41:05.491949+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.493085+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:41:05.493085+0000","last_peered":"2024-08-17T17:41:05.493085+0000","last_clean":"2024-08-17T17:41:05.493085+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:41:05.493085+0000","last_undegraded":"2024-08-17T17:41:05.493085+0000","last_fullsized":"2024-08-17T17:41:05.493085+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"383'10420","reported_seq":15246,"reported_epoch":383,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:14.123859+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:41:14.123859+0000","last_peered":"2024-08-17T17:41:14.123859+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:41:14.123859+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:41:14.123859+0000","mapping_epoch":337,"log_start":"325'3541","ondisk_log_start":"325'3541","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280756224,"num_objects":4284,"num_object_clones":0,"num_object_copies":12852,"num_objects_missing_on_primary":4976,"num_objects_missing":4976,"num_objects_degraded":4284,"num_objects_misplaced":4976,"num_objects_unfound":0,"num_objects_dirty":4284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10420,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2363,"num_bytes_recovered":154796032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":4270},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":580,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.405258+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:41:13.405258+0000","last_peered":"2024-08-17T17:41:13.405258+0000","last_clean":"2024-08-17T17:41:13.405258+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:41:13.405258+0000","last_undegraded":"2024-08-17T17:41:13.405258+0000","last_fullsized":"2024-08-17T17:41:13.405258+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":517,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403102+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:41:13.403102+0000","last_peered":"2024-08-17T17:41:13.403102+0000","last_clean":"2024-08-17T17:41:13.403102+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:41:13.403102+0000","last_undegraded":"2024-08-17T17:41:13.403102+0000","last_fullsized":"2024-08-17T17:41:13.403102+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"383'10539","reported_seq":15357,"reported_epoch":383,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:13.858362+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:41:13.858362+0000","last_peered":"2024-08-17T17:41:13.858362+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:41:13.858362+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:41:13.858362+0000","mapping_epoch":337,"log_start":"327'4944","ondisk_log_start":"327'4944","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285474840,"num_objects":4357,"num_object_clones":0,"num_object_copies":13071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4357,"num_objects_misplaced":5107,"num_objects_unfound":0,"num_objects_dirty":4357,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10539,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2341,"num_bytes_recovered":153354240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4357}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":710,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:13.442503+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:41:13.442503+0000","last_peered":"2024-08-17T17:41:13.442503+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:41:13.442503+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":312,"reported_epoch":382,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:41:05.522501+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:41:05.522501+0000","last_peered":"2024-08-17T17:41:05.522501+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:41:05.522501+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:41:05.522501+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":559,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.404620+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:41:13.404620+0000","last_peered":"2024-08-17T17:41:13.404620+0000","last_clean":"2024-08-17T17:41:13.404620+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:41:13.404620+0000","last_undegraded":"2024-08-17T17:41:13.404620+0000","last_fullsized":"2024-08-17T17:41:13.404620+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":346,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483556+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:40:52.483556+0000","last_peered":"2024-08-17T17:40:52.483556+0000","last_clean":"2024-08-17T17:40:52.483556+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:40:52.483556+0000","last_undegraded":"2024-08-17T17:40:52.483556+0000","last_fullsized":"2024-08-17T17:40:52.483556+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"383'10405","reported_seq":10554,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:14.842454+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:41:14.842454+0000","last_peered":"2024-08-17T17:41:14.842454+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:41:14.842454+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'3421","ondisk_log_start":"325'3421","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284360704,"num_objects":4339,"num_object_clones":0,"num_object_copies":13017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10405,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2308},{"shards":"0,4,7","objects":4339}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":375,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.402114+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:41:13.402114+0000","last_peered":"2024-08-17T17:41:13.402114+0000","last_clean":"2024-08-17T17:41:13.402114+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:41:13.402114+0000","last_undegraded":"2024-08-17T17:41:13.402114+0000","last_fullsized":"2024-08-17T17:41:13.402114+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":319,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491033+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:41:05.491033+0000","last_peered":"2024-08-17T17:41:05.491033+0000","last_clean":"2024-08-17T17:41:05.491033+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:41:05.491033+0000","last_undegraded":"2024-08-17T17:41:05.491033+0000","last_fullsized":"2024-08-17T17:41:05.491033+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.441859+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:41:13.441859+0000","last_peered":"2024-08-17T17:41:13.441859+0000","last_clean":"2024-08-17T17:41:13.441859+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:41:13.441859+0000","last_undegraded":"2024-08-17T17:41:13.441859+0000","last_fullsized":"2024-08-17T17:41:13.441859+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"382'10407","reported_seq":10488,"reported_epoch":382,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:11.291785+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:41:11.291785+0000","last_peered":"2024-08-17T17:41:11.291785+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:41:11.291785+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'3817","ondisk_log_start":"326'3817","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6590,"log_dups_size":0,"ondisk_log_size":6590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287637504,"num_objects":4389,"num_object_clones":0,"num_object_copies":13167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10407,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4389},{"shards":"5,6","objects":-2907}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":467,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.404551+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:41:13.404551+0000","last_peered":"2024-08-17T17:41:13.404551+0000","last_clean":"2024-08-17T17:41:13.404551+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:41:13.404551+0000","last_undegraded":"2024-08-17T17:41:13.404551+0000","last_fullsized":"2024-08-17T17:41:13.404551+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":118,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.482992+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:40:52.482992+0000","last_peered":"2024-08-17T17:40:52.482992+0000","last_clean":"2024-08-17T17:40:52.482992+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:40:52.482992+0000","last_undegraded":"2024-08-17T17:40:52.482992+0000","last_fullsized":"2024-08-17T17:40:52.482992+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":652,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403815+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:41:13.403815+0000","last_peered":"2024-08-17T17:41:13.403815+0000","last_clean":"2024-08-17T17:41:13.403815+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:41:13.403815+0000","last_undegraded":"2024-08-17T17:41:13.403815+0000","last_fullsized":"2024-08-17T17:41:13.403815+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"383'10338","reported_seq":11211,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:14.868738+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:41:14.868738+0000","last_peered":"2024-08-17T17:41:14.868738+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:41:14.868738+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'3316","ondisk_log_start":"325'3316","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282329088,"num_objects":4308,"num_object_clones":0,"num_object_copies":12924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10338,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4308},{"shards":"0,5","objects":-2346}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":443,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403127+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:41:13.403127+0000","last_peered":"2024-08-17T17:41:13.403127+0000","last_clean":"2024-08-17T17:41:13.403127+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:41:13.403127+0000","last_undegraded":"2024-08-17T17:41:13.403127+0000","last_fullsized":"2024-08-17T17:41:13.403127+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":358,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492861+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:41:05.492861+0000","last_peered":"2024-08-17T17:41:05.492861+0000","last_clean":"2024-08-17T17:41:05.492861+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:41:05.492861+0000","last_undegraded":"2024-08-17T17:41:05.492861+0000","last_fullsized":"2024-08-17T17:41:05.492861+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":238,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403325+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:41:13.403325+0000","last_peered":"2024-08-17T17:41:13.403325+0000","last_clean":"2024-08-17T17:41:13.403325+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:41:13.403325+0000","last_undegraded":"2024-08-17T17:41:13.403325+0000","last_fullsized":"2024-08-17T17:41:13.403325+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"383'10546","reported_seq":25538,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.343401+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:41:15.343401+0000","last_peered":"2024-08-17T17:41:15.343401+0000","last_clean":"2024-08-17T17:41:15.343401+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:41:15.343401+0000","last_undegraded":"2024-08-17T17:41:15.343401+0000","last_fullsized":"2024-08-17T17:41:15.343401+0000","mapping_epoch":355,"log_start":"326'4230","ondisk_log_start":"326'4230","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6316,"log_dups_size":0,"ondisk_log_size":6316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281149440,"num_objects":4290,"num_object_clones":0,"num_object_copies":12870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10546,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":297,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.522189+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:41:05.522189+0000","last_peered":"2024-08-17T17:41:05.522189+0000","last_clean":"2024-08-17T17:41:05.522189+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:41:05.522189+0000","last_undegraded":"2024-08-17T17:41:05.522189+0000","last_fullsized":"2024-08-17T17:41:05.522189+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403242+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:41:13.403242+0000","last_peered":"2024-08-17T17:41:13.403242+0000","last_clean":"2024-08-17T17:41:13.403242+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:41:13.403242+0000","last_undegraded":"2024-08-17T17:41:13.403242+0000","last_fullsized":"2024-08-17T17:41:13.403242+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":603,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403426+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:41:13.403426+0000","last_peered":"2024-08-17T17:41:13.403426+0000","last_clean":"2024-08-17T17:41:13.403426+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:41:13.403426+0000","last_undegraded":"2024-08-17T17:41:13.403426+0000","last_fullsized":"2024-08-17T17:41:13.403426+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"383'10191","reported_seq":24751,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.291624+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:41:15.291624+0000","last_peered":"2024-08-17T17:41:15.291624+0000","last_clean":"2024-08-17T17:41:15.291624+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:41:15.291624+0000","last_undegraded":"2024-08-17T17:41:15.291624+0000","last_fullsized":"2024-08-17T17:41:15.291624+0000","mapping_epoch":345,"log_start":"326'3872","ondisk_log_start":"326'3872","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6319,"log_dups_size":0,"ondisk_log_size":6319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276103168,"num_objects":4213,"num_object_clones":0,"num_object_copies":12639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10191,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.523974+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:41:05.523974+0000","last_peered":"2024-08-17T17:41:05.523974+0000","last_clean":"2024-08-17T17:41:05.523974+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:41:05.523974+0000","last_undegraded":"2024-08-17T17:41:05.523974+0000","last_fullsized":"2024-08-17T17:41:05.523974+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":358,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.439985+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:41:13.439985+0000","last_peered":"2024-08-17T17:41:13.439985+0000","last_clean":"2024-08-17T17:41:13.439985+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:41:13.439985+0000","last_undegraded":"2024-08-17T17:41:13.439985+0000","last_fullsized":"2024-08-17T17:41:13.439985+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":435,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.522396+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:41:05.522396+0000","last_peered":"2024-08-17T17:41:05.522396+0000","last_clean":"2024-08-17T17:41:05.522396+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:41:05.522396+0000","last_undegraded":"2024-08-17T17:41:05.522396+0000","last_fullsized":"2024-08-17T17:41:05.522396+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"383'10420","reported_seq":10588,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:15.346370+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:41:15.346370+0000","last_peered":"2024-08-17T17:41:15.346370+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:41:15.346370+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4087","ondisk_log_start":"326'4087","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285999104,"num_objects":4364,"num_object_clones":0,"num_object_copies":13092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10420,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4364},{"shards":"0,3","objects":-2884}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":633,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403189+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:41:13.403189+0000","last_peered":"2024-08-17T17:41:13.403189+0000","last_clean":"2024-08-17T17:41:13.403189+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:41:13.403189+0000","last_undegraded":"2024-08-17T17:41:13.403189+0000","last_fullsized":"2024-08-17T17:41:13.403189+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":601,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403050+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:41:13.403050+0000","last_peered":"2024-08-17T17:41:13.403050+0000","last_clean":"2024-08-17T17:41:13.403050+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:41:13.403050+0000","last_undegraded":"2024-08-17T17:41:13.403050+0000","last_fullsized":"2024-08-17T17:41:13.403050+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":619,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.405093+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:41:13.405093+0000","last_peered":"2024-08-17T17:41:13.405093+0000","last_clean":"2024-08-17T17:41:13.405093+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:41:13.405093+0000","last_undegraded":"2024-08-17T17:41:13.405093+0000","last_fullsized":"2024-08-17T17:41:13.405093+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"382'10263","reported_seq":24942,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:11.813765+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:41:11.813765+0000","last_peered":"2024-08-17T17:41:11.813765+0000","last_clean":"2024-08-17T17:41:11.813765+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:41:11.813765+0000","last_undegraded":"2024-08-17T17:41:11.813765+0000","last_fullsized":"2024-08-17T17:41:11.813765+0000","mapping_epoch":347,"log_start":"326'3672","ondisk_log_start":"326'3672","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282525696,"num_objects":4311,"num_object_clones":0,"num_object_copies":12933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10263,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":291,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.441300+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:41:13.441300+0000","last_peered":"2024-08-17T17:41:13.441300+0000","last_clean":"2024-08-17T17:41:13.441300+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:41:13.441300+0000","last_undegraded":"2024-08-17T17:41:13.441300+0000","last_fullsized":"2024-08-17T17:41:13.441300+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":386,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483622+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:40:52.483622+0000","last_peered":"2024-08-17T17:40:52.483622+0000","last_clean":"2024-08-17T17:40:52.483622+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:40:52.483622+0000","last_undegraded":"2024-08-17T17:40:52.483622+0000","last_fullsized":"2024-08-17T17:40:52.483622+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491817+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:41:05.491817+0000","last_peered":"2024-08-17T17:41:05.491817+0000","last_clean":"2024-08-17T17:41:05.491817+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:41:05.491817+0000","last_undegraded":"2024-08-17T17:41:05.491817+0000","last_fullsized":"2024-08-17T17:41:05.491817+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492703+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:41:05.492703+0000","last_peered":"2024-08-17T17:41:05.492703+0000","last_clean":"2024-08-17T17:41:05.492703+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:41:05.492703+0000","last_undegraded":"2024-08-17T17:41:05.492703+0000","last_fullsized":"2024-08-17T17:41:05.492703+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":380,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492355+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:41:05.492355+0000","last_peered":"2024-08-17T17:41:05.492355+0000","last_clean":"2024-08-17T17:41:05.492355+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:41:05.492355+0000","last_undegraded":"2024-08-17T17:41:05.492355+0000","last_fullsized":"2024-08-17T17:41:05.492355+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":608,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403218+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:41:13.403218+0000","last_peered":"2024-08-17T17:41:13.403218+0000","last_clean":"2024-08-17T17:41:13.403218+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:41:13.403218+0000","last_undegraded":"2024-08-17T17:41:13.403218+0000","last_fullsized":"2024-08-17T17:41:13.403218+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":148,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403894+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:41:13.403894+0000","last_peered":"2024-08-17T17:41:13.403894+0000","last_clean":"2024-08-17T17:41:13.403894+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:41:13.403894+0000","last_undegraded":"2024-08-17T17:41:13.403894+0000","last_fullsized":"2024-08-17T17:41:13.403894+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":287,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.522546+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:41:05.522546+0000","last_peered":"2024-08-17T17:41:05.522546+0000","last_clean":"2024-08-17T17:41:05.522546+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:41:05.522546+0000","last_undegraded":"2024-08-17T17:41:05.522546+0000","last_fullsized":"2024-08-17T17:41:05.522546+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":566,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.404273+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:41:13.404273+0000","last_peered":"2024-08-17T17:41:13.404273+0000","last_clean":"2024-08-17T17:41:13.404273+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:41:13.404273+0000","last_undegraded":"2024-08-17T17:41:13.404273+0000","last_fullsized":"2024-08-17T17:41:13.404273+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":431,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.522951+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:41:05.522951+0000","last_peered":"2024-08-17T17:41:05.522951+0000","last_clean":"2024-08-17T17:41:05.522951+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:41:05.522951+0000","last_undegraded":"2024-08-17T17:41:05.522951+0000","last_fullsized":"2024-08-17T17:41:05.522951+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":560,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.402772+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:41:13.402772+0000","last_peered":"2024-08-17T17:41:13.402772+0000","last_clean":"2024-08-17T17:41:13.402772+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:41:13.402772+0000","last_undegraded":"2024-08-17T17:41:13.402772+0000","last_fullsized":"2024-08-17T17:41:13.402772+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":353,"reported_epoch":381,"state":"active+clean","last_fresh":"2024-08-17T17:40:52.483561+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:40:52.483561+0000","last_peered":"2024-08-17T17:40:52.483561+0000","last_clean":"2024-08-17T17:40:52.483561+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:40:52.483561+0000","last_undegraded":"2024-08-17T17:40:52.483561+0000","last_fullsized":"2024-08-17T17:40:52.483561+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":591,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.405176+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:41:13.405176+0000","last_peered":"2024-08-17T17:41:13.405176+0000","last_clean":"2024-08-17T17:41:13.405176+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:41:13.405176+0000","last_undegraded":"2024-08-17T17:41:13.405176+0000","last_fullsized":"2024-08-17T17:41:13.405176+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":307,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403687+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:41:13.403687+0000","last_peered":"2024-08-17T17:41:13.403687+0000","last_clean":"2024-08-17T17:41:13.403687+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:41:13.403687+0000","last_undegraded":"2024-08-17T17:41:13.403687+0000","last_fullsized":"2024-08-17T17:41:13.403687+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":599,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.402231+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:41:13.402231+0000","last_peered":"2024-08-17T17:41:13.402231+0000","last_clean":"2024-08-17T17:41:13.402231+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:41:13.402231+0000","last_undegraded":"2024-08-17T17:41:13.402231+0000","last_fullsized":"2024-08-17T17:41:13.402231+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":559,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403687+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:41:13.403687+0000","last_peered":"2024-08-17T17:41:13.403687+0000","last_clean":"2024-08-17T17:41:13.403687+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:41:13.403687+0000","last_undegraded":"2024-08-17T17:41:13.403687+0000","last_fullsized":"2024-08-17T17:41:13.403687+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":366,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491558+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:41:05.491558+0000","last_peered":"2024-08-17T17:41:05.491558+0000","last_clean":"2024-08-17T17:41:05.491558+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:41:05.491558+0000","last_undegraded":"2024-08-17T17:41:05.491558+0000","last_fullsized":"2024-08-17T17:41:05.491558+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":248,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403405+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:41:13.403405+0000","last_peered":"2024-08-17T17:41:13.403405+0000","last_clean":"2024-08-17T17:41:13.403405+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:41:13.403405+0000","last_undegraded":"2024-08-17T17:41:13.403405+0000","last_fullsized":"2024-08-17T17:41:13.403405+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":304,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.440402+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:41:13.440402+0000","last_peered":"2024-08-17T17:41:13.440402+0000","last_clean":"2024-08-17T17:41:13.440402+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:41:13.440402+0000","last_undegraded":"2024-08-17T17:41:13.440402+0000","last_fullsized":"2024-08-17T17:41:13.440402+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":374,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492616+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:41:05.492616+0000","last_peered":"2024-08-17T17:41:05.492616+0000","last_clean":"2024-08-17T17:41:05.492616+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:41:05.492616+0000","last_undegraded":"2024-08-17T17:41:05.492616+0000","last_fullsized":"2024-08-17T17:41:05.492616+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":237,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492273+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:41:05.492273+0000","last_peered":"2024-08-17T17:41:05.492273+0000","last_clean":"2024-08-17T17:41:05.492273+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:41:05.492273+0000","last_undegraded":"2024-08-17T17:41:05.492273+0000","last_fullsized":"2024-08-17T17:41:05.492273+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":374,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491426+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:41:05.491426+0000","last_peered":"2024-08-17T17:41:05.491426+0000","last_clean":"2024-08-17T17:41:05.491426+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:41:05.491426+0000","last_undegraded":"2024-08-17T17:41:05.491426+0000","last_fullsized":"2024-08-17T17:41:05.491426+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":237,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.492273+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:41:05.492273+0000","last_peered":"2024-08-17T17:41:05.492273+0000","last_clean":"2024-08-17T17:41:05.492273+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:41:05.492273+0000","last_undegraded":"2024-08-17T17:41:05.492273+0000","last_fullsized":"2024-08-17T17:41:05.492273+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":611,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.404624+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:41:13.404624+0000","last_peered":"2024-08-17T17:41:13.404624+0000","last_clean":"2024-08-17T17:41:13.404624+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:41:13.404624+0000","last_undegraded":"2024-08-17T17:41:13.404624+0000","last_fullsized":"2024-08-17T17:41:13.404624+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":585,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.403956+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:41:13.403956+0000","last_peered":"2024-08-17T17:41:13.403956+0000","last_clean":"2024-08-17T17:41:13.403956+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:41:13.403956+0000","last_undegraded":"2024-08-17T17:41:13.403956+0000","last_fullsized":"2024-08-17T17:41:13.403956+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":337,"reported_epoch":382,"state":"active+clean","last_fresh":"2024-08-17T17:41:05.491080+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:41:05.491080+0000","last_peered":"2024-08-17T17:41:05.491080+0000","last_clean":"2024-08-17T17:41:05.491080+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:41:05.491080+0000","last_undegraded":"2024-08-17T17:41:05.491080+0000","last_fullsized":"2024-08-17T17:41:05.491080+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":307,"reported_epoch":383,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.440335+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:41:13.440335+0000","last_peered":"2024-08-17T17:41:13.440335+0000","last_clean":"2024-08-17T17:41:13.440335+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:41:13.440335+0000","last_undegraded":"2024-08-17T17:41:13.440335+0000","last_fullsized":"2024-08-17T17:41:13.440335+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4561174552,"num_objects":69599,"num_object_clones":0,"num_object_copies":208797,"num_objects_missing_on_primary":4976,"num_objects_missing":4976,"num_objects_degraded":53060,"num_objects_misplaced":27662,"num_objects_unfound":0,"num_objects_dirty":69599,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":166791,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29279,"num_bytes_recovered":1796276224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15326654464,"data_stored":15326642248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104810,"ondisk_log_size":104810,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170098,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463467,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2850616,"kb_used_data":1961072,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":90894536,"statfs":{"total":95995035648,"available":93076004864,"internally_reserved":0,"allocated":2008137728,"data_stored":2006060563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492142,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":960620,"kb_used_data":561124,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92784532,"statfs":{"total":95995035648,"available":95011360768,"internally_reserved":0,"allocated":574590976,"data_stored":572528318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986733,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2286528,"kb_used_data":1109684,"kb_used_omap":1,"kb_used_meta":1176830,"kb_avail":91458624,"statfs":{"total":95995035648,"available":93653630976,"internally_reserved":0,"allocated":1136316416,"data_stored":1134242398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1205074264},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225561,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3159280,"kb_used_data":2225508,"kb_used_omap":4,"kb_used_meta":933755,"kb_avail":90585872,"statfs":{"total":95995035648,"available":92759932928,"internally_reserved":0,"allocated":2278920192,"data_stored":2276857832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":956165568},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075886,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533779,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3000820,"kb_used_data":1954664,"kb_used_omap":5,"kb_used_meta":1046138,"kb_avail":90744332,"statfs":{"total":95995035648,"available":92922195968,"internally_reserved":0,"allocated":2001575936,"data_stored":1999508254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1071245552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089756,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2681868,"kb_used_data":1676800,"kb_used_omap":5,"kb_used_meta":1005050,"kb_avail":91063284,"statfs":{"total":95995035648,"available":93248802816,"internally_reserved":0,"allocated":1717043200,"data_stored":1714958301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1029171259},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1710358528,"data_stored":1710358528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1995509760,"data_stored":1995505688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2269249536,"data_stored":2269249536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1131216896,"data_stored":1131216896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":569180160,"data_stored":569180160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2002321408,"data_stored":2002321408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2309558272,"data_stored":2309554200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:19.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:41:19.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:19.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:19.531 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:19 smithi152 ceph-mon[93456]: from='client.56968 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:19.531 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:19 smithi152 ceph-mon[93456]: pgmap v1445: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 53063/208419 objects degraded (25.460%); 27633/208419 objects misplaced (13.258%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:41:19.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:19.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:19.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:19 smithi142 ceph-mon[103614]: from='client.56968 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:19.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:19 smithi142 ceph-mon[103614]: pgmap v1445: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 53063/208419 objects degraded (25.460%); 27633/208419 objects misplaced (13.258%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:41:19.697 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:19 smithi142 ceph-mon[106729]: from='client.56968 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:19.697 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:19 smithi142 ceph-mon[106729]: pgmap v1445: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 53063/208419 objects degraded (25.460%); 27633/208419 objects misplaced (13.258%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:41:19.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124385 12.9 12:9a4c7f9d:::benchmark_data_smithi142_261410_object6188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.895222+0000", 2024-08-17T17:41:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 469.86518012200003, 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34914930900000002, 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124385 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895222+0000", 2024-08-17T17:41:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895222+0000", 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895221+0000", 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895226+0000", 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2820000000000004e-06 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895237+0000", 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0229e-05 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.895242+0000", 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4219999999999999e-06 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.897492+0000", 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022502989999999999 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.897643+0000", 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015033399999999999 2024-08-17T17:41:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.897739+0000", 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6673000000000001e-05 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.900260+0000", 2024-08-17T17:41:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025210559999999998 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.901269+0000", 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001008985 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.239730+0000", 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33846039500000002 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.239766+0000", 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6640000000000002e-05 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.244372+0000", 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046050800000000001 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129851 12.f 12:fa60de86:::benchmark_data_smithi142_261410_object11654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 415.11300943999998, 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34876531300000002, 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129851 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647391+0000", 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647400+0000", 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1290000000000002e-06 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647417+0000", 2024-08-17T17:41:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7139e-05 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647425+0000", 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3310000000000004e-06 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647482+0000", 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6628000000000001e-05 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647642+0000", 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000160402 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647812+0000", 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016993499999999999 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.650735+0000", 2024-08-17T17:41:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002922371 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.651441+0000", 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070664600000000001 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.992978+0000", 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34153630699999998 2024-08-17T17:41:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993043+0000", 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4863999999999994e-05 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.996158+0000", 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031156859999999999 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131200 12.f 12:fe694d48:::benchmark_data_smithi142_261410_object13003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:54.647650+0000", 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 385.11275239899999, 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34767719600000002, 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131200 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647650+0000", 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647650+0000", 2024-08-17T17:41:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647649+0000", 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647654+0000", 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8719999999999999e-06 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647665+0000", 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0196e-05 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647670+0000", 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.772e-06 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647707+0000", 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6968999999999998e-05 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647816+0000", 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010883900000000001 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.647924+0000", 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107377 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.651120+0000", 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031966210000000002 2024-08-17T17:41:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.651133+0000", 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2846e-05 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.991970+0000", 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34083642600000003 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.992029+0000", 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9824000000000002e-05 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:54.995327+0000", 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032977639999999999 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131638 12.f 12:fb3cd103:::benchmark_data_smithi142_261410_object13441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 376.498261265, 2024-08-17T17:41:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35214242000000001, 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131638 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262140+0000", 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262145+0000", 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8250000000000004e-06 2024-08-17T17:41:19.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262153+0000", 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2870000000000004e-06 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262157+0000", 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6629999999999999e-06 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262194+0000", 2024-08-17T17:41:19.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7626000000000002e-05 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262367+0000", 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000172366 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262545+0000", 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017861400000000001 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.265416+0000", 2024-08-17T17:41:19.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028709809999999999 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.266086+0000", 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066993100000000002 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.611065+0000", 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34497888500000001 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.611104+0000", 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8411999999999997e-05 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.614283+0000", 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003179856 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131763 12.f 12:f549739a:::benchmark_data_smithi142_261410_object13566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 375.20812677700002, 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37794038699999999, 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131763 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552275+0000", 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:19.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552279+0000", 2024-08-17T17:41:19.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3080000000000001e-06 2024-08-17T17:41:19.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552290+0000", 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0900000000000001e-05 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552295+0000", 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8879999999999996e-06 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552328+0000", 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3683999999999998e-05 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552433+0000", 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104164 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552533+0000", 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100392 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.555327+0000", 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002794019 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.555992+0000", 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066522 2024-08-17T17:41:19.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.927119+0000", 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.371126601 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.927153+0000", 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3800000000000002e-05 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.930216+0000", 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030633309999999999 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131820 12.9 12:9a926c4c:::benchmark_data_smithi142_261410_object13623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 374.109189894, 2024-08-17T17:41:19.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81736743099999998, 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131820 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651211+0000", 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651218+0000", 2024-08-17T17:41:19.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6810000000000001e-06 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651229+0000", 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1258e-05 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651234+0000", 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5379999999999996e-06 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651287+0000", 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3192000000000002e-05 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651298+0000", 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.15e-05 2024-08-17T17:41:19.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461578+0000", 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81027978499999997 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461702+0000", 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012391299999999999 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461829+0000", 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012717300000000001 2024-08-17T17:41:19.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.465382+0000", 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035532430000000002 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.466095+0000", 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00071217299999999997 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468482+0000", 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023874 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468535+0000", 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3319000000000002e-05 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468580+0000", 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4569e-05 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.805 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:134422 12.f 12:fa29169b:::benchmark_data_smithi142_261410_object16225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 339.31013970800001, 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.45255063099999998, 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 134422 2024-08-17T17:41:19.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450261+0000", 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450270+0000", 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5539999999999998e-06 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450286+0000", 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6568000000000001e-05 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450292+0000", 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1550000000000004e-06 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450333+0000", 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0472000000000001e-05 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450457+0000", 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012431599999999999 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450565+0000", 2024-08-17T17:41:19.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108013 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.453491+0000", 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029262239999999998 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.453504+0000", 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2989999999999999e-05 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.898771+0000", 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.44526617200000002 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.898830+0000", 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9506999999999998e-05 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.902813+0000", 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039831399999999996 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.812 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:134490 12.f 12:f6a35853:::benchmark_data_smithi142_261410_object16293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 338.416477321, 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36548067000000001, 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 134490 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.813 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343924+0000", 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343929+0000", 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3769999999999998e-06 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343937+0000", 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1969999999999996e-06 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343940+0000", 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.641e-06 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343972+0000", 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1508999999999999e-05 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.344083+0000", 2024-08-17T17:41:19.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011066200000000001 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.344183+0000", 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9959999999999998e-05 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.347096+0000", 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029132110000000002 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.347261+0000", 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016498399999999999 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.706519+0000", 2024-08-17T17:41:19.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35925865099999998 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.706556+0000", 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6850999999999997e-05 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.709406+0000", 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028494760000000001 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.819 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:135448 12.f 12:fe02232f:::benchmark_data_smithi142_261410_object17251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 325.32705328700001, 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.46790546399999999, 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 135448 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433347+0000", 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:19.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433357+0000", 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0322999999999999e-05 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433376+0000", 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8244999999999999e-05 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433382+0000", 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433433+0000", 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0903999999999999e-05 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433554+0000", 2024-08-17T17:41:19.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012107099999999999 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433661+0000", 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010699 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.436511+0000", 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028494309999999999 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.436665+0000", 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015454999999999999 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.897905+0000", 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.46124029100000002 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.897963+0000", 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7336000000000001e-05 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.901255+0000", 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00329168 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:135735 12.f 12:ff30985f:::benchmark_data_smithi142_261410_object17538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 321.32763870299999, 2024-08-17T17:41:19.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35154203099999998, 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 135735 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432762+0000", 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432767+0000", 2024-08-17T17:41:19.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.262e-06 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432776+0000", 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7870000000000007e-06 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432779+0000", 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8859999999999997e-06 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436659+0000", 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003879263 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436779+0000", 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119862 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436904+0000", 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012594099999999999 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.439927+0000", 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030223350000000001 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.440109+0000", 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018190799999999999 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.781461+0000", 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34135219100000003 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.781503+0000", 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1789000000000003e-05 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.784306+0000", 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028029550000000002 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:139285 12.f 12:fc2a058b:::benchmark_data_smithi142_261410_object21088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:42.471011+0000", 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 277.289391292, 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37445746299999999, 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 139285 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.833 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471011+0000", 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471011+0000", 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471010+0000", 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471016+0000", 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3559999999999998e-06 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471028+0000", 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2196e-05 2024-08-17T17:41:19.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471033+0000", 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7040000000000002e-06 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471098+0000", 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4480000000000006e-05 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471275+0000", 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017774300000000001 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471459+0000", 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018332600000000001 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.474531+0000", 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003072342 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.474709+0000", 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017838500000000001 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.842685+0000", 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36797549899999998 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.842733+0000", 2024-08-17T17:41:19.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8000999999999997e-05 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.845468+0000", 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027356500000000001 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:142271 12.f 12:fb14d2d7:::benchmark_data_smithi142_261410_object24074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:41:19.839 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:15.842039+0000", 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 243.91836352000001, 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.58777459499999996, 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 142271 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842039+0000", 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842039+0000", 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842037+0000", 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842046+0000", 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0520000000000007e-06 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842062+0000", 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5586999999999998e-05 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842068+0000", 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.241e-06 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842120+0000", 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2330000000000002e-05 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842310+0000", 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018918799999999999 2024-08-17T17:41:19.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842435+0000", 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012570499999999999 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.845440+0000", 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030047049999999999 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.846107+0000", 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066691400000000001 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.426983+0000", 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.58087590600000005 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.427023+0000", 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0154999999999998e-05 2024-08-17T17:41:19.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.429813+0000", 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027903659999999999 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:142273 12.f 12:ffe89de1:::benchmark_data_smithi142_261410_object24076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:41:19.846 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:16.012532+0000", 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 243.74786993699999, 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.42049373800000001, 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 142273 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012532+0000", 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012532+0000", 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012531+0000", 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012537+0000", 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9869999999999996e-06 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012566+0000", 2024-08-17T17:41:19.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8473999999999999e-05 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012571+0000", 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9200000000000001e-06 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012628+0000", 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6849999999999999e-05 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012810+0000", 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018125500000000001 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.013010+0000", 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020028099999999999 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.016348+0000", 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033381299999999999 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.016420+0000", 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1633999999999999e-05 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.429875+0000", 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41345523499999998 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.429915+0000", 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9950000000000002e-05 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.433026+0000", 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003111309 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:144433 12.f 12:f489411b:::benchmark_data_smithi142_261410_object26236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e363)", 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:39.903296+0000", 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 219.85710656000001, 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.357136756, 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144433 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903296+0000", 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903296+0000", 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903294+0000", 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:19.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903320+0000", 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5673e-05 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903337+0000", 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7578999999999999e-05 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903361+0000", 2024-08-17T17:41:19.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4402000000000001e-05 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903436+0000", 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4136999999999998e-05 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903559+0000", 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123647 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903669+0000", 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109633 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.906501+0000", 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00283169 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.906652+0000", 2024-08-17T17:41:19.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000151108 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:40.257021+0000", 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.350369667 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:40.257060+0000", 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9096e-05 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:40.260433+0000", 2024-08-17T17:41:19.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033720550000000001 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:144895 12.f 12:f9c382c1:::benchmark_data_smithi142_261410_object26698:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:45.567171+0000", 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 214.193231285, 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37522107199999999, 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.860 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144895 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567171+0000", 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567171+0000", 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567170+0000", 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567177+0000", 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2169999999999997e-06 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567192+0000", 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5469e-05 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567199+0000", 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7020000000000004e-06 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567238+0000", 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8900999999999999e-05 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567356+0000", 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011850599999999999 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567461+0000", 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010467999999999999 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.570434+0000", 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029725390000000002 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.570526+0000", 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2232999999999996e-05 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.938453+0000", 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36792676899999999 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.938517+0000", 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4128000000000007e-05 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.942392+0000", 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003875351 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:146628 12.f 12:fe6860be:::benchmark_data_smithi142_261410_object28431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:02.033397+0000", 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 197.727005378, 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36051549100000002, 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 146628 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033397+0000", 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033397+0000", 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033394+0000", 2024-08-17T17:41:19.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999971 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033407+0000", 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2415999999999999e-05 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033428+0000", 2024-08-17T17:41:19.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1021000000000001e-05 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033437+0000", 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0990000000000005e-06 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033492+0000", 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5220000000000003e-05 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033612+0000", 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011965500000000001 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033715+0000", 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103764 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.036595+0000", 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028794129999999999 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.036700+0000", 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105485 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.390972+0000", 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35427146399999998 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.391013+0000", 2024-08-17T17:41:19.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1621e-05 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.393912+0000", 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028990109999999999 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:146630 12.f 12:fe52152e:::benchmark_data_smithi142_261410_object28433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:02.046096+0000", 2024-08-17T17:41:19.874 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 197.71430625400001, 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35065413000000001, 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 146630 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046096+0000", 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046096+0000", 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046094+0000", 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:19.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046105+0000", 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1138000000000001e-05 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046141+0000", 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5884999999999999e-05 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046166+0000", 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4876e-05 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046218+0000", 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2132000000000002e-05 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046344+0000", 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000126239 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046452+0000", 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108155 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.049292+0000", 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028396860000000001 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.049329+0000", 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6860000000000003e-05 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.393971+0000", 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34464169700000002 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.394029+0000", 2024-08-17T17:41:19.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7855000000000001e-05 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.396750+0000", 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027216889999999998 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:155660 12.f 12:fdb35f6e:::benchmark_data_smithi142_261410_object37463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2024-08-17T17:41:19.881 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:39:31.626313+0000", 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 108.13408894, 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.42314358899999999, 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155660 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626313+0000", 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626313+0000", 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626312+0000", 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626320+0000", 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2519999999999999e-06 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626337+0000", 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6727000000000001e-05 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626343+0000", 2024-08-17T17:41:19.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5590000000000002e-06 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626387+0000", 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3513999999999997e-05 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626506+0000", 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118824 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626618+0000", 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111768 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.629419+0000", 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028014849999999998 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.629565+0000", 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014616999999999999 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:32.046449+0000", 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41688390600000003 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:32.046510+0000", 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1415999999999996e-05 2024-08-17T17:41:19.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:32.049457+0000", 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029464840000000001 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165832 12.f 12:f9f5aeac:::benchmark_data_smithi142_261410_object47635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.723175+0000", 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 16.037226843999999, 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35249307400000002, 2024-08-17T17:41:19.888 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165832 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723175+0000", 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723175+0000", 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723174+0000", 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723180+0000", 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2230000000000004e-06 2024-08-17T17:41:19.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723193+0000", 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2278e-05 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723198+0000", 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2079999999999999e-06 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723235+0000", 2024-08-17T17:41:19.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6912999999999997e-05 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723354+0000", 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011915499999999999 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723459+0000", 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104681 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.726254+0000", 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027948930000000001 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.726419+0000", 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016569499999999999 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.072089+0000", 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34567017999999999 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.072152+0000", 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2698000000000002e-05 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.075669+0000", 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035164739999999999 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:167333 12.f 12:f036c062:::benchmark_data_smithi142_261410_object49136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e383)", 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:18.525405+0000", 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.2349977990000001, 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.374992718, 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 167333 2024-08-17T17:41:19.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525405+0000", 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525405+0000", 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:19.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525403+0000", 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525412+0000", 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2499999999999995e-06 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525434+0000", 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1280999999999998e-05 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525442+0000", 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1289999999999996e-06 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525483+0000", 2024-08-17T17:41:19.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1233999999999999e-05 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525648+0000", 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000165076 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525761+0000", 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011281 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:19.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.528578+0000", 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028169430000000001 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.528795+0000", 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021709499999999999 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.897555+0000", 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36876039900000002 2024-08-17T17:41:19.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.897596+0000", 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0664000000000002e-05 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.900397+0000", 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028013920000000002 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.901 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:19.902 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:19.902 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:41:19.924 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:19.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:19.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:20.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:20.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:20.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:20.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:20.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:20.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:20.294 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:20.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:20.411 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:20.419 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:20.421 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:20.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:20.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:20.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:41:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:41:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:20.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:20.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:20.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:20.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:20.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:20.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:20.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:41:20.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:20.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:20.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:20.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:21.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:21.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:21.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:21.191 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:21.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:21.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:21.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:21.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:21.520 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:21 smithi142 ceph-mon[106729]: pgmap v1446: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 52982/207540 objects degraded (25.529%); 27447/207540 objects misplaced (13.225%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:21.520 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52982/207540 objects degraded (25.529%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:21.520 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:21 smithi142 ceph-mon[103614]: pgmap v1446: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 52982/207540 objects degraded (25.529%); 27447/207540 objects misplaced (13.225%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:21.520 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52982/207540 objects degraded (25.529%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:21.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:21.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:21.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:21.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:21.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:21.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:21 smithi152 ceph-mon[93456]: pgmap v1446: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 52982/207540 objects degraded (25.529%); 27447/207540 objects misplaced (13.225%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:21.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52982/207540 objects degraded (25.529%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:21.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:21.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:21.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:21.942 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:21.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:22.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:22.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:22.107 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:41:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:41:21] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:41:22.154 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:22.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:22.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:22.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:22.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:22.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:22.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:22.432 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:41:22.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:22.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:22.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:22.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3843634230' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:22.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:22.694 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:22.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:22.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:22.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3843634230' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:22.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3843634230' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:22.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:22.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:22.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:23.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:23.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:23.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:23.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:23.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:23.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:23.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:23.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:23.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[103614]: pgmap v1447: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 52982/206586 objects degraded (25.646%); 27402/206586 objects misplaced (13.264%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3843634230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[103614]: osdmap e384: 8 total, 8 up, 6 in 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[106729]: pgmap v1447: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 52982/206586 objects degraded (25.646%); 27402/206586 objects misplaced (13.264%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3843634230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:23.494 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:23 smithi142 ceph-mon[106729]: osdmap e384: 8 total, 8 up, 6 in 2024-08-17T17:41:23.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:23 smithi152 ceph-mon[93456]: pgmap v1447: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 52982/206586 objects degraded (25.646%); 27402/206586 objects misplaced (13.264%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:41:23.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:23 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:41:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3843634230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:23.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:23 smithi152 ceph-mon[93456]: osdmap e384: 8 total, 8 up, 6 in 2024-08-17T17:41:23.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:23.618 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:41:23.618 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:41:23.619 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:23.619 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:23.634 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:41:23.635 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:23.653 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:41:23.654 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:23.670 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:41:23.671 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:23.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:23.686 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:41:23.687 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:23.702 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:41:23.702 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:23.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:23.717 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:41:23.717 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:23.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:23.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:23.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:23.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:23.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:23.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:23.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:24.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:24.075 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:24.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:24.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:24.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:24.323 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:24.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:24.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:24.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3921023360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:24.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:24.573 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:24.609 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:24.609 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:24.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:24.700 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:24.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:24.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:24.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3921023360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:24.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3921023360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:24.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:24.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:24.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:24.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:25.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:25.076 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:25.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:25.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:25.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:25.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:25.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:41:25.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:25.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:25.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:25.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:25.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:25.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:25 smithi152 ceph-mon[93456]: pgmap v1449: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52903/205986 objects degraded (25.683%); 27212/205986 objects misplaced (13.211%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:25.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:25.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:25.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:25.705 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:25 smithi142 ceph-mon[106729]: pgmap v1449: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52903/205986 objects degraded (25.683%); 27212/205986 objects misplaced (13.211%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:25.705 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:25 smithi142 ceph-mon[103614]: pgmap v1449: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52903/205986 objects degraded (25.683%); 27212/205986 objects misplaced (13.211%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:25.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:25.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:25.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:25.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:25.964 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:25.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:25.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:26.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:26.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:26.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:26.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:26.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:26.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:26.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:26.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:26.456 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:26.457 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 52903/205260 objects degraded (25.774%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2167,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":384,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":68420,"data_bytes":4482799589,"bytes_used":15149613056,"bytes_avail":560820600832,"bytes_total":575970213888,"degraded_objects":52903,"degraded_total":205260,"degraded_ratio":0.25773652927993762,"misplaced_objects":27212,"misplaced_total":205260,"misplaced_ratio":0.13257332164084576,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1487276,"recovering_keys_per_sec":0,"num_objects_recovered":227,"num_bytes_recovered":14876672,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":120},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:41:26.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:26.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:26.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:26.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:26.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:26.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52903/205986 objects degraded (25.683%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52903/205986 objects degraded (25.683%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:26.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:26.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52903/205986 objects degraded (25.683%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:26.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:26.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:26.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:26.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:26.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:41:26.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:26.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:27.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:27.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:27.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:27.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:27.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:27.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:27.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:27.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:41:27.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:27.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:27.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:27.564 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:41:27.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:27.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:27.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:27.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:27.707 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:27.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:27 smithi152 ceph-mon[93456]: pgmap v1450: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52903/205260 objects degraded (25.774%); 27212/205260 objects misplaced (13.257%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1510285380' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:27.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[106729]: pgmap v1450: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52903/205260 objects degraded (25.774%); 27212/205260 objects misplaced (13.257%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1510285380' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[103614]: pgmap v1450: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52903/205260 objects degraded (25.774%); 27212/205260 objects misplaced (13.257%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:27.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1510285380' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:27.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:27.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:27.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:27.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:27.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:28.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:28.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:28.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:28.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:28.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:28.365 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:28.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:28.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:28.549 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:28.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:28.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:28.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:28.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:28.783 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:28.784 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:28.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:28.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:28.970 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:28.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:28.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:28.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:29.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:29.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:29.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:29.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:29.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:29.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:29.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:29.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:29.567 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:29.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:29.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:29.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:29.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:41:29.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:29.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:29 smithi152 ceph-mon[93456]: pgmap v1451: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 52903/204615 objects degraded (25.855%); 27178/204615 objects misplaced (13.283%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:29.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:29.848 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:29 smithi142 ceph-mon[106729]: pgmap v1451: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 52903/204615 objects degraded (25.855%); 27178/204615 objects misplaced (13.283%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:29.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:29 smithi142 ceph-mon[103614]: pgmap v1451: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 52903/204615 objects degraded (25.855%); 27178/204615 objects misplaced (13.283%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:29.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:29.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:29.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:29.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:30.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:30.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:30.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:30.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:30.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:30.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:30.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:30.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:30.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:30.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:30.510 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:30.539 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:41:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:41:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:30.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:30.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:30.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:30.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:30.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:30.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:30.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:30.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:30.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:30.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:30.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:31.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:31.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:31.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:31.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:31.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:31.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:31.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:31.456 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:41:31.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:31.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:31.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:31.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:31.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[106729]: pgmap v1452: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 52827/203658 objects degraded (25.939%); 26960/203658 objects misplaced (13.238%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:31.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2056143187' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52827/203658 objects degraded (25.939%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[103614]: pgmap v1452: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 52827/203658 objects degraded (25.939%); 26960/203658 objects misplaced (13.238%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2056143187' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52827/203658 objects degraded (25.939%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:31.600 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:31.615 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:31.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:31 smithi152 ceph-mon[93456]: pgmap v1452: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 52827/203658 objects degraded (25.939%); 26960/203658 objects misplaced (13.238%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:31.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2056143187' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:31.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52827/203658 objects degraded (25.939%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:31.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:31.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:31 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:31.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:31.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:31.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:31.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:41:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:41:31] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:41:31.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:31.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:31.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:32.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:32.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:32.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:32.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:32.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:32.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:32.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:32.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:32.425 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:32.425 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:32.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:32.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:32.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:32 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2056143187' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:41:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:32 smithi152 ceph-mon[93456]: osdmap e385: 8 total, 8 up, 6 in 2024-08-17T17:41:32.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:32.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:32.677 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:32 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:32.678 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2056143187' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:41:32.678 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:32 smithi142 ceph-mon[103614]: osdmap e385: 8 total, 8 up, 6 in 2024-08-17T17:41:32.678 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:32 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:32.678 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2056143187' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:41:32.678 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:32 smithi142 ceph-mon[106729]: osdmap e385: 8 total, 8 up, 6 in 2024-08-17T17:41:32.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:32.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:32.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:32.861 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:32.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:32.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:32.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:32.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:33.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:33.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:33.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:41:33.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:33.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:33.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:33.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:33.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:33.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:33.476 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:33.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:33.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:33.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:33.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:33.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:33.765 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:33 smithi152 ceph-mon[93456]: pgmap v1454: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 52827/202782 objects degraded (26.051%); 26926/202782 objects misplaced (13.278%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:41:33.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:33 smithi142 ceph-mon[106729]: pgmap v1454: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 52827/202782 objects degraded (26.051%); 26926/202782 objects misplaced (13.278%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:41:33.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:33 smithi142 ceph-mon[103614]: pgmap v1454: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 52827/202782 objects degraded (26.051%); 26926/202782 objects misplaced (13.278%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:41:33.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:33.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:33.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:34.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:34.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:34.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:34.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:34.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:34.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:34.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:34.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:41:34.473 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:34.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:34.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:34.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:34.718 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:34.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:34.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:34.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:34.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:34.975 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:34.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:34.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:34.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:35.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.122 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:35.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:35.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:35.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:35.268 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:41:35.269 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:35.293 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:41:35.294 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:35.314 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:41:35.314 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:35.331 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:41:35.331 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:35.349 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:41:35.350 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:35.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:35.371 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:41:35.372 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:35.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:35.389 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:41:35.389 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:35.406 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:41:35.406 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:35.408 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:41:35.408 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:41:35.409 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:41:35.409 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:41:35.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:35.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:35.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:35.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:35.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:35.728 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:35 smithi152 ceph-mon[93456]: pgmap v1455: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 52801/202599 objects degraded (26.062%); 26865/202599 objects misplaced (13.260%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:41:35.728 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2912642692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:35.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:35.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:35.860 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:35 smithi142 ceph-mon[106729]: pgmap v1455: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 52801/202599 objects degraded (26.062%); 26865/202599 objects misplaced (13.260%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:41:35.860 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2912642692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:35.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:35 smithi142 ceph-mon[103614]: pgmap v1455: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 52801/202599 objects degraded (26.062%); 26865/202599 objects misplaced (13.260%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:41:35.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2912642692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:35.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:35.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:35.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:35.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:36.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:36.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:36.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:36.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:41:36.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:36.240 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:36.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:36.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:36.446 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:36.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:36.467 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52801/202599 objects degraded (26.062%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:36.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:36.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:36.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:36.665 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52801/202599 objects degraded (26.062%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:36.665 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52801/202599 objects degraded (26.062%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:36.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:36.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:36.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:36.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:36.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:36.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:36.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:37.080 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:37.081 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:41:37.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:37.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:37.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:37.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:37.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:37.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:37.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:37.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:37.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:37.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:37.558 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:37 smithi142 ceph-mon[106729]: pgmap v1456: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52763/201717 objects degraded (26.157%); 26764/201717 objects misplaced (13.268%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:37.558 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:37.558 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:37.559 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:37 smithi142 ceph-mon[103614]: pgmap v1456: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52763/201717 objects degraded (26.157%); 26764/201717 objects misplaced (13.268%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:37.559 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:37.559 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:37.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:37.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:37.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:37.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:37.754 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:37 smithi152 ceph-mon[93456]: pgmap v1456: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52763/201717 objects degraded (26.157%); 26764/201717 objects misplaced (13.268%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:37.754 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:37.754 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:37.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:37.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:37.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:37.989 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1456,"stamp":"2024-08-17T17:41:36.345395+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4405401573,"num_objects":67239,"num_object_clones":0,"num_object_copies":201717,"num_objects_missing_on_primary":4830,"num_objects_missing":4830,"num_objects_degraded":52763,"num_objects_misplaced":26764,"num_objects_unfound":0,"num_objects_dirty":67239,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":169455,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29764,"num_bytes_recovered":1827742350,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104703,"ondisk_log_size":104703,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14629212,"kb_used_data":9164452,"kb_used_omap":27,"kb_used_meta":5464676,"kb_avail":547841700,"statfs":{"total":575970213888,"available":560989900800,"internally_reserved":0,"allocated":9384398848,"data_stored":9371868757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5595828968},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-77398016,"num_objects":-1181,"num_object_clones":0,"num_object_copies":-3543,"num_objects_missing_on_primary":-74,"num_objects_missing":-74,"num_objects_degraded":-140,"num_objects_misplaced":-448,"num_objects_unfound":0,"num_objects_dirty":-1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1181,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":217,"num_bytes_recovered":14221312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002849"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":613,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458547+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:41:31.458547+0000","last_peered":"2024-08-17T17:41:31.458547+0000","last_clean":"2024-08-17T17:41:31.458547+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:41:31.458547+0000","last_undegraded":"2024-08-17T17:41:31.458547+0000","last_fullsized":"2024-08-17T17:41:31.458547+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":94,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.456462+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:41:22.456462+0000","last_peered":"2024-08-17T17:41:22.456462+0000","last_clean":"2024-08-17T17:41:22.456462+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:41:22.456462+0000","last_undegraded":"2024-08-17T17:41:22.456462+0000","last_fullsized":"2024-08-17T17:41:22.456462+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":288,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523840+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:41:22.523840+0000","last_peered":"2024-08-17T17:41:22.523840+0000","last_clean":"2024-08-17T17:41:22.523840+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:41:22.523840+0000","last_undegraded":"2024-08-17T17:41:22.523840+0000","last_fullsized":"2024-08-17T17:41:22.523840+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":311,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459048+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:41:31.459048+0000","last_peered":"2024-08-17T17:41:31.459048+0000","last_clean":"2024-08-17T17:41:31.459048+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:41:31.459048+0000","last_undegraded":"2024-08-17T17:41:31.459048+0000","last_fullsized":"2024-08-17T17:41:31.459048+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":488,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458249+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:41:31.458249+0000","last_peered":"2024-08-17T17:41:31.458249+0000","last_clean":"2024-08-17T17:41:31.458249+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:41:31.458249+0000","last_undegraded":"2024-08-17T17:41:31.458249+0000","last_fullsized":"2024-08-17T17:41:31.458249+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":237,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453847+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:41:22.453847+0000","last_peered":"2024-08-17T17:41:22.453847+0000","last_clean":"2024-08-17T17:41:22.453847+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:41:22.453847+0000","last_undegraded":"2024-08-17T17:41:22.453847+0000","last_fullsized":"2024-08-17T17:41:22.453847+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":581,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457921+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:41:31.457921+0000","last_peered":"2024-08-17T17:41:31.457921+0000","last_clean":"2024-08-17T17:41:31.457921+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:41:31.457921+0000","last_undegraded":"2024-08-17T17:41:31.457921+0000","last_fullsized":"2024-08-17T17:41:31.457921+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":405,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453899+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:41:22.453899+0000","last_peered":"2024-08-17T17:41:22.453899+0000","last_clean":"2024-08-17T17:41:22.453899+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:41:22.453899+0000","last_undegraded":"2024-08-17T17:41:22.453899+0000","last_fullsized":"2024-08-17T17:41:22.453899+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"385'10522","reported_seq":15501,"reported_epoch":385,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:41:34.177542+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:41:34.177542+0000","last_peered":"2024-08-17T17:41:34.177542+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:41:34.177542+0000","last_undegraded":"2024-08-17T17:41:34.177542+0000","last_fullsized":"2024-08-17T17:41:34.177542+0000","mapping_epoch":337,"log_start":"326'3663","ondisk_log_start":"326'3663","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274071552,"num_objects":4182,"num_object_clones":0,"num_object_copies":12546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4182,"num_objects_unfound":0,"num_objects_dirty":4182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10522,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2441,"num_bytes_recovered":159973376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4182}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":90,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.553493+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:41:22.553493+0000","last_peered":"2024-08-17T17:41:22.553493+0000","last_clean":"2024-08-17T17:41:22.553493+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:41:22.553493+0000","last_undegraded":"2024-08-17T17:41:22.553493+0000","last_fullsized":"2024-08-17T17:41:22.553493+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523502+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:41:22.523502+0000","last_peered":"2024-08-17T17:41:22.523502+0000","last_clean":"2024-08-17T17:41:22.523502+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:41:22.523502+0000","last_undegraded":"2024-08-17T17:41:22.523502+0000","last_fullsized":"2024-08-17T17:41:22.523502+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"385'10728","reported_seq":10811,"reported_epoch":385,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:32.053575+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:41:32.053575+0000","last_peered":"2024-08-17T17:41:32.053575+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:41:32.053575+0000","last_undegraded":"2024-08-17T17:41:32.053575+0000","last_fullsized":"2024-08-17T17:41:32.053575+0000","mapping_epoch":337,"log_start":"326'4182","ondisk_log_start":"326'4182","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6546,"log_dups_size":0,"ondisk_log_size":6546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284819456,"num_objects":4346,"num_object_clones":0,"num_object_copies":13038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4346,"num_objects_unfound":0,"num_objects_dirty":4346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10728,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4346}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":92,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.455010+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:41:22.455010+0000","last_peered":"2024-08-17T17:41:22.455010+0000","last_clean":"2024-08-17T17:41:22.455010+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:41:22.455010+0000","last_undegraded":"2024-08-17T17:41:22.455010+0000","last_fullsized":"2024-08-17T17:41:22.455010+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":356,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453784+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:41:22.453784+0000","last_peered":"2024-08-17T17:41:22.453784+0000","last_clean":"2024-08-17T17:41:22.453784+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:41:22.453784+0000","last_undegraded":"2024-08-17T17:41:22.453784+0000","last_fullsized":"2024-08-17T17:41:22.453784+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"385'10533","reported_seq":10619,"reported_epoch":385,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:34.190102+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:41:34.190102+0000","last_peered":"2024-08-17T17:41:34.190102+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:41:34.190102+0000","last_undegraded":"2024-08-17T17:41:34.190102+0000","last_fullsized":"2024-08-17T17:41:34.190102+0000","mapping_epoch":337,"log_start":"326'3653","ondisk_log_start":"326'3653","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275316736,"num_objects":4201,"num_object_clones":0,"num_object_copies":12603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4201,"num_objects_unfound":0,"num_objects_dirty":4201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10533,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4201}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":88,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523766+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:41:22.523766+0000","last_peered":"2024-08-17T17:41:22.523766+0000","last_clean":"2024-08-17T17:41:22.523766+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:41:22.523766+0000","last_undegraded":"2024-08-17T17:41:22.523766+0000","last_fullsized":"2024-08-17T17:41:22.523766+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":366,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.457006+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:41:22.457006+0000","last_peered":"2024-08-17T17:41:22.457006+0000","last_clean":"2024-08-17T17:41:22.457006+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:41:22.457006+0000","last_undegraded":"2024-08-17T17:41:22.457006+0000","last_fullsized":"2024-08-17T17:41:22.457006+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"385'10735","reported_seq":10885,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:34.589919+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:41:34.589919+0000","last_peered":"2024-08-17T17:41:34.589919+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:41:34.589919+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'3713","ondisk_log_start":"326'3713","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276758528,"num_objects":4223,"num_object_clones":0,"num_object_copies":12669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10735,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2546},{"shards":"0,3,7","objects":4223}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":525,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458694+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:41:31.458694+0000","last_peered":"2024-08-17T17:41:31.458694+0000","last_clean":"2024-08-17T17:41:31.458694+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:41:31.458694+0000","last_undegraded":"2024-08-17T17:41:31.458694+0000","last_fullsized":"2024-08-17T17:41:31.458694+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":291,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.625994+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:41:31.625994+0000","last_peered":"2024-08-17T17:41:31.625994+0000","last_clean":"2024-08-17T17:41:31.625994+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:41:31.625994+0000","last_undegraded":"2024-08-17T17:41:31.625994+0000","last_fullsized":"2024-08-17T17:41:31.625994+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"385'10409","reported_seq":10483,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:32.198533+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:41:32.198533+0000","last_peered":"2024-08-17T17:41:32.198533+0000","last_clean":"2024-08-17T17:41:32.198533+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:41:32.198533+0000","last_undegraded":"2024-08-17T17:41:32.198533+0000","last_fullsized":"2024-08-17T17:41:32.198533+0000","mapping_epoch":319,"log_start":"326'3874","ondisk_log_start":"326'3874","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274268160,"num_objects":4185,"num_object_clones":0,"num_object_copies":12555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10409,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":614,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460344+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:41:31.460344+0000","last_peered":"2024-08-17T17:41:31.460344+0000","last_clean":"2024-08-17T17:41:31.460344+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:41:31.460344+0000","last_undegraded":"2024-08-17T17:41:31.460344+0000","last_fullsized":"2024-08-17T17:41:31.460344+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454309+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:41:22.454309+0000","last_peered":"2024-08-17T17:41:22.454309+0000","last_clean":"2024-08-17T17:41:22.454309+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:41:22.454309+0000","last_undegraded":"2024-08-17T17:41:22.454309+0000","last_fullsized":"2024-08-17T17:41:22.454309+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"385'10618","reported_seq":10702,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:31.780806+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:41:31.780806+0000","last_peered":"2024-08-17T17:41:31.780806+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:41:31.780806+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'4035","ondisk_log_start":"326'4035","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282722304,"num_objects":4314,"num_object_clones":0,"num_object_copies":12942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10618,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4314},{"shards":"1,6","objects":-3052}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":340,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.457109+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:41:22.457109+0000","last_peered":"2024-08-17T17:41:22.457109+0000","last_clean":"2024-08-17T17:41:22.457109+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:41:22.457109+0000","last_undegraded":"2024-08-17T17:41:22.457109+0000","last_fullsized":"2024-08-17T17:41:22.457109+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":366,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454789+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:41:22.454789+0000","last_peered":"2024-08-17T17:41:22.454789+0000","last_clean":"2024-08-17T17:41:22.454789+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:41:22.454789+0000","last_undegraded":"2024-08-17T17:41:22.454789+0000","last_fullsized":"2024-08-17T17:41:22.454789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"385'10760","reported_seq":10911,"reported_epoch":385,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:33.876730+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:41:33.876730+0000","last_peered":"2024-08-17T17:41:33.876730+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:41:33.876730+0000","last_undegraded":"2024-08-17T17:41:33.876730+0000","last_fullsized":"2024-08-17T17:41:33.876730+0000","mapping_epoch":337,"log_start":"327'5161","ondisk_log_start":"327'5161","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5599,"log_dups_size":0,"ondisk_log_size":5599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278528000,"num_objects":4250,"num_object_clones":0,"num_object_copies":12750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4250,"num_objects_unfound":0,"num_objects_dirty":4250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10760,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4250}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":377,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453997+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:41:22.453997+0000","last_peered":"2024-08-17T17:41:22.453997+0000","last_clean":"2024-08-17T17:41:22.453997+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:41:22.453997+0000","last_undegraded":"2024-08-17T17:41:22.453997+0000","last_fullsized":"2024-08-17T17:41:22.453997+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.455118+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:41:22.455118+0000","last_peered":"2024-08-17T17:41:22.455118+0000","last_clean":"2024-08-17T17:41:22.455118+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:41:22.455118+0000","last_undegraded":"2024-08-17T17:41:22.455118+0000","last_fullsized":"2024-08-17T17:41:22.455118+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"385'10568","reported_seq":15688,"reported_epoch":385,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:34.301251+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:41:34.301251+0000","last_peered":"2024-08-17T17:41:34.301251+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:41:34.301251+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:41:34.301251+0000","mapping_epoch":337,"log_start":"326'3741","ondisk_log_start":"326'3741","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271056896,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":4830,"num_objects_missing":4830,"num_objects_degraded":4136,"num_objects_misplaced":4830,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10568,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2510,"num_bytes_recovered":164429824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":4124}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":584,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459673+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:41:31.459673+0000","last_peered":"2024-08-17T17:41:31.459673+0000","last_clean":"2024-08-17T17:41:31.459673+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:41:31.459673+0000","last_undegraded":"2024-08-17T17:41:31.459673+0000","last_fullsized":"2024-08-17T17:41:31.459673+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":521,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458000+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:41:31.458000+0000","last_peered":"2024-08-17T17:41:31.458000+0000","last_clean":"2024-08-17T17:41:31.458000+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:41:31.458000+0000","last_undegraded":"2024-08-17T17:41:31.458000+0000","last_fullsized":"2024-08-17T17:41:31.458000+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"385'10691","reported_seq":15817,"reported_epoch":385,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:33.893645+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:41:33.893645+0000","last_peered":"2024-08-17T17:41:33.893645+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:41:33.893645+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:41:33.893645+0000","mapping_epoch":337,"log_start":"327'5044","ondisk_log_start":"327'5044","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275513368,"num_objects":4205,"num_object_clones":0,"num_object_copies":12615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4205,"num_objects_misplaced":4955,"num_objects_unfound":0,"num_objects_dirty":4205,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10691,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2493,"num_bytes_recovered":163315712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4205}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":714,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:31.624300+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:41:31.624300+0000","last_peered":"2024-08-17T17:41:31.624300+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:41:31.624300+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":314,"reported_epoch":384,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:41:22.523722+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:41:22.523722+0000","last_peered":"2024-08-17T17:41:22.523722+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:41:22.523722+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:41:22.523722+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":563,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458959+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:41:31.458959+0000","last_peered":"2024-08-17T17:41:31.458959+0000","last_clean":"2024-08-17T17:41:31.458959+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:41:31.458959+0000","last_undegraded":"2024-08-17T17:41:31.458959+0000","last_fullsized":"2024-08-17T17:41:31.458959+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.464726+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:41:22.464726+0000","last_peered":"2024-08-17T17:41:22.464726+0000","last_clean":"2024-08-17T17:41:22.464726+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:41:22.464726+0000","last_undegraded":"2024-08-17T17:41:22.464726+0000","last_fullsized":"2024-08-17T17:41:22.464726+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"385'10534","reported_seq":10687,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:33.850080+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:41:33.850080+0000","last_peered":"2024-08-17T17:41:33.850080+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:41:33.850080+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'3521","ondisk_log_start":"325'3521","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275906560,"num_objects":4210,"num_object_clones":0,"num_object_copies":12630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10534,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2437},{"shards":"0,4,7","objects":4210}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":379,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457267+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:41:31.457267+0000","last_peered":"2024-08-17T17:41:31.457267+0000","last_clean":"2024-08-17T17:41:31.457267+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:41:31.457267+0000","last_undegraded":"2024-08-17T17:41:31.457267+0000","last_fullsized":"2024-08-17T17:41:31.457267+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":321,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453673+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:41:22.453673+0000","last_peered":"2024-08-17T17:41:22.453673+0000","last_clean":"2024-08-17T17:41:22.453673+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:41:22.453673+0000","last_undegraded":"2024-08-17T17:41:22.453673+0000","last_fullsized":"2024-08-17T17:41:22.453673+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":305,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.625133+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:41:31.625133+0000","last_peered":"2024-08-17T17:41:31.625133+0000","last_clean":"2024-08-17T17:41:31.625133+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:41:31.625133+0000","last_undegraded":"2024-08-17T17:41:31.625133+0000","last_fullsized":"2024-08-17T17:41:31.625133+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"385'10562","reported_seq":10646,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:31.849983+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:41:31.849983+0000","last_peered":"2024-08-17T17:41:31.849983+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:41:31.849983+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4017","ondisk_log_start":"326'4017","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277479424,"num_objects":4234,"num_object_clones":0,"num_object_copies":12702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10562,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4234},{"shards":"5,6","objects":-3062}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":471,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459958+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:41:31.459958+0000","last_peered":"2024-08-17T17:41:31.459958+0000","last_clean":"2024-08-17T17:41:31.459958+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:41:31.459958+0000","last_undegraded":"2024-08-17T17:41:31.459958+0000","last_fullsized":"2024-08-17T17:41:31.459958+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453678+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:41:22.453678+0000","last_peered":"2024-08-17T17:41:22.453678+0000","last_clean":"2024-08-17T17:41:22.453678+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:41:22.453678+0000","last_undegraded":"2024-08-17T17:41:22.453678+0000","last_fullsized":"2024-08-17T17:41:22.453678+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":656,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459293+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:41:31.459293+0000","last_peered":"2024-08-17T17:41:31.459293+0000","last_clean":"2024-08-17T17:41:31.459293+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:41:31.459293+0000","last_undegraded":"2024-08-17T17:41:31.459293+0000","last_fullsized":"2024-08-17T17:41:31.459293+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"385'10484","reported_seq":11361,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:34.838173+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:41:34.838173+0000","last_peered":"2024-08-17T17:41:34.838173+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:41:34.838173+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'3416","ondisk_log_start":"325'3416","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272760832,"num_objects":4162,"num_object_clones":0,"num_object_copies":12486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10484,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4162},{"shards":"0,5","objects":-2492}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":448,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457595+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:41:31.457595+0000","last_peered":"2024-08-17T17:41:31.457595+0000","last_clean":"2024-08-17T17:41:31.457595+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:41:31.457595+0000","last_undegraded":"2024-08-17T17:41:31.457595+0000","last_fullsized":"2024-08-17T17:41:31.457595+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":360,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.457040+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:41:22.457040+0000","last_peered":"2024-08-17T17:41:22.457040+0000","last_clean":"2024-08-17T17:41:22.457040+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:41:22.457040+0000","last_undegraded":"2024-08-17T17:41:22.457040+0000","last_fullsized":"2024-08-17T17:41:22.457040+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":242,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458479+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:41:31.458479+0000","last_peered":"2024-08-17T17:41:31.458479+0000","last_clean":"2024-08-17T17:41:31.458479+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:41:31.458479+0000","last_undegraded":"2024-08-17T17:41:31.458479+0000","last_fullsized":"2024-08-17T17:41:31.458479+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"385'10706","reported_seq":25702,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:35.510040+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:41:35.510040+0000","last_peered":"2024-08-17T17:41:35.510040+0000","last_clean":"2024-08-17T17:41:35.510040+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:41:35.510040+0000","last_undegraded":"2024-08-17T17:41:35.510040+0000","last_fullsized":"2024-08-17T17:41:35.510040+0000","mapping_epoch":355,"log_start":"326'4430","ondisk_log_start":"326'4430","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6276,"log_dups_size":0,"ondisk_log_size":6276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270663680,"num_objects":4130,"num_object_clones":0,"num_object_copies":12390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10706,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":299,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523673+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:41:22.523673+0000","last_peered":"2024-08-17T17:41:22.523673+0000","last_clean":"2024-08-17T17:41:22.523673+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:41:22.523673+0000","last_undegraded":"2024-08-17T17:41:22.523673+0000","last_fullsized":"2024-08-17T17:41:22.523673+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458413+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:41:31.458413+0000","last_peered":"2024-08-17T17:41:31.458413+0000","last_clean":"2024-08-17T17:41:31.458413+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:41:31.458413+0000","last_undegraded":"2024-08-17T17:41:31.458413+0000","last_fullsized":"2024-08-17T17:41:31.458413+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":607,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457894+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:41:31.457894+0000","last_peered":"2024-08-17T17:41:31.457894+0000","last_clean":"2024-08-17T17:41:31.457894+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:41:31.457894+0000","last_undegraded":"2024-08-17T17:41:31.457894+0000","last_fullsized":"2024-08-17T17:41:31.457894+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"385'10356","reported_seq":24920,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:35.469889+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:41:35.469889+0000","last_peered":"2024-08-17T17:41:35.469889+0000","last_clean":"2024-08-17T17:41:35.469889+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:41:35.469889+0000","last_undegraded":"2024-08-17T17:41:35.469889+0000","last_fullsized":"2024-08-17T17:41:35.469889+0000","mapping_epoch":345,"log_start":"326'4072","ondisk_log_start":"326'4072","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265289728,"num_objects":4048,"num_object_clones":0,"num_object_copies":12144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":295,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523994+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:41:22.523994+0000","last_peered":"2024-08-17T17:41:22.523994+0000","last_clean":"2024-08-17T17:41:22.523994+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:41:22.523994+0000","last_undegraded":"2024-08-17T17:41:22.523994+0000","last_fullsized":"2024-08-17T17:41:22.523994+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":362,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.624478+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:41:31.624478+0000","last_peered":"2024-08-17T17:41:31.624478+0000","last_clean":"2024-08-17T17:41:31.624478+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:41:31.624478+0000","last_undegraded":"2024-08-17T17:41:31.624478+0000","last_fullsized":"2024-08-17T17:41:31.624478+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523499+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:41:22.523499+0000","last_peered":"2024-08-17T17:41:22.523499+0000","last_clean":"2024-08-17T17:41:22.523499+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:41:22.523499+0000","last_undegraded":"2024-08-17T17:41:22.523499+0000","last_fullsized":"2024-08-17T17:41:22.523499+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"385'10562","reported_seq":10734,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:35.432025+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:41:35.432025+0000","last_peered":"2024-08-17T17:41:35.432025+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:41:35.432025+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4287","ondisk_log_start":"326'4287","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276692992,"num_objects":4222,"num_object_clones":0,"num_object_copies":12666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10562,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4222},{"shards":"0,3","objects":-3026}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":637,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458091+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:41:31.458091+0000","last_peered":"2024-08-17T17:41:31.458091+0000","last_clean":"2024-08-17T17:41:31.458091+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:41:31.458091+0000","last_undegraded":"2024-08-17T17:41:31.458091+0000","last_fullsized":"2024-08-17T17:41:31.458091+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":605,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458938+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:41:31.458938+0000","last_peered":"2024-08-17T17:41:31.458938+0000","last_clean":"2024-08-17T17:41:31.458938+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:41:31.458938+0000","last_undegraded":"2024-08-17T17:41:31.458938+0000","last_fullsized":"2024-08-17T17:41:31.458938+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":623,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459512+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:41:31.459512+0000","last_peered":"2024-08-17T17:41:31.459512+0000","last_clean":"2024-08-17T17:41:31.459512+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:41:31.459512+0000","last_undegraded":"2024-08-17T17:41:31.459512+0000","last_fullsized":"2024-08-17T17:41:31.459512+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"385'10409","reported_seq":25091,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.895544+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:41:31.895544+0000","last_peered":"2024-08-17T17:41:31.895544+0000","last_clean":"2024-08-17T17:41:31.895544+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:41:31.895544+0000","last_undegraded":"2024-08-17T17:41:31.895544+0000","last_fullsized":"2024-08-17T17:41:31.895544+0000","mapping_epoch":347,"log_start":"326'3872","ondisk_log_start":"326'3872","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272957440,"num_objects":4165,"num_object_clones":0,"num_object_copies":12495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10409,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":295,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.625045+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:41:31.625045+0000","last_peered":"2024-08-17T17:41:31.625045+0000","last_clean":"2024-08-17T17:41:31.625045+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:41:31.625045+0000","last_undegraded":"2024-08-17T17:41:31.625045+0000","last_fullsized":"2024-08-17T17:41:31.625045+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":389,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.462510+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:41:22.462510+0000","last_peered":"2024-08-17T17:41:22.462510+0000","last_clean":"2024-08-17T17:41:22.462510+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:41:22.462510+0000","last_undegraded":"2024-08-17T17:41:22.462510+0000","last_fullsized":"2024-08-17T17:41:22.462510+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453934+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:41:22.453934+0000","last_peered":"2024-08-17T17:41:22.453934+0000","last_clean":"2024-08-17T17:41:22.453934+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:41:22.453934+0000","last_undegraded":"2024-08-17T17:41:22.453934+0000","last_fullsized":"2024-08-17T17:41:22.453934+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.456800+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:41:22.456800+0000","last_peered":"2024-08-17T17:41:22.456800+0000","last_clean":"2024-08-17T17:41:22.456800+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:41:22.456800+0000","last_undegraded":"2024-08-17T17:41:22.456800+0000","last_fullsized":"2024-08-17T17:41:22.456800+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":382,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454463+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:41:22.454463+0000","last_peered":"2024-08-17T17:41:22.454463+0000","last_clean":"2024-08-17T17:41:22.454463+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:41:22.454463+0000","last_undegraded":"2024-08-17T17:41:22.454463+0000","last_fullsized":"2024-08-17T17:41:22.454463+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":612,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457690+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:41:31.457690+0000","last_peered":"2024-08-17T17:41:31.457690+0000","last_clean":"2024-08-17T17:41:31.457690+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:41:31.457690+0000","last_undegraded":"2024-08-17T17:41:31.457690+0000","last_fullsized":"2024-08-17T17:41:31.457690+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":152,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460296+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:41:31.460296+0000","last_peered":"2024-08-17T17:41:31.460296+0000","last_clean":"2024-08-17T17:41:31.460296+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:41:31.460296+0000","last_undegraded":"2024-08-17T17:41:31.460296+0000","last_fullsized":"2024-08-17T17:41:31.460296+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523603+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:41:22.523603+0000","last_peered":"2024-08-17T17:41:22.523603+0000","last_clean":"2024-08-17T17:41:22.523603+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:41:22.523603+0000","last_undegraded":"2024-08-17T17:41:22.523603+0000","last_fullsized":"2024-08-17T17:41:22.523603+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":570,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459496+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:41:31.459496+0000","last_peered":"2024-08-17T17:41:31.459496+0000","last_clean":"2024-08-17T17:41:31.459496+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:41:31.459496+0000","last_undegraded":"2024-08-17T17:41:31.459496+0000","last_fullsized":"2024-08-17T17:41:31.459496+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":433,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.525398+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:41:22.525398+0000","last_peered":"2024-08-17T17:41:22.525398+0000","last_clean":"2024-08-17T17:41:22.525398+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:41:22.525398+0000","last_undegraded":"2024-08-17T17:41:22.525398+0000","last_fullsized":"2024-08-17T17:41:22.525398+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":564,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459402+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:41:31.459402+0000","last_peered":"2024-08-17T17:41:31.459402+0000","last_clean":"2024-08-17T17:41:31.459402+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:41:31.459402+0000","last_undegraded":"2024-08-17T17:41:31.459402+0000","last_fullsized":"2024-08-17T17:41:31.459402+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":356,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.462448+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:41:22.462448+0000","last_peered":"2024-08-17T17:41:22.462448+0000","last_clean":"2024-08-17T17:41:22.462448+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:41:22.462448+0000","last_undegraded":"2024-08-17T17:41:22.462448+0000","last_fullsized":"2024-08-17T17:41:22.462448+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":595,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459586+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:41:31.459586+0000","last_peered":"2024-08-17T17:41:31.459586+0000","last_clean":"2024-08-17T17:41:31.459586+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:41:31.459586+0000","last_undegraded":"2024-08-17T17:41:31.459586+0000","last_fullsized":"2024-08-17T17:41:31.459586+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":311,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457494+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:41:31.457494+0000","last_peered":"2024-08-17T17:41:31.457494+0000","last_clean":"2024-08-17T17:41:31.457494+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:41:31.457494+0000","last_undegraded":"2024-08-17T17:41:31.457494+0000","last_fullsized":"2024-08-17T17:41:31.457494+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":603,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458456+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:41:31.458456+0000","last_peered":"2024-08-17T17:41:31.458456+0000","last_clean":"2024-08-17T17:41:31.458456+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:41:31.458456+0000","last_undegraded":"2024-08-17T17:41:31.458456+0000","last_fullsized":"2024-08-17T17:41:31.458456+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":563,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460104+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:41:31.460104+0000","last_peered":"2024-08-17T17:41:31.460104+0000","last_clean":"2024-08-17T17:41:31.460104+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:41:31.460104+0000","last_undegraded":"2024-08-17T17:41:31.460104+0000","last_fullsized":"2024-08-17T17:41:31.460104+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":368,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454430+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:41:22.454430+0000","last_peered":"2024-08-17T17:41:22.454430+0000","last_clean":"2024-08-17T17:41:22.454430+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:41:22.454430+0000","last_undegraded":"2024-08-17T17:41:22.454430+0000","last_fullsized":"2024-08-17T17:41:22.454430+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":252,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458621+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:41:31.458621+0000","last_peered":"2024-08-17T17:41:31.458621+0000","last_clean":"2024-08-17T17:41:31.458621+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:41:31.458621+0000","last_undegraded":"2024-08-17T17:41:31.458621+0000","last_fullsized":"2024-08-17T17:41:31.458621+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":308,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.624569+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:41:31.624569+0000","last_peered":"2024-08-17T17:41:31.624569+0000","last_clean":"2024-08-17T17:41:31.624569+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:41:31.624569+0000","last_undegraded":"2024-08-17T17:41:31.624569+0000","last_fullsized":"2024-08-17T17:41:31.624569+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":376,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.456669+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:41:22.456669+0000","last_peered":"2024-08-17T17:41:22.456669+0000","last_clean":"2024-08-17T17:41:22.456669+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:41:22.456669+0000","last_undegraded":"2024-08-17T17:41:22.456669+0000","last_fullsized":"2024-08-17T17:41:22.456669+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":239,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454380+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:41:22.454380+0000","last_peered":"2024-08-17T17:41:22.454380+0000","last_clean":"2024-08-17T17:41:22.454380+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:41:22.454380+0000","last_undegraded":"2024-08-17T17:41:22.454380+0000","last_fullsized":"2024-08-17T17:41:22.454380+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":376,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454273+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:41:22.454273+0000","last_peered":"2024-08-17T17:41:22.454273+0000","last_clean":"2024-08-17T17:41:22.454273+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:41:22.454273+0000","last_undegraded":"2024-08-17T17:41:22.454273+0000","last_fullsized":"2024-08-17T17:41:22.454273+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":239,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454677+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:41:22.454677+0000","last_peered":"2024-08-17T17:41:22.454677+0000","last_clean":"2024-08-17T17:41:22.454677+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:41:22.454677+0000","last_undegraded":"2024-08-17T17:41:22.454677+0000","last_fullsized":"2024-08-17T17:41:22.454677+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":615,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459958+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:41:31.459958+0000","last_peered":"2024-08-17T17:41:31.459958+0000","last_clean":"2024-08-17T17:41:31.459958+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:41:31.459958+0000","last_undegraded":"2024-08-17T17:41:31.459958+0000","last_fullsized":"2024-08-17T17:41:31.459958+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":589,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458320+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:41:31.458320+0000","last_peered":"2024-08-17T17:41:31.458320+0000","last_clean":"2024-08-17T17:41:31.458320+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:41:31.458320+0000","last_undegraded":"2024-08-17T17:41:31.458320+0000","last_fullsized":"2024-08-17T17:41:31.458320+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453937+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:41:22.453937+0000","last_peered":"2024-08-17T17:41:22.453937+0000","last_clean":"2024-08-17T17:41:22.453937+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:41:22.453937+0000","last_undegraded":"2024-08-17T17:41:22.453937+0000","last_fullsized":"2024-08-17T17:41:22.453937+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":311,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.624788+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:41:31.624788+0000","last_peered":"2024-08-17T17:41:31.624788+0000","last_clean":"2024-08-17T17:41:31.624788+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:41:31.624788+0000","last_undegraded":"2024-08-17T17:41:31.624788+0000","last_fullsized":"2024-08-17T17:41:31.624788+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4404805656,"num_objects":67213,"num_object_clones":0,"num_object_copies":201639,"num_objects_missing_on_primary":4830,"num_objects_missing":4830,"num_objects_degraded":52760,"num_objects_misplaced":26764,"num_objects_unfound":0,"num_objects_dirty":67213,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169177,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29723,"num_bytes_recovered":1825374208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14942285824,"data_stored":14942273608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104496,"ondisk_log_size":104496,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170102,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463471,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2789548,"kb_used_data":1895840,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":90955604,"statfs":{"total":95995035648,"available":93138538496,"internally_reserved":0,"allocated":1941340160,"data_stored":1939239616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492146,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928908,"kb_used_data":541892,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92816244,"statfs":{"total":95995035648,"available":95043833856,"internally_reserved":0,"allocated":554897408,"data_stored":552819164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986737,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2258016,"kb_used_data":1072852,"kb_used_omap":1,"kb_used_meta":1185150,"kb_avail":91487136,"statfs":{"total":95995035648,"available":93682827264,"internally_reserved":0,"allocated":1098600448,"data_stored":1096510844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213593944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225565,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3090124,"kb_used_data":2148036,"kb_used_omap":4,"kb_used_meta":942075,"kb_avail":90655028,"statfs":{"total":95995035648,"available":92830748672,"internally_reserved":0,"allocated":2199588864,"data_stored":2197510918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":964685248},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075890,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533783,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2937552,"kb_used_data":1885832,"kb_used_omap":5,"kb_used_meta":1051706,"kb_avail":90807600,"statfs":{"total":95995035648,"available":92986982400,"internally_reserved":0,"allocated":1931091968,"data_stored":1929008700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1076947184},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089760,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2625064,"kb_used_data":1620000,"kb_used_omap":5,"kb_used_meta":1005050,"kb_avail":91120088,"statfs":{"total":95995035648,"available":93306970112,"internally_reserved":0,"allocated":1658880000,"data_stored":1656779515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1029171259},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1651965952,"data_stored":1651965952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1924075520,"data_stored":1924071448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2191065088,"data_stored":2191065088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1092091904,"data_stored":1092091904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":549388288,"data_stored":549388288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1934032896,"data_stored":1934032896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2260406272,"data_stored":2260402200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:37.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:37.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:37.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:38.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:38.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:38.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:38.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:38.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:38.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:38.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:38.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:38.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:38.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:38.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:38.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:38.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[106729]: from='client.47507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:38.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:38.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:38.597 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:38.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[103614]: from='client.47507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:38.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:38.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:38.598 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:38.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:38.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:38.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:38.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:38 smithi152 ceph-mon[93456]: from='client.47507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:38.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:38.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:38.753 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:38.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:38.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:38.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:38.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:38.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:39.005 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:41:39.005 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:41:39.005 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:41:39.005 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:41:39.005 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:41:39.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:39.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:39.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:39.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:39.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:39.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:39.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:39.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:39.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:39.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:39.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:39 smithi142 ceph-mon[106729]: pgmap v1457: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52763/201198 objects degraded (26.224%); 26727/201198 objects misplaced (13.284%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:39.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:39 smithi142 ceph-mon[103614]: pgmap v1457: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52763/201198 objects degraded (26.224%); 26727/201198 objects misplaced (13.284%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:39.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:39.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:39.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:39.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:39 smithi152 ceph-mon[93456]: pgmap v1457: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52763/201198 objects degraded (26.224%); 26727/201198 objects misplaced (13.284%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:41:39.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:39.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:41:39.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:39.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:39.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:40.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:40.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:40.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:40.185 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:40.213 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:41:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=cleanup t=2024-08-17T17:41:39.86891296Z level=info msg="Completed cleanup jobs" duration=11.310147ms 2024-08-17T17:41:40.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:40.255 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:40.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:40.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:40.476 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:40.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:40.485 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:41:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:41:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:40.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:40.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:40.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:40.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:40.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:40.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:40.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:40.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:40.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:40.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:40.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:40.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:41.101 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:41.123 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:41.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:41.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:41.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:41.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:41.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:41.481 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:41:41.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:41.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:41.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:41.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:41.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:41.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:41 smithi152 ceph-mon[93456]: pgmap v1458: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52680/200265 objects degraded (26.305%); 26538/200265 objects misplaced (13.251%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:41.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4289903610' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:41.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:41 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:41.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52680/200265 objects degraded (26.305%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:41.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:41.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[106729]: pgmap v1458: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52680/200265 objects degraded (26.305%); 26538/200265 objects misplaced (13.251%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4289903610' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52680/200265 objects degraded (26.305%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[103614]: pgmap v1458: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 52680/200265 objects degraded (26.305%); 26538/200265 objects misplaced (13.251%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4289903610' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52680/200265 objects degraded (26.305%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:41:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:41.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:41.869 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:41.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:41.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:41.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:42.080 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:41:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:41:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:41:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:41:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:56992 12.7 12:e742fdef:::benchmark_data_smithi142_261410_object56991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.010017+0000", 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 566.06770679500005, 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.337228266, 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 56992 2024-08-17T17:41:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010017+0000", 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010017+0000", 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010015+0000", 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010052+0000", 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7256999999999999e-05 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010065+0000", 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2224e-05 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010070+0000", 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2519999999999999e-06 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010156+0000", 2024-08-17T17:41:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6216000000000003e-05 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010273+0000", 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011677 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.010404+0000", 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013071099999999999 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.012461+0000", 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020577550000000001 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.017253+0000", 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047918869999999999 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347188+0000", 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.329934544 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347221+0000", 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3050999999999999e-05 2024-08-17T17:41:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347245+0000", 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4017999999999999e-05 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:57015 12.7 12:e52ffa75:::benchmark_data_smithi142_261410_object57014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.030200+0000", 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 566.047523351, 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.317115541, 2024-08-17T17:41:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 57015 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030200+0000", 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030200+0000", 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030198+0000", 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030237+0000", 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8411000000000002e-05 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030249+0000", 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2276000000000001e-05 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030255+0000", 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8830000000000003e-06 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030293+0000", 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8220000000000003e-05 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030374+0000", 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0674999999999996e-05 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.030495+0000", 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120895 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.031973+0000", 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014781639999999999 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.037942+0000", 2024-08-17T17:41:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059694730000000003 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347270+0000", 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30932789399999999 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347296+0000", 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5816000000000002e-05 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347316+0000", 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9386e-05 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:57019 12.d 12:bd37fd84:::benchmark_data_smithi142_261410_object57018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.033511+0000", 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 566.04421201800005, 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31647268200000001, 2024-08-17T17:41:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 57019 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033511+0000", 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033511+0000", 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033511+0000", 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033579+0000", 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8262999999999993e-05 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033589+0000", 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.011e-05 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033593+0000", 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3329999999999998e-06 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033639+0000", 2024-08-17T17:41:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6090000000000001e-05 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033757+0000", 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011773300000000001 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.033909+0000", 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015202999999999999 2024-08-17T17:41:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.035488+0000", 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001578795 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.037966+0000", 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024782789999999999 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.349892+0000", 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31192589100000001 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.349943+0000", 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1100000000000002e-05 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.349984+0000", 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0706e-05 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:57031 12.d 12:b18fed84:::benchmark_data_smithi142_261410_object57030:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.041710+0000", 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 566.03601343499997, 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.314622609, 2024-08-17T17:41:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 57031 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041710+0000", 2024-08-17T17:41:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041710+0000", 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041708+0000", 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041761+0000", 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3044999999999998e-05 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041773+0000", 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1956e-05 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041778+0000", 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3100000000000002e-06 2024-08-17T17:41:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041824+0000", 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6103000000000002e-05 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041984+0000", 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015979500000000001 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.042121+0000", 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013763800000000001 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.043902+0000", 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017812730000000001 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.048417+0000", 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045142369999999999 2024-08-17T17:41:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.356237+0000", 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30782047899999998 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.356288+0000", 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0695999999999997e-05 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.356333+0000", 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4651999999999999e-05 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:57032 12.7 12:e8fe13d2:::benchmark_data_smithi142_261410_object57031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.041788+0000", 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 566.03593515099999, 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30576635600000002, 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 57032 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041788+0000", 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041788+0000", 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041788+0000", 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:41:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041818+0000", 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0213e-05 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041825+0000", 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3560000000000004e-06 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041830+0000", 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6959999999999996e-06 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.041900+0000", 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9873e-05 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.042020+0000", 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120359 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.042113+0000", 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2782999999999996e-05 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044154+0000", 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020409590000000002 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.047421+0000", 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032672249999999999 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347515+0000", 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30009355399999998 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347538+0000", 2024-08-17T17:41:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2904e-05 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.347555+0000", 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6932999999999999e-05 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:57034 12.d 12:b4d8bcc2:::benchmark_data_smithi142_261410_object57033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2024-08-17T17:41:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:32:16.044593+0000", 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 566.03313028800005, 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31188506999999999, 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 57034 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044593+0000", 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044593+0000", 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044592+0000", 2024-08-17T17:41:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044646+0000", 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4014999999999997e-05 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044656+0000", 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0033e-05 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044661+0000", 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3669999999999996e-06 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044692+0000", 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0732999999999999e-05 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044765+0000", 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2504999999999998e-05 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.044839+0000", 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4370999999999997e-05 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.046276+0000", 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00143687 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.048503+0000", 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022267070000000001 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.356391+0000", 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30788816299999999 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.356438+0000", 2024-08-17T17:41:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6910000000000003e-05 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:32:16.356478+0000", 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0431e-05 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112418 12.7 12:e0d90dce:::benchmark_data_smithi142_261410_object112417:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.974584+0000", 2024-08-17T17:41:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 512.10313984200002, 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33876536400000001, 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112418 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974584+0000", 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974584+0000", 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974573+0000", 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.99999 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974616+0000", 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2945000000000003e-05 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974632+0000", 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5503999999999999e-05 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974638+0000", 2024-08-17T17:41:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.494e-06 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974691+0000", 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3108e-05 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974830+0000", 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000138307 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.974983+0000", 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015321099999999999 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.978617+0000", 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003634436 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.983082+0000", 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044643210000000003 2024-08-17T17:41:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.313275+0000", 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33019332000000001 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.313316+0000", 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1301999999999999e-05 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.313349+0000", 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2589999999999998e-05 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112439 12.7 12:eb8f05af:::benchmark_data_smithi142_261410_object112438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.993615+0000", 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 512.08410789000004, 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32641100000000001, 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112439 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993615+0000", 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993615+0000", 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993608+0000", 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999924 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993674+0000", 2024-08-17T17:41:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6631999999999995e-05 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993689+0000", 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4518999999999999e-05 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993694+0000", 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.631e-06 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993755+0000", 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0006000000000001e-05 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993896+0000", 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014139900000000001 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.994035+0000", 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013935900000000001 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.001058+0000", 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070223830000000001 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.001088+0000", 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0371000000000001e-05 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.319949+0000", 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31886114700000001 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.319992+0000", 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2595000000000001e-05 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320026+0000", 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4709999999999998e-05 2024-08-17T17:41:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112440 12.7 12:ece66962:::benchmark_data_smithi142_261410_object112439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.993941+0000", 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 512.08378214499999, 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33023244899999998, 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112440 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993941+0000", 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993941+0000", 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993940+0000", 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.993999+0000", 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8616999999999998e-05 2024-08-17T17:41:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.994010+0000", 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0913e-05 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.994015+0000", 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.921e-06 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997506+0000", 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034914730000000001 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997634+0000", 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000127477 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997774+0000", 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014007700000000001 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.001174+0000", 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033996460000000001 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.001698+0000", 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052442899999999998 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.324106+0000", 2024-08-17T17:41:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32240784099999997 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.324137+0000", 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1191e-05 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.324174+0000", 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6662999999999998e-05 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112442 12.d 12:b21bf28a:::benchmark_data_smithi142_261410_object112441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:09.997797+0000", 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 512.079926011, 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32233632099999998, 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112442 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997797+0000", 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997797+0000", 2024-08-17T17:41:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997796+0000", 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997859+0000", 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3088000000000003e-05 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997868+0000", 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1370000000000008e-06 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997875+0000", 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5640000000000002e-06 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.997912+0000", 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7555000000000002e-05 2024-08-17T17:41:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.998014+0000", 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010181199999999999 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:09.998105+0000", 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0750999999999999e-05 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.000879+0000", 2024-08-17T17:41:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027745920000000002 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.001986+0000", 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00110636 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320047+0000", 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31806123400000003 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320099+0000", 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2475999999999997e-05 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320134+0000", 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4221000000000003e-05 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112443 12.d 12:b86a3973:::benchmark_data_smithi142_261410_object112442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:10.004167+0000", 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 512.07355651499995, 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31609321600000001, 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112443 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004167+0000", 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004167+0000", 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004166+0000", 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004219+0000", 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3875999999999997e-05 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004231+0000", 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.128e-05 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004235+0000", 2024-08-17T17:41:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4800000000000003e-06 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004283+0000", 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7729999999999999e-05 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004424+0000", 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014103100000000001 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004585+0000", 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016059700000000001 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.007538+0000", 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029533979999999999 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.008478+0000", 2024-08-17T17:41:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00094011099999999998 2024-08-17T17:41:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320189+0000", 2024-08-17T17:41:42.161 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:41:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:41:41] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:41:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31171048400000001 2024-08-17T17:41:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320227+0000", 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8220999999999999e-05 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.320260+0000", 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3287000000000002e-05 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:112445 12.7 12:e2f9c7fc:::benchmark_data_smithi142_261410_object112444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:10.004394+0000", 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 512.07332924100001, 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31998142000000002, 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112445 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004394+0000", 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004394+0000", 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004394+0000", 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004456+0000", 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2257999999999999e-05 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004461+0000", 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6069999999999998e-06 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004466+0000", 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.69e-06 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004504+0000", 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8012999999999997e-05 2024-08-17T17:41:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004621+0000", 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011698500000000001 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.004752+0000", 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013056999999999999 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.008253+0000", 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035015480000000002 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.008566+0000", 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00031270700000000001 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.324322+0000", 2024-08-17T17:41:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31575637699999998 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.324346+0000", 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3601999999999999e-05 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:10.324376+0000", 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9626000000000001e-05 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124358 12.7 12:ee40e7c7:::benchmark_data_smithi142_261410_object6161:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.826122+0000", 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.251601596, 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37423901999999998, 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124358 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.826122+0000", 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.826122+0000", 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.826120+0000", 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.826133+0000", 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2696e-05 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.826148+0000", 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5038000000000001e-05 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.826154+0000", 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1179999999999997e-06 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.833907+0000", 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077531220000000003 2024-08-17T17:41:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.834079+0000", 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017126899999999999 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.834235+0000", 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015679299999999999 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.839090+0000", 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004854382 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.846377+0000", 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072872249999999996 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.195084+0000", 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.348707092 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.195149+0000", 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4491000000000003e-05 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.200361+0000", 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052122560000000002 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124361 12.d 12:ba55b179:::benchmark_data_smithi142_261410_object6164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.832100+0000", 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.24562291000001, 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37162414799999999, 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124361 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.832100+0000", 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.832100+0000", 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.832099+0000", 2024-08-17T17:41:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.832108+0000", 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4800000000000007e-06 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.832124+0000", 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5628999999999999e-05 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.832131+0000", 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1999999999999997e-06 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.836941+0000", 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048095630000000002 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.837327+0000", 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038612800000000001 2024-08-17T17:41:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.837737+0000", 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000410159 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.842616+0000", 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048790689999999998 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.848097+0000", 2024-08-17T17:41:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054810989999999997 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.200099+0000", 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35200150299999999 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.200167+0000", 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8262000000000005e-05 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.203725+0000", 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035575110000000002 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124374 12.d 12:b3038b28:::benchmark_data_smithi142_261410_object6177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.862372+0000", 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.21535184300001, 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.345699812, 2024-08-17T17:41:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124374 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862372+0000", 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862372+0000", 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862370+0000", 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862378+0000", 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5820000000000004e-06 2024-08-17T17:41:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862391+0000", 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2889e-05 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862397+0000", 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.827e-06 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862514+0000", 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116589 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862663+0000", 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148701 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.862797+0000", 2024-08-17T17:41:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134225 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.867657+0000", 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048598169999999998 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.871585+0000", 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039285179999999998 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.203806+0000", 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.332220348 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.203868+0000", 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2225999999999997e-05 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.208071+0000", 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004203549 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124379 12.7 12:ef120d2b:::benchmark_data_smithi142_261410_object6182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.881806+0000", 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.19591726300001, 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32336235200000002, 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124379 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881806+0000", 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881806+0000", 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881805+0000", 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881813+0000", 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3019999999999995e-06 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881829+0000", 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6586999999999999e-05 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881837+0000", 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3799999999999996e-06 2024-08-17T17:41:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881889+0000", 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2413000000000001e-05 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.882001+0000", 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011228800000000001 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.882106+0000", 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010457400000000001 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.885226+0000", 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031203950000000002 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.890933+0000", 2024-08-17T17:41:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057069640000000001 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.200408+0000", 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30947454299999999 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.200457+0000", 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9252999999999998e-05 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.205168+0000", 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047112329999999996 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124380 12.d 12:b411ac4b:::benchmark_data_smithi142_261410_object6183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.881858+0000", 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.19586567499999, 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33054249800000002, 2024-08-17T17:41:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124380 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881858+0000", 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881858+0000", 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881857+0000", 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881861+0000", 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.41e-06 2024-08-17T17:41:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881868+0000", 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4000000000000003e-06 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881872+0000", 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1799999999999998e-06 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.881913+0000", 2024-08-17T17:41:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0818999999999997e-05 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.882012+0000", 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8808999999999995e-05 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.882106+0000", 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4265000000000006e-05 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.888630+0000", 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065238650000000002 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.898006+0000", 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093764780000000006 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.208131+0000", 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31012473099999999 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.208197+0000", 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6006000000000005e-05 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.212400+0000", 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042030069999999999 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:124381 12.d 12:b035055e:::benchmark_data_smithi142_261410_object6184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:33:29.884987+0000", 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.19273649199999, 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33200826300000003, 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 124381 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.884987+0000", 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.884987+0000", 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.884985+0000", 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.884996+0000", 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1651000000000001e-05 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.885013+0000", 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.658e-05 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.885021+0000", 2024-08-17T17:41:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.469e-06 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.885068+0000", 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7852999999999997e-05 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.885231+0000", 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000162759 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.885408+0000", 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017707000000000001 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.888767+0000", 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033590859999999998 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:29.898051+0000", 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092834230000000007 2024-08-17T17:41:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.212466+0000", 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31441499899999997 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.212523+0000", 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7265000000000001e-05 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:33:30.216995+0000", 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044721530000000004 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:151753 12.0 e370/337 12:0476e471:::benchmark_data_smithi142_261410_object33556:head v 370'9486, mlcod=324'731)", 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:53.724531+0000", 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 168.353192699, 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010731508000000001, 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724531+0000", 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724531+0000", 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724530+0000", 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724537+0000", 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4000000000000003e-06 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724551+0000", 2024-08-17T17:41:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4076e-05 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724566+0000", 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5116000000000001e-05 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724644+0000", 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7929999999999994e-05 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.724670+0000", 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5701999999999999e-05 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.735232+0000", 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010561935999999999 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.735262+0000", 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0218000000000001e-05 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:151784 12.0 e370/337 12:092a7334:::benchmark_data_smithi142_261410_object33587:head v 370'9487, mlcod=324'731)", 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:53.826251+0000", 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 168.25147217, 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010667776, 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:41:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826251+0000", 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826251+0000", 2024-08-17T17:41:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826250+0000", 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826258+0000", 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2179999999999999e-06 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826274+0000", 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.552e-05 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826281+0000", 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.34e-06 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826328+0000", 2024-08-17T17:41:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7172000000000002e-05 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.826355+0000", 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6650000000000001e-05 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.836882+0000", 2024-08-17T17:41:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010526594 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:53.836919+0000", 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7234000000000003e-05 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:41:42.220 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:41:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:42.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:42.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:42.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:42.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:42.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:42.470 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:42.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:42.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:42.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:42.718 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:42 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:42.718 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:42 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:42.718 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:42 smithi142 ceph-mon[103614]: osdmap e386: 8 total, 8 up, 6 in 2024-08-17T17:41:42.719 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:42 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:42.719 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:42 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:42.719 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:42 smithi142 ceph-mon[106729]: osdmap e386: 8 total, 8 up, 6 in 2024-08-17T17:41:42.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:42.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:42 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:42 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:42 smithi152 ceph-mon[93456]: osdmap e386: 8 total, 8 up, 6 in 2024-08-17T17:41:42.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:42.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:42.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:42.952 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:41:42.953 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:42.961 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:41:42.961 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:42.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:42.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:42.988 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:41:42.988 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:43.002 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:43.008 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:41:43.009 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:43.025 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:41:43.025 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:43.040 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:41:43.040 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:43.055 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:41:43.055 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:43.071 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:41:43.072 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:43.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:43.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:43.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:43.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:43.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:41:43.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:43.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:43.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:43.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:43.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:43 smithi152 ceph-mon[93456]: pgmap v1460: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52680/199317 objects degraded (26.430%); 26500/199317 objects misplaced (13.295%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1772591660' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:43.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:43.610 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:43 smithi142 ceph-mon[106729]: pgmap v1460: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52680/199317 objects degraded (26.430%); 26500/199317 objects misplaced (13.295%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:43.610 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1772591660' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:43.610 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:43 smithi142 ceph-mon[103614]: pgmap v1460: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52680/199317 objects degraded (26.430%); 26500/199317 objects misplaced (13.295%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:43.610 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1772591660' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:43.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:43.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:43.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:43.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:43.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:43.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:43.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:43.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:44.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:44.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:44.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:44.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:44.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:44.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:44.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:44.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:44.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:44.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:44.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:44.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:44.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:44.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:41:44.812 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:44.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:44.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:44.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:44.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:45.052 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:45.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:45.083 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:45.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:45.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:45.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:45.338 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:45.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:45.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:45.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:45.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:45.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:45.729 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:45 smithi142 ceph-mon[106729]: pgmap v1461: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52649/199131 objects degraded (26.439%); 26438/199131 objects misplaced (13.277%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:45.729 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:45 smithi142 ceph-mon[103614]: pgmap v1461: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52649/199131 objects degraded (26.439%); 26438/199131 objects misplaced (13.277%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:45.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:45 smithi152 ceph-mon[93456]: pgmap v1461: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52649/199131 objects degraded (26.439%); 26438/199131 objects misplaced (13.277%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:41:45.833 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:45.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:45.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:45.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:45.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:45.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:41:46.020 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:46.021 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:41:46.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:46.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:46.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:46.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:46.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:46.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:46.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:46.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:46.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:46.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:46.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:46.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:46.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52649/199131 objects degraded (26.439%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:46.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:46.841 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52649/199131 objects degraded (26.439%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:46.841 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52649/199131 objects degraded (26.439%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:46.876 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1461,"stamp":"2024-08-17T17:41:44.347688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4348909541,"num_objects":66377,"num_object_clones":0,"num_object_copies":199131,"num_objects_missing_on_primary":4792,"num_objects_missing":4792,"num_objects_degraded":52649,"num_objects_misplaced":26438,"num_objects_unfound":0,"num_objects_dirty":66377,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":170317,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29914,"num_bytes_recovered":1837572750,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105165,"ondisk_log_size":105165,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14523388,"kb_used_data":9046148,"kb_used_omap":27,"kb_used_meta":5477156,"kb_avail":547947524,"statfs":{"total":575970213888,"available":561098264576,"internally_reserved":0,"allocated":9263255552,"data_stored":9250709875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5608608488},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,0,3,2,2,0,0,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-75759616,"num_objects":-1156,"num_object_clones":0,"num_object_copies":-3468,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-152,"num_objects_misplaced":-427,"num_objects_unfound":0,"num_objects_dirty":-1156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1156,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":216,"num_bytes_recovered":14155776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002832"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":613,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458547+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:41:31.458547+0000","last_peered":"2024-08-17T17:41:31.458547+0000","last_clean":"2024-08-17T17:41:31.458547+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:41:31.458547+0000","last_undegraded":"2024-08-17T17:41:31.458547+0000","last_fullsized":"2024-08-17T17:41:31.458547+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":94,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.456462+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:41:22.456462+0000","last_peered":"2024-08-17T17:41:22.456462+0000","last_clean":"2024-08-17T17:41:22.456462+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:41:22.456462+0000","last_undegraded":"2024-08-17T17:41:22.456462+0000","last_fullsized":"2024-08-17T17:41:22.456462+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":288,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523840+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:41:22.523840+0000","last_peered":"2024-08-17T17:41:22.523840+0000","last_clean":"2024-08-17T17:41:22.523840+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:41:22.523840+0000","last_undegraded":"2024-08-17T17:41:22.523840+0000","last_fullsized":"2024-08-17T17:41:22.523840+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":311,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459048+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:41:31.459048+0000","last_peered":"2024-08-17T17:41:31.459048+0000","last_clean":"2024-08-17T17:41:31.459048+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:41:31.459048+0000","last_undegraded":"2024-08-17T17:41:31.459048+0000","last_fullsized":"2024-08-17T17:41:31.459048+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":488,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458249+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:41:31.458249+0000","last_peered":"2024-08-17T17:41:31.458249+0000","last_clean":"2024-08-17T17:41:31.458249+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:41:31.458249+0000","last_undegraded":"2024-08-17T17:41:31.458249+0000","last_fullsized":"2024-08-17T17:41:31.458249+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":237,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453847+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:41:22.453847+0000","last_peered":"2024-08-17T17:41:22.453847+0000","last_clean":"2024-08-17T17:41:22.453847+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:41:22.453847+0000","last_undegraded":"2024-08-17T17:41:22.453847+0000","last_fullsized":"2024-08-17T17:41:22.453847+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":582,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457958+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:41:31.457958+0000","last_peered":"2024-08-17T17:41:31.457958+0000","last_clean":"2024-08-17T17:41:31.457958+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:41:31.457958+0000","last_undegraded":"2024-08-17T17:41:31.457958+0000","last_fullsized":"2024-08-17T17:41:31.457958+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":406,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.668353+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:41:31.668353+0000","last_peered":"2024-08-17T17:41:31.668353+0000","last_clean":"2024-08-17T17:41:31.668353+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:41:31.668353+0000","last_undegraded":"2024-08-17T17:41:31.668353+0000","last_fullsized":"2024-08-17T17:41:31.668353+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"385'10557","reported_seq":15606,"reported_epoch":385,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:41:39.205653+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:41:39.205653+0000","last_peered":"2024-08-17T17:41:39.205653+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:41:39.205653+0000","last_undegraded":"2024-08-17T17:41:39.205653+0000","last_fullsized":"2024-08-17T17:41:39.205653+0000","mapping_epoch":337,"log_start":"326'3663","ondisk_log_start":"326'3663","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6894,"log_dups_size":0,"ondisk_log_size":6894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271777792,"num_objects":4147,"num_object_clones":0,"num_object_copies":12441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4147,"num_objects_unfound":0,"num_objects_dirty":4147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10557,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2476,"num_bytes_recovered":162267136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4147}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":90,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.553493+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:41:22.553493+0000","last_peered":"2024-08-17T17:41:22.553493+0000","last_clean":"2024-08-17T17:41:22.553493+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:41:22.553493+0000","last_undegraded":"2024-08-17T17:41:22.553493+0000","last_fullsized":"2024-08-17T17:41:22.553493+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523502+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:41:22.523502+0000","last_peered":"2024-08-17T17:41:22.523502+0000","last_clean":"2024-08-17T17:41:22.523502+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:41:22.523502+0000","last_undegraded":"2024-08-17T17:41:22.523502+0000","last_fullsized":"2024-08-17T17:41:22.523502+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"386'10803","reported_seq":10887,"reported_epoch":386,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:42.144429+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:41:42.144429+0000","last_peered":"2024-08-17T17:41:42.144429+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:41:42.144429+0000","last_undegraded":"2024-08-17T17:41:42.144429+0000","last_fullsized":"2024-08-17T17:41:42.144429+0000","mapping_epoch":337,"log_start":"326'4182","ondisk_log_start":"326'4182","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279904256,"num_objects":4271,"num_object_clones":0,"num_object_copies":12813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4271,"num_objects_unfound":0,"num_objects_dirty":4271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10803,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4271}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":92,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.455010+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:41:22.455010+0000","last_peered":"2024-08-17T17:41:22.455010+0000","last_clean":"2024-08-17T17:41:22.455010+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:41:22.455010+0000","last_undegraded":"2024-08-17T17:41:22.455010+0000","last_fullsized":"2024-08-17T17:41:22.455010+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":356,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453784+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:41:22.453784+0000","last_peered":"2024-08-17T17:41:22.453784+0000","last_clean":"2024-08-17T17:41:22.453784+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:41:22.453784+0000","last_undegraded":"2024-08-17T17:41:22.453784+0000","last_fullsized":"2024-08-17T17:41:22.453784+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"385'10567","reported_seq":10653,"reported_epoch":385,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:38.591681+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:41:38.591681+0000","last_peered":"2024-08-17T17:41:38.591681+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:41:38.591681+0000","last_undegraded":"2024-08-17T17:41:38.591681+0000","last_fullsized":"2024-08-17T17:41:38.591681+0000","mapping_epoch":337,"log_start":"326'3653","ondisk_log_start":"326'3653","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4167,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10567,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4167}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":88,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523766+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:41:22.523766+0000","last_peered":"2024-08-17T17:41:22.523766+0000","last_clean":"2024-08-17T17:41:22.523766+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:41:22.523766+0000","last_undegraded":"2024-08-17T17:41:22.523766+0000","last_fullsized":"2024-08-17T17:41:22.523766+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":367,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.667994+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:41:31.667994+0000","last_peered":"2024-08-17T17:41:31.667994+0000","last_clean":"2024-08-17T17:41:31.667994+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:41:31.667994+0000","last_undegraded":"2024-08-17T17:41:31.667994+0000","last_fullsized":"2024-08-17T17:41:31.667994+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"385'10781","reported_seq":10931,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:39.490749+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:41:39.490749+0000","last_peered":"2024-08-17T17:41:39.490749+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:41:39.490749+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'3713","ondisk_log_start":"326'3713","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273743872,"num_objects":4177,"num_object_clones":0,"num_object_copies":12531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10781,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2592},{"shards":"0,3,7","objects":4177}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":525,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458694+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:41:31.458694+0000","last_peered":"2024-08-17T17:41:31.458694+0000","last_clean":"2024-08-17T17:41:31.458694+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:41:31.458694+0000","last_undegraded":"2024-08-17T17:41:31.458694+0000","last_fullsized":"2024-08-17T17:41:31.458694+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":291,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.625994+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:41:31.625994+0000","last_peered":"2024-08-17T17:41:31.625994+0000","last_clean":"2024-08-17T17:41:31.625994+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:41:31.625994+0000","last_undegraded":"2024-08-17T17:41:31.625994+0000","last_fullsized":"2024-08-17T17:41:31.625994+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"386'10471","reported_seq":10546,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:42.007684+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:41:42.007684+0000","last_peered":"2024-08-17T17:41:42.007684+0000","last_clean":"2024-08-17T17:41:42.007684+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:41:42.007684+0000","last_undegraded":"2024-08-17T17:41:42.007684+0000","last_fullsized":"2024-08-17T17:41:42.007684+0000","mapping_epoch":319,"log_start":"326'3874","ondisk_log_start":"326'3874","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270204928,"num_objects":4123,"num_object_clones":0,"num_object_copies":12369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10471,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":614,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460344+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:41:31.460344+0000","last_peered":"2024-08-17T17:41:31.460344+0000","last_clean":"2024-08-17T17:41:31.460344+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:41:31.460344+0000","last_undegraded":"2024-08-17T17:41:31.460344+0000","last_fullsized":"2024-08-17T17:41:31.460344+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454309+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:41:22.454309+0000","last_peered":"2024-08-17T17:41:22.454309+0000","last_clean":"2024-08-17T17:41:22.454309+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:41:22.454309+0000","last_undegraded":"2024-08-17T17:41:22.454309+0000","last_fullsized":"2024-08-17T17:41:22.454309+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"385'10704","reported_seq":10789,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:41.462693+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:41:41.462693+0000","last_peered":"2024-08-17T17:41:41.462693+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:41:41.462693+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'4135","ondisk_log_start":"326'4135","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277086208,"num_objects":4228,"num_object_clones":0,"num_object_copies":12684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10704,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4228},{"shards":"1,6","objects":-3138}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":341,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.668127+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:41:31.668127+0000","last_peered":"2024-08-17T17:41:31.668127+0000","last_clean":"2024-08-17T17:41:31.668127+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:41:31.668127+0000","last_undegraded":"2024-08-17T17:41:31.668127+0000","last_fullsized":"2024-08-17T17:41:31.668127+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":366,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454789+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:41:22.454789+0000","last_peered":"2024-08-17T17:41:22.454789+0000","last_clean":"2024-08-17T17:41:22.454789+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:41:22.454789+0000","last_undegraded":"2024-08-17T17:41:22.454789+0000","last_fullsized":"2024-08-17T17:41:22.454789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"386'10829","reported_seq":10982,"reported_epoch":386,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:43.950376+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:41:43.950376+0000","last_peered":"2024-08-17T17:41:43.950376+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:41:43.950376+0000","last_undegraded":"2024-08-17T17:41:43.950376+0000","last_fullsized":"2024-08-17T17:41:43.950376+0000","mapping_epoch":337,"log_start":"327'5261","ondisk_log_start":"327'5261","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274006016,"num_objects":4181,"num_object_clones":0,"num_object_copies":12543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4181,"num_objects_unfound":0,"num_objects_dirty":4181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10829,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4181}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":377,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453997+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:41:22.453997+0000","last_peered":"2024-08-17T17:41:22.453997+0000","last_clean":"2024-08-17T17:41:22.453997+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:41:22.453997+0000","last_undegraded":"2024-08-17T17:41:22.453997+0000","last_fullsized":"2024-08-17T17:41:22.453997+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.455118+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:41:22.455118+0000","last_peered":"2024-08-17T17:41:22.455118+0000","last_clean":"2024-08-17T17:41:22.455118+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:41:22.455118+0000","last_undegraded":"2024-08-17T17:41:22.455118+0000","last_fullsized":"2024-08-17T17:41:22.455118+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"385'10607","reported_seq":15807,"reported_epoch":385,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:39.340831+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:41:39.340831+0000","last_peered":"2024-08-17T17:41:39.340831+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:41:39.340831+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:41:39.340831+0000","mapping_epoch":337,"log_start":"326'3741","ondisk_log_start":"326'3741","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268500992,"num_objects":4097,"num_object_clones":0,"num_object_copies":12291,"num_objects_missing_on_primary":4792,"num_objects_missing":4792,"num_objects_degraded":4097,"num_objects_misplaced":4792,"num_objects_unfound":0,"num_objects_dirty":4097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10607,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2550,"num_bytes_recovered":167051264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":10},{"shards":"1,2,7","objects":4086},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":584,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459673+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:41:31.459673+0000","last_peered":"2024-08-17T17:41:31.459673+0000","last_clean":"2024-08-17T17:41:31.459673+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:41:31.459673+0000","last_undegraded":"2024-08-17T17:41:31.459673+0000","last_fullsized":"2024-08-17T17:41:31.459673+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":521,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458000+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:41:31.458000+0000","last_peered":"2024-08-17T17:41:31.458000+0000","last_clean":"2024-08-17T17:41:31.458000+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:41:31.458000+0000","last_undegraded":"2024-08-17T17:41:31.458000+0000","last_fullsized":"2024-08-17T17:41:31.458000+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"386'10766","reported_seq":16044,"reported_epoch":386,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:43.345973+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:41:43.345973+0000","last_peered":"2024-08-17T17:41:43.345973+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:41:43.345973+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:41:43.345973+0000","mapping_epoch":337,"log_start":"327'5144","ondisk_log_start":"327'5144","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5622,"log_dups_size":0,"ondisk_log_size":5622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598168,"num_objects":4130,"num_object_clones":0,"num_object_copies":12390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4130,"num_objects_misplaced":4880,"num_objects_unfound":0,"num_objects_dirty":4130,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10766,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2568,"num_bytes_recovered":168230912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4130}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":714,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:31.624300+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:41:31.624300+0000","last_peered":"2024-08-17T17:41:31.624300+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:41:31.624300+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":314,"reported_epoch":384,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:41:22.523722+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:41:22.523722+0000","last_peered":"2024-08-17T17:41:22.523722+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:41:22.523722+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:41:22.523722+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":563,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458959+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:41:31.458959+0000","last_peered":"2024-08-17T17:41:31.458959+0000","last_clean":"2024-08-17T17:41:31.458959+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:41:31.458959+0000","last_undegraded":"2024-08-17T17:41:31.458959+0000","last_fullsized":"2024-08-17T17:41:31.458959+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":350,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.678475+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:41:31.678475+0000","last_peered":"2024-08-17T17:41:31.678475+0000","last_clean":"2024-08-17T17:41:31.678475+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:41:31.678475+0000","last_undegraded":"2024-08-17T17:41:31.678475+0000","last_fullsized":"2024-08-17T17:41:31.678475+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"385'10576","reported_seq":10729,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:39.452426+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:41:39.452426+0000","last_peered":"2024-08-17T17:41:39.452426+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:41:39.452426+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"325'3521","ondisk_log_start":"325'3521","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273154048,"num_objects":4168,"num_object_clones":0,"num_object_copies":12504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10576,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2479},{"shards":"0,4,7","objects":4168}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":382,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479567+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:41:41.479567+0000","last_peered":"2024-08-17T17:41:41.479567+0000","last_clean":"2024-08-17T17:41:41.479567+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:41:41.479567+0000","last_undegraded":"2024-08-17T17:41:41.479567+0000","last_fullsized":"2024-08-17T17:41:41.479567+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":321,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453673+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:41:22.453673+0000","last_peered":"2024-08-17T17:41:22.453673+0000","last_clean":"2024-08-17T17:41:22.453673+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:41:22.453673+0000","last_undegraded":"2024-08-17T17:41:22.453673+0000","last_fullsized":"2024-08-17T17:41:22.453673+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":305,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.625133+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:41:31.625133+0000","last_peered":"2024-08-17T17:41:31.625133+0000","last_clean":"2024-08-17T17:41:31.625133+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:41:31.625133+0000","last_undegraded":"2024-08-17T17:41:31.625133+0000","last_fullsized":"2024-08-17T17:41:31.625133+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"386'10617","reported_seq":10702,"reported_epoch":386,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:42.085113+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:41:42.085113+0000","last_peered":"2024-08-17T17:41:42.085113+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:41:42.085113+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4017","ondisk_log_start":"326'4017","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6600,"log_dups_size":0,"ondisk_log_size":6600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273874944,"num_objects":4179,"num_object_clones":0,"num_object_copies":12537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10617,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4179},{"shards":"5,6","objects":-3117}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":472,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460003+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:41:31.460003+0000","last_peered":"2024-08-17T17:41:31.460003+0000","last_clean":"2024-08-17T17:41:31.460003+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:41:31.460003+0000","last_undegraded":"2024-08-17T17:41:31.460003+0000","last_fullsized":"2024-08-17T17:41:31.460003+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.668043+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:41:31.668043+0000","last_peered":"2024-08-17T17:41:31.668043+0000","last_clean":"2024-08-17T17:41:31.668043+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:41:31.668043+0000","last_undegraded":"2024-08-17T17:41:31.668043+0000","last_fullsized":"2024-08-17T17:41:31.668043+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":658,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480310+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:41:41.480310+0000","last_peered":"2024-08-17T17:41:41.480310+0000","last_clean":"2024-08-17T17:41:41.480310+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:41:41.480310+0000","last_undegraded":"2024-08-17T17:41:41.480310+0000","last_fullsized":"2024-08-17T17:41:41.480310+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"385'10517","reported_seq":11394,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:39.327042+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:41:39.327042+0000","last_peered":"2024-08-17T17:41:39.327042+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:41:39.327042+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"325'3516","ondisk_log_start":"325'3516","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598144,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10517,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4129},{"shards":"0,5","objects":-2525}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":450,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479773+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:41:41.479773+0000","last_peered":"2024-08-17T17:41:41.479773+0000","last_clean":"2024-08-17T17:41:41.479773+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:41:41.479773+0000","last_undegraded":"2024-08-17T17:41:41.479773+0000","last_fullsized":"2024-08-17T17:41:41.479773+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":360,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.457040+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:41:22.457040+0000","last_peered":"2024-08-17T17:41:22.457040+0000","last_clean":"2024-08-17T17:41:22.457040+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:41:22.457040+0000","last_undegraded":"2024-08-17T17:41:22.457040+0000","last_fullsized":"2024-08-17T17:41:22.457040+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458507+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:41:31.458507+0000","last_peered":"2024-08-17T17:41:31.458507+0000","last_clean":"2024-08-17T17:41:31.458507+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:41:31.458507+0000","last_undegraded":"2024-08-17T17:41:31.458507+0000","last_fullsized":"2024-08-17T17:41:31.458507+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"385'10759","reported_seq":25755,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:40.381604+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:41:40.381604+0000","last_peered":"2024-08-17T17:41:40.381604+0000","last_clean":"2024-08-17T17:41:40.381604+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:41:40.381604+0000","last_undegraded":"2024-08-17T17:41:40.381604+0000","last_fullsized":"2024-08-17T17:41:40.381604+0000","mapping_epoch":355,"log_start":"326'4430","ondisk_log_start":"326'4430","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267190272,"num_objects":4077,"num_object_clones":0,"num_object_copies":12231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10759,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":299,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523673+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:41:22.523673+0000","last_peered":"2024-08-17T17:41:22.523673+0000","last_clean":"2024-08-17T17:41:22.523673+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:41:22.523673+0000","last_undegraded":"2024-08-17T17:41:22.523673+0000","last_fullsized":"2024-08-17T17:41:22.523673+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458413+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:41:31.458413+0000","last_peered":"2024-08-17T17:41:31.458413+0000","last_clean":"2024-08-17T17:41:31.458413+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:41:31.458413+0000","last_undegraded":"2024-08-17T17:41:31.458413+0000","last_fullsized":"2024-08-17T17:41:31.458413+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":609,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479954+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:41:41.479954+0000","last_peered":"2024-08-17T17:41:41.479954+0000","last_clean":"2024-08-17T17:41:41.479954+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:41:41.479954+0000","last_undegraded":"2024-08-17T17:41:41.479954+0000","last_fullsized":"2024-08-17T17:41:41.479954+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"385'10399","reported_seq":24963,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:40.339554+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:41:40.339554+0000","last_peered":"2024-08-17T17:41:40.339554+0000","last_clean":"2024-08-17T17:41:40.339554+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:41:40.339554+0000","last_undegraded":"2024-08-17T17:41:40.339554+0000","last_fullsized":"2024-08-17T17:41:40.339554+0000","mapping_epoch":345,"log_start":"326'4072","ondisk_log_start":"326'4072","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262471680,"num_objects":4005,"num_object_clones":0,"num_object_copies":12015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10399,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":295,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523994+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:41:22.523994+0000","last_peered":"2024-08-17T17:41:22.523994+0000","last_clean":"2024-08-17T17:41:22.523994+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:41:22.523994+0000","last_undegraded":"2024-08-17T17:41:22.523994+0000","last_fullsized":"2024-08-17T17:41:22.523994+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":362,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.624478+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:41:31.624478+0000","last_peered":"2024-08-17T17:41:31.624478+0000","last_clean":"2024-08-17T17:41:31.624478+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:41:31.624478+0000","last_undegraded":"2024-08-17T17:41:31.624478+0000","last_fullsized":"2024-08-17T17:41:31.624478+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523499+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:41:22.523499+0000","last_peered":"2024-08-17T17:41:22.523499+0000","last_clean":"2024-08-17T17:41:22.523499+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:41:22.523499+0000","last_undegraded":"2024-08-17T17:41:22.523499+0000","last_fullsized":"2024-08-17T17:41:22.523499+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"385'10600","reported_seq":10772,"reported_epoch":385,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:40.207069+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:41:40.207069+0000","last_peered":"2024-08-17T17:41:40.207069+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:41:40.207069+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4287","ondisk_log_start":"326'4287","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274202624,"num_objects":4184,"num_object_clones":0,"num_object_copies":12552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10600,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4184},{"shards":"0,3","objects":-3064}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":637,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458091+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:41:31.458091+0000","last_peered":"2024-08-17T17:41:31.458091+0000","last_clean":"2024-08-17T17:41:31.458091+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:41:31.458091+0000","last_undegraded":"2024-08-17T17:41:31.458091+0000","last_fullsized":"2024-08-17T17:41:31.458091+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":607,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480314+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:41:41.480314+0000","last_peered":"2024-08-17T17:41:41.480314+0000","last_clean":"2024-08-17T17:41:41.480314+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:41:41.480314+0000","last_undegraded":"2024-08-17T17:41:41.480314+0000","last_fullsized":"2024-08-17T17:41:41.480314+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":623,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459512+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:41:31.459512+0000","last_peered":"2024-08-17T17:41:31.459512+0000","last_clean":"2024-08-17T17:41:31.459512+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:41:31.459512+0000","last_undegraded":"2024-08-17T17:41:31.459512+0000","last_fullsized":"2024-08-17T17:41:31.459512+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"386'10486","reported_seq":25169,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.617691+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:41:41.617691+0000","last_peered":"2024-08-17T17:41:41.617691+0000","last_clean":"2024-08-17T17:41:41.617691+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:41:41.617691+0000","last_undegraded":"2024-08-17T17:41:41.617691+0000","last_fullsized":"2024-08-17T17:41:41.617691+0000","mapping_epoch":347,"log_start":"326'3872","ondisk_log_start":"326'3872","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267911168,"num_objects":4088,"num_object_clones":0,"num_object_copies":12264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10486,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":295,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.625045+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:41:31.625045+0000","last_peered":"2024-08-17T17:41:31.625045+0000","last_clean":"2024-08-17T17:41:31.625045+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:41:31.625045+0000","last_undegraded":"2024-08-17T17:41:31.625045+0000","last_fullsized":"2024-08-17T17:41:31.625045+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":390,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.668783+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:41:31.668783+0000","last_peered":"2024-08-17T17:41:31.668783+0000","last_clean":"2024-08-17T17:41:31.668783+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:41:31.668783+0000","last_undegraded":"2024-08-17T17:41:31.668783+0000","last_fullsized":"2024-08-17T17:41:31.668783+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453934+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:41:22.453934+0000","last_peered":"2024-08-17T17:41:22.453934+0000","last_clean":"2024-08-17T17:41:22.453934+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:41:22.453934+0000","last_undegraded":"2024-08-17T17:41:22.453934+0000","last_fullsized":"2024-08-17T17:41:22.453934+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.456800+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:41:22.456800+0000","last_peered":"2024-08-17T17:41:22.456800+0000","last_clean":"2024-08-17T17:41:22.456800+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:41:22.456800+0000","last_undegraded":"2024-08-17T17:41:22.456800+0000","last_fullsized":"2024-08-17T17:41:22.456800+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":382,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454463+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:41:22.454463+0000","last_peered":"2024-08-17T17:41:22.454463+0000","last_clean":"2024-08-17T17:41:22.454463+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:41:22.454463+0000","last_undegraded":"2024-08-17T17:41:22.454463+0000","last_fullsized":"2024-08-17T17:41:22.454463+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":614,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479774+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:41:41.479774+0000","last_peered":"2024-08-17T17:41:41.479774+0000","last_clean":"2024-08-17T17:41:41.479774+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:41:41.479774+0000","last_undegraded":"2024-08-17T17:41:41.479774+0000","last_fullsized":"2024-08-17T17:41:41.479774+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":152,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460296+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:41:31.460296+0000","last_peered":"2024-08-17T17:41:31.460296+0000","last_clean":"2024-08-17T17:41:31.460296+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:41:31.460296+0000","last_undegraded":"2024-08-17T17:41:31.460296+0000","last_fullsized":"2024-08-17T17:41:31.460296+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.523603+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:41:22.523603+0000","last_peered":"2024-08-17T17:41:22.523603+0000","last_clean":"2024-08-17T17:41:22.523603+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:41:22.523603+0000","last_undegraded":"2024-08-17T17:41:22.523603+0000","last_fullsized":"2024-08-17T17:41:22.523603+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":570,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459496+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:41:31.459496+0000","last_peered":"2024-08-17T17:41:31.459496+0000","last_clean":"2024-08-17T17:41:31.459496+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:41:31.459496+0000","last_undegraded":"2024-08-17T17:41:31.459496+0000","last_fullsized":"2024-08-17T17:41:31.459496+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":433,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.525398+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:41:22.525398+0000","last_peered":"2024-08-17T17:41:22.525398+0000","last_clean":"2024-08-17T17:41:22.525398+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:41:22.525398+0000","last_undegraded":"2024-08-17T17:41:22.525398+0000","last_fullsized":"2024-08-17T17:41:22.525398+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":564,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459402+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:41:31.459402+0000","last_peered":"2024-08-17T17:41:31.459402+0000","last_clean":"2024-08-17T17:41:31.459402+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:41:31.459402+0000","last_undegraded":"2024-08-17T17:41:31.459402+0000","last_fullsized":"2024-08-17T17:41:31.459402+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":357,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.668715+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:41:31.668715+0000","last_peered":"2024-08-17T17:41:31.668715+0000","last_clean":"2024-08-17T17:41:31.668715+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:41:31.668715+0000","last_undegraded":"2024-08-17T17:41:31.668715+0000","last_fullsized":"2024-08-17T17:41:31.668715+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":596,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.459626+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:41:31.459626+0000","last_peered":"2024-08-17T17:41:31.459626+0000","last_clean":"2024-08-17T17:41:31.459626+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:41:31.459626+0000","last_undegraded":"2024-08-17T17:41:31.459626+0000","last_fullsized":"2024-08-17T17:41:31.459626+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":312,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.457535+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:41:31.457535+0000","last_peered":"2024-08-17T17:41:31.457535+0000","last_clean":"2024-08-17T17:41:31.457535+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:41:31.457535+0000","last_undegraded":"2024-08-17T17:41:31.457535+0000","last_fullsized":"2024-08-17T17:41:31.457535+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":606,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479697+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:41:41.479697+0000","last_peered":"2024-08-17T17:41:41.479697+0000","last_clean":"2024-08-17T17:41:41.479697+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:41:41.479697+0000","last_undegraded":"2024-08-17T17:41:41.479697+0000","last_fullsized":"2024-08-17T17:41:41.479697+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":563,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460104+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:41:31.460104+0000","last_peered":"2024-08-17T17:41:31.460104+0000","last_clean":"2024-08-17T17:41:31.460104+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:41:31.460104+0000","last_undegraded":"2024-08-17T17:41:31.460104+0000","last_fullsized":"2024-08-17T17:41:31.460104+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":368,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454430+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:41:22.454430+0000","last_peered":"2024-08-17T17:41:22.454430+0000","last_clean":"2024-08-17T17:41:22.454430+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:41:22.454430+0000","last_undegraded":"2024-08-17T17:41:22.454430+0000","last_fullsized":"2024-08-17T17:41:22.454430+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":252,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458621+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:41:31.458621+0000","last_peered":"2024-08-17T17:41:31.458621+0000","last_clean":"2024-08-17T17:41:31.458621+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:41:31.458621+0000","last_undegraded":"2024-08-17T17:41:31.458621+0000","last_fullsized":"2024-08-17T17:41:31.458621+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":308,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.624569+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:41:31.624569+0000","last_peered":"2024-08-17T17:41:31.624569+0000","last_clean":"2024-08-17T17:41:31.624569+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:41:31.624569+0000","last_undegraded":"2024-08-17T17:41:31.624569+0000","last_fullsized":"2024-08-17T17:41:31.624569+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":376,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.456669+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:41:22.456669+0000","last_peered":"2024-08-17T17:41:22.456669+0000","last_clean":"2024-08-17T17:41:22.456669+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:41:22.456669+0000","last_undegraded":"2024-08-17T17:41:22.456669+0000","last_fullsized":"2024-08-17T17:41:22.456669+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":239,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454380+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:41:22.454380+0000","last_peered":"2024-08-17T17:41:22.454380+0000","last_clean":"2024-08-17T17:41:22.454380+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:41:22.454380+0000","last_undegraded":"2024-08-17T17:41:22.454380+0000","last_fullsized":"2024-08-17T17:41:22.454380+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":376,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454273+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:41:22.454273+0000","last_peered":"2024-08-17T17:41:22.454273+0000","last_clean":"2024-08-17T17:41:22.454273+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:41:22.454273+0000","last_undegraded":"2024-08-17T17:41:22.454273+0000","last_fullsized":"2024-08-17T17:41:22.454273+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":239,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.454677+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:41:22.454677+0000","last_peered":"2024-08-17T17:41:22.454677+0000","last_clean":"2024-08-17T17:41:22.454677+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:41:22.454677+0000","last_undegraded":"2024-08-17T17:41:22.454677+0000","last_fullsized":"2024-08-17T17:41:22.454677+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":616,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.460007+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:41:31.460007+0000","last_peered":"2024-08-17T17:41:31.460007+0000","last_clean":"2024-08-17T17:41:31.460007+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:41:31.460007+0000","last_undegraded":"2024-08-17T17:41:31.460007+0000","last_fullsized":"2024-08-17T17:41:31.460007+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":589,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.458320+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:41:31.458320+0000","last_peered":"2024-08-17T17:41:31.458320+0000","last_clean":"2024-08-17T17:41:31.458320+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:41:31.458320+0000","last_undegraded":"2024-08-17T17:41:31.458320+0000","last_fullsized":"2024-08-17T17:41:31.458320+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":339,"reported_epoch":384,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.453937+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:41:22.453937+0000","last_peered":"2024-08-17T17:41:22.453937+0000","last_clean":"2024-08-17T17:41:22.453937+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:41:22.453937+0000","last_undegraded":"2024-08-17T17:41:22.453937+0000","last_fullsized":"2024-08-17T17:41:22.453937+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":311,"reported_epoch":385,"state":"active+clean","last_fresh":"2024-08-17T17:41:31.624788+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:41:31.624788+0000","last_peered":"2024-08-17T17:41:31.624788+0000","last_clean":"2024-08-17T17:41:31.624788+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:41:31.624788+0000","last_undegraded":"2024-08-17T17:41:31.624788+0000","last_fullsized":"2024-08-17T17:41:31.624788+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4348313624,"num_objects":66351,"num_object_clones":0,"num_object_copies":199053,"num_objects_missing_on_primary":4792,"num_objects_missing":4792,"num_objects_degraded":52646,"num_objects_misplaced":26438,"num_objects_unfound":0,"num_objects_dirty":66351,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170039,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29873,"num_bytes_recovered":1835204608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14811279360,"data_stored":14811267144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104958,"ondisk_log_size":104958,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170103,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463473,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2761468,"kb_used_data":1863600,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":90983684,"statfs":{"total":95995035648,"available":93167292416,"internally_reserved":0,"allocated":1908326400,"data_stored":1906218063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492147,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":924364,"kb_used_data":537348,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":92820788,"statfs":{"total":95995035648,"available":95048486912,"internally_reserved":0,"allocated":550244352,"data_stored":548166108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986738,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2248540,"kb_used_data":1063380,"kb_used_omap":1,"kb_used_meta":1185150,"kb_avail":91496612,"statfs":{"total":95995035648,"available":93692530688,"internally_reserved":0,"allocated":1088901120,"data_stored":1086811516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1213593944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3,2,2,0,0,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225566,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3072972,"kb_used_data":2126724,"kb_used_omap":4,"kb_used_meta":946235,"kb_avail":90672180,"statfs":{"total":95995035648,"available":92848312320,"internally_reserved":0,"allocated":2177765376,"data_stored":2175687430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":968945088},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075891,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533785,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2903524,"kb_used_data":1851800,"kb_used_omap":5,"kb_used_meta":1051706,"kb_avail":90841628,"statfs":{"total":95995035648,"available":93021827072,"internally_reserved":0,"allocated":1896243200,"data_stored":1894152139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1076947184},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089761,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2612520,"kb_used_data":1603296,"kb_used_omap":5,"kb_used_meta":1009210,"kb_avail":91132632,"statfs":{"total":95995035648,"available":93319815168,"internally_reserved":0,"allocated":1641775104,"data_stored":1639674619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1033431099},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1635319808,"data_stored":1635319808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1889996800,"data_stored":1889992728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2169962496,"data_stored":2169962496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1081868288,"data_stored":1081868288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":545193984,"data_stored":545193984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1901330432,"data_stored":1901330432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2248347648,"data_stored":2248343576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:46.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:41:46.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:46.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:46.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:47.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:47.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:47.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:47.209 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:47.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:47.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:47.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:47.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:47.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:47.562 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:47.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:47.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:47.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:47.785 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:47 smithi142 ceph-mon[106729]: from='client.47525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:47.786 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:47 smithi142 ceph-mon[106729]: pgmap v1462: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52609/198150 objects degraded (26.550%); 26329/198150 objects misplaced (13.287%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:47.786 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:47 smithi142 ceph-mon[103614]: from='client.47525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:47.786 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:47 smithi142 ceph-mon[103614]: pgmap v1462: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52609/198150 objects degraded (26.550%); 26329/198150 objects misplaced (13.287%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:47.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:47 smithi152 ceph-mon[93456]: from='client.47525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:47 smithi152 ceph-mon[93456]: pgmap v1462: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52609/198150 objects degraded (26.550%); 26329/198150 objects misplaced (13.287%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:47.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:47.834 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:47.893 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:47.934 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:47.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:47.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:48.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:48.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:48.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:48.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:48.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:48.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:48.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:48.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:48.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:48.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:48.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:48 smithi142 ceph-mon[106729]: pgmap v1463: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52609/197577 objects degraded (26.627%); 26287/197577 objects misplaced (13.305%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:48.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:48 smithi142 ceph-mon[103614]: pgmap v1463: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52609/197577 objects degraded (26.627%); 26287/197577 objects misplaced (13.305%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:48.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:48.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:48.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:48.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:48 smithi152 ceph-mon[93456]: pgmap v1463: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52609/197577 objects degraded (26.627%); 26287/197577 objects misplaced (13.305%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:41:48.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:48.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:48.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:48.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:48.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:48.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:49.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:49.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:49.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:49.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:49.215 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:49.308 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:49.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:49.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:49.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:49.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:49.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:49.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:49.497 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:41:49.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:49.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:49.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:49.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:49.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3682023042' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:49.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:49.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3682023042' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3682023042' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:49.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:49.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:49.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:50.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:50.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:50.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:50.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:50.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:50.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:50.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:50.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:50.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:50.492 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:41:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:41:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:50.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:50.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:50.689 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:50.689 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3682023042' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:50.689 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[106729]: osdmap e387: 8 total, 8 up, 6 in 2024-08-17T17:41:50.690 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[106729]: pgmap v1465: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 52520/196572 objects degraded (26.718%); 26076/196572 objects misplaced (13.265%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:41:50.690 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:50.690 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3682023042' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:50.690 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[103614]: osdmap e387: 8 total, 8 up, 6 in 2024-08-17T17:41:50.690 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:50 smithi142 ceph-mon[103614]: pgmap v1465: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 52520/196572 objects degraded (26.718%); 26076/196572 objects misplaced (13.265%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:41:50.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:50.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:50.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:50 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:50.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3682023042' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:50 smithi152 ceph-mon[93456]: osdmap e387: 8 total, 8 up, 6 in 2024-08-17T17:41:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:50 smithi152 ceph-mon[93456]: pgmap v1465: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 52520/196572 objects degraded (26.718%); 26076/196572 objects misplaced (13.265%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:41:50.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:50.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:50.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:50.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:51.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:51.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:51.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:51.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:51.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:51.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:51.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:51.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:51.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:51.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:51.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:51.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:51.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:51.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52520/196572 objects degraded (26.718%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:51.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52520/196572 objects degraded (26.718%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:51.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52520/196572 objects degraded (26.718%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:51.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:51.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:51.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:51.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:51.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:52.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:52.079 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:41:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:41:51] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:41:52.083 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:52.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:52.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:52.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:52.331 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:52.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:52.352 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:41:52.352 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:52.374 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:41:52.375 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:52.393 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:41:52.393 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:52.411 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:41:52.411 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:52.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:52.433 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:41:52.434 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:52.453 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:41:52.454 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:52.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:52.474 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:41:52.474 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:52.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:41:52.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:52.557 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:52.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2289622543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:52 smithi142 ceph-mon[106729]: pgmap v1466: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52520/195711 objects degraded (26.835%); 26051/195711 objects misplaced (13.311%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:41:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2289622543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:52 smithi142 ceph-mon[103614]: pgmap v1466: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52520/195711 objects degraded (26.835%); 26051/195711 objects misplaced (13.311%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:41:52.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:52.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:52.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:52.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2289622543' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:52 smithi152 ceph-mon[93456]: pgmap v1466: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 52520/195711 objects degraded (26.835%); 26051/195711 objects misplaced (13.311%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:41:52.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:52.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:52.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:52.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:53.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.074 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:53.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:53.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:53.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:53.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:53.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:53.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:53.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:53.450 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:53.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:53.573 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:53.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:53.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.697 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:53.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:53.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:53.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:41:53.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:53.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:53.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:53.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:53.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:54.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:54.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:54.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:54.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:54.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:54.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:54.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:54.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:54.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:54.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:54.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:54.737 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:54.737 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 52477/195468 objects degraded (26.847%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2195,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":387,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":65156,"data_bytes":4268890085,"bytes_used":14707064832,"bytes_avail":561263149056,"bytes_total":575970213888,"degraded_objects":52477,"degraded_total":195468,"degraded_ratio":0.26846849612212742,"misplaced_objects":25970,"misplaced_total":195468,"misplaced_ratio":0.13286062168743734,"recovering_objects_per_sec":24,"recovering_bytes_per_sec":1618299,"recovering_keys_per_sec":0,"num_objects_recovered":247,"num_bytes_recovered":16187392,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":122},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:41:54.770 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:41:54.771 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:41:54.771 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:41:54.771 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:41:54.798 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:54.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:54.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:54.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:54.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:54.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:55.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:55.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:55.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:55.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:55.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:55.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:55.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:55.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:55.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:55.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:55.564 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:55 smithi142 ceph-mon[106729]: pgmap v1467: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 52477/195468 objects degraded (26.847%); 25970/195468 objects misplaced (13.286%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:41:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3079542029' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:55 smithi142 ceph-mon[103614]: pgmap v1467: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 52477/195468 objects degraded (26.847%); 25970/195468 objects misplaced (13.286%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:41:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3079542029' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:55.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:55.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:41:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:55.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:55.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:55.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:55 smithi152 ceph-mon[93456]: pgmap v1467: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 52477/195468 objects degraded (26.847%); 25970/195468 objects misplaced (13.286%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:41:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3079542029' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:55.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:55.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:55.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:56.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:56.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:56.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:56.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:56.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:56.289 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:56.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:56.380 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:41:56.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:56.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:56.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:56.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:56.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52477/195468 objects degraded (26.847%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52477/195468 objects degraded (26.847%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:56.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:56.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:56.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:56.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52477/195468 objects degraded (26.847%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:41:56.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:56.935 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:56.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:56.974 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:56.977 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:56.978 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:57.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:57.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:57.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:57.183 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:57.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:57.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:57.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:57.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:57.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:57.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:57.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:57.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:57.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:57.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:57.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:57.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:57.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:57 smithi152 ceph-mon[93456]: pgmap v1468: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 52434/194292 objects degraded (26.987%); 25837/194292 objects misplaced (13.298%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:41:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:57 smithi142 ceph-mon[106729]: pgmap v1468: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 52434/194292 objects degraded (26.987%); 25837/194292 objects misplaced (13.298%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:41:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:57 smithi142 ceph-mon[103614]: pgmap v1468: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 52434/194292 objects degraded (26.987%); 25837/194292 objects misplaced (13.298%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:41:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:57.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:57.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:57.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:57.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:58.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:58.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:58.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:58.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:58.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:58.301 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:58.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:58.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:58.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:58.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:58.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:58.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:58.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:58.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:58.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:58.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:58.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:58.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:58.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:58.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:59.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:59.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:59.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:59.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:59.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:59.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:59.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:59.305 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:41:59.306 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:41:59.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:59.422 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:59.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:59.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:59.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:41:59 smithi142 ceph-mon[106729]: pgmap v1469: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52434/193521 objects degraded (27.095%); 25790/193521 objects misplaced (13.327%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:41:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:41:59 smithi142 ceph-mon[103614]: pgmap v1469: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52434/193521 objects degraded (27.095%); 25790/193521 objects misplaced (13.327%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:41:59.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:59.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:59.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:59.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:41:59.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:59.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:41:59 smithi152 ceph-mon[93456]: pgmap v1469: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52434/193521 objects degraded (27.095%); 25790/193521 objects misplaced (13.327%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:41:59.898 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:59.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:41:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:59.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:00.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:00.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:00.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:00.146 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:00.169 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:00.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:00.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:00.294 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:00.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:00.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:00.420 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:00.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:00.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:00.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:00.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:00 smithi152 ceph-mon[93456]: from='client.57046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:42:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:42:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:00.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:00 smithi142 ceph-mon[106729]: from='client.57046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:00.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:00 smithi142 ceph-mon[103614]: from='client.57046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:00.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:00.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:00.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:00.794 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:00.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:00.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:00.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:00.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:01.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.042 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:01.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:01.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:01.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:01.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:01.168 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1469,"stamp":"2024-08-17T17:41:58.351497+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4226357221,"num_objects":64507,"num_object_clones":0,"num_object_copies":193521,"num_objects_missing_on_primary":4671,"num_objects_missing":4671,"num_objects_degraded":52434,"num_objects_misplaced":25790,"num_objects_unfound":0,"num_objects_dirty":64507,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":172187,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30248,"num_bytes_recovered":1859461774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104835,"ondisk_log_size":104835,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14285396,"kb_used_data":8803940,"kb_used_omap":27,"kb_used_meta":5481316,"kb_avail":548185516,"statfs":{"total":575970213888,"available":561341968384,"internally_reserved":0,"allocated":9015234560,"data_stored":9002610953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5612868328},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1,0,1,1,2,4,4],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-88604672,"num_objects":-1352,"num_object_clones":0,"num_object_copies":-4056,"num_objects_missing_on_primary":-79,"num_objects_missing":-79,"num_objects_degraded":-175,"num_objects_misplaced":-497,"num_objects_unfound":0,"num_objects_dirty":-1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1352,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":259,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002658"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":617,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508227+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:41:49.508227+0000","last_peered":"2024-08-17T17:41:49.508227+0000","last_clean":"2024-08-17T17:41:49.508227+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:41:49.508227+0000","last_undegraded":"2024-08-17T17:41:49.508227+0000","last_fullsized":"2024-08-17T17:41:49.508227+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":96,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605990+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:41:41.605990+0000","last_peered":"2024-08-17T17:41:41.605990+0000","last_clean":"2024-08-17T17:41:41.605990+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:41:41.605990+0000","last_undegraded":"2024-08-17T17:41:41.605990+0000","last_fullsized":"2024-08-17T17:41:41.605990+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":290,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480323+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:41:41.480323+0000","last_peered":"2024-08-17T17:41:41.480323+0000","last_clean":"2024-08-17T17:41:41.480323+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:41:41.480323+0000","last_undegraded":"2024-08-17T17:41:41.480323+0000","last_fullsized":"2024-08-17T17:41:41.480323+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":315,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.511232+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:41:49.511232+0000","last_peered":"2024-08-17T17:41:49.511232+0000","last_clean":"2024-08-17T17:41:49.511232+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:41:49.511232+0000","last_undegraded":"2024-08-17T17:41:49.511232+0000","last_fullsized":"2024-08-17T17:41:49.511232+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":492,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507854+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:41:49.507854+0000","last_peered":"2024-08-17T17:41:49.507854+0000","last_clean":"2024-08-17T17:41:49.507854+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:41:49.507854+0000","last_undegraded":"2024-08-17T17:41:49.507854+0000","last_fullsized":"2024-08-17T17:41:49.507854+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":239,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605767+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:41:41.605767+0000","last_peered":"2024-08-17T17:41:41.605767+0000","last_clean":"2024-08-17T17:41:41.605767+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:41:41.605767+0000","last_undegraded":"2024-08-17T17:41:41.605767+0000","last_fullsized":"2024-08-17T17:41:41.605767+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":586,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507813+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:41:49.507813+0000","last_peered":"2024-08-17T17:41:49.507813+0000","last_clean":"2024-08-17T17:41:49.507813+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:41:49.507813+0000","last_undegraded":"2024-08-17T17:41:49.507813+0000","last_fullsized":"2024-08-17T17:41:49.507813+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":407,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.617821+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:41:41.617821+0000","last_peered":"2024-08-17T17:41:41.617821+0000","last_clean":"2024-08-17T17:41:41.617821+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:41:41.617821+0000","last_undegraded":"2024-08-17T17:41:41.617821+0000","last_fullsized":"2024-08-17T17:41:41.617821+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"387'10672","reported_seq":15963,"reported_epoch":387,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:41:54.503070+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:41:54.503070+0000","last_peered":"2024-08-17T17:41:54.503070+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:41:54.503070+0000","last_undegraded":"2024-08-17T17:41:54.503070+0000","last_fullsized":"2024-08-17T17:41:54.503070+0000","mapping_epoch":337,"log_start":"326'3763","ondisk_log_start":"326'3763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264241152,"num_objects":4032,"num_object_clones":0,"num_object_copies":12096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4027,"num_objects_unfound":0,"num_objects_dirty":4032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10672,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2596,"num_bytes_recovered":170131456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4027},{"shards":"4,6,7","objects":5}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":92,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.661752+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:41:41.661752+0000","last_peered":"2024-08-17T17:41:41.661752+0000","last_clean":"2024-08-17T17:41:41.661752+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:41:41.661752+0000","last_undegraded":"2024-08-17T17:41:41.661752+0000","last_fullsized":"2024-08-17T17:41:41.661752+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":273,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479940+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:41:41.479940+0000","last_peered":"2024-08-17T17:41:41.479940+0000","last_clean":"2024-08-17T17:41:41.479940+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:41:41.479940+0000","last_undegraded":"2024-08-17T17:41:41.479940+0000","last_fullsized":"2024-08-17T17:41:41.479940+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"387'10917","reported_seq":11002,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:56.884544+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:41:56.884544+0000","last_peered":"2024-08-17T17:41:56.884544+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:41:56.884544+0000","last_undegraded":"2024-08-17T17:41:56.884544+0000","last_fullsized":"2024-08-17T17:41:56.884544+0000","mapping_epoch":337,"log_start":"326'4382","ondisk_log_start":"326'4382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272433152,"num_objects":4157,"num_object_clones":0,"num_object_copies":12471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4157,"num_objects_unfound":0,"num_objects_dirty":4157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10917,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4157}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":94,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606330+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:41:41.606330+0000","last_peered":"2024-08-17T17:41:41.606330+0000","last_clean":"2024-08-17T17:41:41.606330+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:41:41.606330+0000","last_undegraded":"2024-08-17T17:41:41.606330+0000","last_fullsized":"2024-08-17T17:41:41.606330+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":358,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605732+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:41:41.605732+0000","last_peered":"2024-08-17T17:41:41.605732+0000","last_clean":"2024-08-17T17:41:41.605732+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:41:41.605732+0000","last_undegraded":"2024-08-17T17:41:41.605732+0000","last_fullsized":"2024-08-17T17:41:41.605732+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"387'10693","reported_seq":10781,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:54.477274+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:41:54.477274+0000","last_peered":"2024-08-17T17:41:54.477274+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:41:54.477274+0000","last_undegraded":"2024-08-17T17:41:54.477274+0000","last_fullsized":"2024-08-17T17:41:54.477274+0000","mapping_epoch":337,"log_start":"326'3853","ondisk_log_start":"326'3853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6840,"log_dups_size":0,"ondisk_log_size":6840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264830976,"num_objects":4041,"num_object_clones":0,"num_object_copies":12123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4041,"num_objects_unfound":0,"num_objects_dirty":4041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10693,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":4041}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":90,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480385+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:41:41.480385+0000","last_peered":"2024-08-17T17:41:41.480385+0000","last_clean":"2024-08-17T17:41:41.480385+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:41:41.480385+0000","last_undegraded":"2024-08-17T17:41:41.480385+0000","last_fullsized":"2024-08-17T17:41:41.480385+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":368,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.617821+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:41:41.617821+0000","last_peered":"2024-08-17T17:41:41.617821+0000","last_clean":"2024-08-17T17:41:41.617821+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:41:41.617821+0000","last_undegraded":"2024-08-17T17:41:41.617821+0000","last_fullsized":"2024-08-17T17:41:41.617821+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"387'10915","reported_seq":11069,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:54.540737+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:41:54.540737+0000","last_peered":"2024-08-17T17:41:54.540737+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:41:54.540737+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'3913","ondisk_log_start":"326'3913","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264962048,"num_objects":4043,"num_object_clones":0,"num_object_copies":12129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10915,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2726},{"shards":"0,3,7","objects":4043}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":529,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508374+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:41:49.508374+0000","last_peered":"2024-08-17T17:41:49.508374+0000","last_clean":"2024-08-17T17:41:49.508374+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:41:49.508374+0000","last_undegraded":"2024-08-17T17:41:49.508374+0000","last_fullsized":"2024-08-17T17:41:49.508374+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":295,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579581+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:41:49.579581+0000","last_peered":"2024-08-17T17:41:49.579581+0000","last_clean":"2024-08-17T17:41:49.579581+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:41:49.579581+0000","last_undegraded":"2024-08-17T17:41:49.579581+0000","last_fullsized":"2024-08-17T17:41:49.579581+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"387'10580","reported_seq":10656,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:57.065949+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:41:57.065949+0000","last_peered":"2024-08-17T17:41:57.065949+0000","last_clean":"2024-08-17T17:41:57.065949+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:41:57.065949+0000","last_undegraded":"2024-08-17T17:41:57.065949+0000","last_fullsized":"2024-08-17T17:41:57.065949+0000","mapping_epoch":319,"log_start":"326'3974","ondisk_log_start":"326'3974","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6606,"log_dups_size":0,"ondisk_log_size":6606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263061504,"num_objects":4014,"num_object_clones":0,"num_object_copies":12042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10580,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":619,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508489+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:41:49.508489+0000","last_peered":"2024-08-17T17:41:49.508489+0000","last_clean":"2024-08-17T17:41:49.508489+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:41:49.508489+0000","last_undegraded":"2024-08-17T17:41:49.508489+0000","last_fullsized":"2024-08-17T17:41:49.508489+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605984+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:41:41.605984+0000","last_peered":"2024-08-17T17:41:41.605984+0000","last_clean":"2024-08-17T17:41:41.605984+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:41:41.605984+0000","last_undegraded":"2024-08-17T17:41:41.605984+0000","last_fullsized":"2024-08-17T17:41:41.605984+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"387'10832","reported_seq":10918,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:57.013648+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:41:57.013648+0000","last_peered":"2024-08-17T17:41:57.013648+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:41:57.013648+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'4235","ondisk_log_start":"326'4235","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10832,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4100},{"shards":"1,6","objects":-3266}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":342,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.617956+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:41:41.617956+0000","last_peered":"2024-08-17T17:41:41.617956+0000","last_clean":"2024-08-17T17:41:41.617956+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:41:41.617956+0000","last_undegraded":"2024-08-17T17:41:41.617956+0000","last_fullsized":"2024-08-17T17:41:41.617956+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":368,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606517+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:41:41.606517+0000","last_peered":"2024-08-17T17:41:41.606517+0000","last_clean":"2024-08-17T17:41:41.606517+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:41:41.606517+0000","last_undegraded":"2024-08-17T17:41:41.606517+0000","last_fullsized":"2024-08-17T17:41:41.606517+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"387'10905","reported_seq":11060,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:41:53.391056+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:41:53.391056+0000","last_peered":"2024-08-17T17:41:53.391056+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:41:53.391056+0000","last_undegraded":"2024-08-17T17:41:53.391056+0000","last_fullsized":"2024-08-17T17:41:53.391056+0000","mapping_epoch":337,"log_start":"327'5261","ondisk_log_start":"327'5261","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269025280,"num_objects":4105,"num_object_clones":0,"num_object_copies":12315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4105,"num_objects_unfound":0,"num_objects_dirty":4105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10905,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4105}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":379,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605851+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:41:41.605851+0000","last_peered":"2024-08-17T17:41:41.605851+0000","last_clean":"2024-08-17T17:41:41.605851+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:41:41.605851+0000","last_undegraded":"2024-08-17T17:41:41.605851+0000","last_fullsized":"2024-08-17T17:41:41.605851+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606403+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:41:41.606403+0000","last_peered":"2024-08-17T17:41:41.606403+0000","last_clean":"2024-08-17T17:41:41.606403+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:41:41.606403+0000","last_undegraded":"2024-08-17T17:41:41.606403+0000","last_fullsized":"2024-08-17T17:41:41.606403+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"387'10731","reported_seq":16178,"reported_epoch":387,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:54.396212+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:41:54.396212+0000","last_peered":"2024-08-17T17:41:54.396212+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:41:54.396212+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:41:54.396212+0000","mapping_epoch":337,"log_start":"326'3841","ondisk_log_start":"326'3841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260374528,"num_objects":3973,"num_object_clones":0,"num_object_copies":11919,"num_objects_missing_on_primary":4671,"num_objects_missing":4671,"num_objects_degraded":3973,"num_objects_misplaced":4671,"num_objects_unfound":0,"num_objects_dirty":3973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10731,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2673,"num_bytes_recovered":175112192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":8},{"shards":"1,2,7","objects":3965}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":588,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510577+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:41:49.510577+0000","last_peered":"2024-08-17T17:41:49.510577+0000","last_clean":"2024-08-17T17:41:49.510577+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:41:49.510577+0000","last_undegraded":"2024-08-17T17:41:49.510577+0000","last_fullsized":"2024-08-17T17:41:49.510577+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":525,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508067+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:41:49.508067+0000","last_peered":"2024-08-17T17:41:49.508067+0000","last_clean":"2024-08-17T17:41:49.508067+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:41:49.508067+0000","last_undegraded":"2024-08-17T17:41:49.508067+0000","last_fullsized":"2024-08-17T17:41:49.508067+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"387'10857","reported_seq":16319,"reported_epoch":387,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:41:53.974429+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:41:53.974429+0000","last_peered":"2024-08-17T17:41:53.974429+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:41:53.974429+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:41:53.974429+0000","mapping_epoch":337,"log_start":"327'5244","ondisk_log_start":"327'5244","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264634392,"num_objects":4039,"num_object_clones":0,"num_object_copies":12117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4039,"num_objects_misplaced":4789,"num_objects_unfound":0,"num_objects_dirty":4039,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10857,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":174194688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4039}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":719,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:49.579302+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:41:49.579302+0000","last_peered":"2024-08-17T17:41:49.579302+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:41:49.579302+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":316,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:41:41.480208+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:41:41.480208+0000","last_peered":"2024-08-17T17:41:41.480208+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:41:41.480208+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:41:41.480208+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":568,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.511202+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:41:49.511202+0000","last_peered":"2024-08-17T17:41:49.511202+0000","last_clean":"2024-08-17T17:41:49.511202+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:41:49.511202+0000","last_undegraded":"2024-08-17T17:41:49.511202+0000","last_fullsized":"2024-08-17T17:41:49.511202+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":351,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.618355+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:41:41.618355+0000","last_peered":"2024-08-17T17:41:41.618355+0000","last_clean":"2024-08-17T17:41:41.618355+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:41:41.618355+0000","last_undegraded":"2024-08-17T17:41:41.618355+0000","last_fullsized":"2024-08-17T17:41:41.618355+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"387'10702","reported_seq":10859,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:54.548386+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:41:54.548386+0000","last_peered":"2024-08-17T17:41:54.548386+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:41:54.548386+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'3721","ondisk_log_start":"326'3721","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10702,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2605},{"shards":"0,4,7","objects":4042}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":384,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.499608+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:41:49.499608+0000","last_peered":"2024-08-17T17:41:49.499608+0000","last_clean":"2024-08-17T17:41:49.499608+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:41:49.499608+0000","last_undegraded":"2024-08-17T17:41:49.499608+0000","last_fullsized":"2024-08-17T17:41:49.499608+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":323,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605662+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:41:41.605662+0000","last_peered":"2024-08-17T17:41:41.605662+0000","last_clean":"2024-08-17T17:41:41.605662+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:41:41.605662+0000","last_undegraded":"2024-08-17T17:41:41.605662+0000","last_fullsized":"2024-08-17T17:41:41.605662+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":310,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579991+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:41:49.579991+0000","last_peered":"2024-08-17T17:41:49.579991+0000","last_clean":"2024-08-17T17:41:49.579991+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:41:49.579991+0000","last_undegraded":"2024-08-17T17:41:49.579991+0000","last_fullsized":"2024-08-17T17:41:49.579991+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"387'10754","reported_seq":10840,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:56.931432+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:41:56.931432+0000","last_peered":"2024-08-17T17:41:56.931432+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:41:56.931432+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4217","ondisk_log_start":"326'4217","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4042},{"shards":"5,6","objects":-3254}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":476,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508829+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:41:49.508829+0000","last_peered":"2024-08-17T17:41:49.508829+0000","last_clean":"2024-08-17T17:41:49.508829+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:41:49.508829+0000","last_undegraded":"2024-08-17T17:41:49.508829+0000","last_fullsized":"2024-08-17T17:41:49.508829+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.617649+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:41:41.617649+0000","last_peered":"2024-08-17T17:41:41.617649+0000","last_clean":"2024-08-17T17:41:41.617649+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:41:41.617649+0000","last_undegraded":"2024-08-17T17:41:41.617649+0000","last_fullsized":"2024-08-17T17:41:41.617649+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":660,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500436+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:41:49.500436+0000","last_peered":"2024-08-17T17:41:49.500436+0000","last_clean":"2024-08-17T17:41:49.500436+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:41:49.500436+0000","last_undegraded":"2024-08-17T17:41:49.500436+0000","last_fullsized":"2024-08-17T17:41:49.500436+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"387'10637","reported_seq":11518,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:54.556866+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:41:54.556866+0000","last_peered":"2024-08-17T17:41:54.556866+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:41:54.556866+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'3616","ondisk_log_start":"326'3616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262733824,"num_objects":4009,"num_object_clones":0,"num_object_copies":12027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10637,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":4009},{"shards":"0,5","objects":-2645}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":452,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500079+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:41:49.500079+0000","last_peered":"2024-08-17T17:41:49.500079+0000","last_clean":"2024-08-17T17:41:49.500079+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:41:49.500079+0000","last_undegraded":"2024-08-17T17:41:49.500079+0000","last_fullsized":"2024-08-17T17:41:49.500079+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":362,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606435+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:41:41.606435+0000","last_peered":"2024-08-17T17:41:41.606435+0000","last_clean":"2024-08-17T17:41:41.606435+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:41:41.606435+0000","last_undegraded":"2024-08-17T17:41:41.606435+0000","last_fullsized":"2024-08-17T17:41:41.606435+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508154+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:41:49.508154+0000","last_peered":"2024-08-17T17:41:49.508154+0000","last_clean":"2024-08-17T17:41:49.508154+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:41:49.508154+0000","last_undegraded":"2024-08-17T17:41:49.508154+0000","last_fullsized":"2024-08-17T17:41:49.508154+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"387'10874","reported_seq":25874,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:55.369841+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:41:55.369841+0000","last_peered":"2024-08-17T17:41:55.369841+0000","last_clean":"2024-08-17T17:41:55.369841+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:41:55.369841+0000","last_undegraded":"2024-08-17T17:41:55.369841+0000","last_fullsized":"2024-08-17T17:41:55.369841+0000","mapping_epoch":355,"log_start":"326'4530","ondisk_log_start":"326'4530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259653632,"num_objects":3962,"num_object_clones":0,"num_object_copies":11886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10874,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":301,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480202+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:41:41.480202+0000","last_peered":"2024-08-17T17:41:41.480202+0000","last_clean":"2024-08-17T17:41:41.480202+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:41:41.480202+0000","last_undegraded":"2024-08-17T17:41:41.480202+0000","last_fullsized":"2024-08-17T17:41:41.480202+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":115,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508074+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:41:49.508074+0000","last_peered":"2024-08-17T17:41:49.508074+0000","last_clean":"2024-08-17T17:41:49.508074+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:41:49.508074+0000","last_undegraded":"2024-08-17T17:41:49.508074+0000","last_fullsized":"2024-08-17T17:41:49.508074+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":611,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500399+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:41:49.500399+0000","last_peered":"2024-08-17T17:41:49.500399+0000","last_clean":"2024-08-17T17:41:49.500399+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:41:49.500399+0000","last_undegraded":"2024-08-17T17:41:49.500399+0000","last_fullsized":"2024-08-17T17:41:49.500399+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"387'10523","reported_seq":25091,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:55.388730+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:41:55.388730+0000","last_peered":"2024-08-17T17:41:55.388730+0000","last_clean":"2024-08-17T17:41:55.388730+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:41:55.388730+0000","last_undegraded":"2024-08-17T17:41:55.388730+0000","last_fullsized":"2024-08-17T17:41:55.388730+0000","mapping_epoch":345,"log_start":"326'4272","ondisk_log_start":"326'4272","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6251,"log_dups_size":0,"ondisk_log_size":6251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254345216,"num_objects":3881,"num_object_clones":0,"num_object_copies":11643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10523,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":297,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480291+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:41:41.480291+0000","last_peered":"2024-08-17T17:41:41.480291+0000","last_clean":"2024-08-17T17:41:41.480291+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:41:41.480291+0000","last_undegraded":"2024-08-17T17:41:41.480291+0000","last_fullsized":"2024-08-17T17:41:41.480291+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":366,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579690+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:41:49.579690+0000","last_peered":"2024-08-17T17:41:49.579690+0000","last_clean":"2024-08-17T17:41:49.579690+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:41:49.579690+0000","last_undegraded":"2024-08-17T17:41:49.579690+0000","last_fullsized":"2024-08-17T17:41:49.579690+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":439,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480095+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:41:41.480095+0000","last_peered":"2024-08-17T17:41:41.480095+0000","last_clean":"2024-08-17T17:41:41.480095+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:41:41.480095+0000","last_undegraded":"2024-08-17T17:41:41.480095+0000","last_fullsized":"2024-08-17T17:41:41.480095+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"387'10711","reported_seq":10887,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:55.396412+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:41:55.396412+0000","last_peered":"2024-08-17T17:41:55.396412+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:41:55.396412+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4387","ondisk_log_start":"326'4387","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6324,"log_dups_size":0,"ondisk_log_size":6324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266928128,"num_objects":4073,"num_object_clones":0,"num_object_copies":12219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10711,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4073},{"shards":"0,3","objects":-3175}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":641,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507945+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:41:49.507945+0000","last_peered":"2024-08-17T17:41:49.507945+0000","last_clean":"2024-08-17T17:41:49.507945+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:41:49.507945+0000","last_undegraded":"2024-08-17T17:41:49.507945+0000","last_fullsized":"2024-08-17T17:41:49.507945+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":609,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500711+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:41:49.500711+0000","last_peered":"2024-08-17T17:41:49.500711+0000","last_clean":"2024-08-17T17:41:49.500711+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:41:49.500711+0000","last_undegraded":"2024-08-17T17:41:49.500711+0000","last_fullsized":"2024-08-17T17:41:49.500711+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":627,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510377+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:41:49.510377+0000","last_peered":"2024-08-17T17:41:49.510377+0000","last_clean":"2024-08-17T17:41:49.510377+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:41:49.510377+0000","last_undegraded":"2024-08-17T17:41:49.510377+0000","last_fullsized":"2024-08-17T17:41:49.510377+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"387'10606","reported_seq":25290,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:56.957402+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:41:56.957402+0000","last_peered":"2024-08-17T17:41:56.957402+0000","last_clean":"2024-08-17T17:41:56.957402+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:41:56.957402+0000","last_undegraded":"2024-08-17T17:41:56.957402+0000","last_fullsized":"2024-08-17T17:41:56.957402+0000","mapping_epoch":347,"log_start":"326'4072","ondisk_log_start":"326'4072","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260046848,"num_objects":3968,"num_object_clones":0,"num_object_copies":11904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10606,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":300,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579916+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:41:49.579916+0000","last_peered":"2024-08-17T17:41:49.579916+0000","last_clean":"2024-08-17T17:41:49.579916+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:41:49.579916+0000","last_undegraded":"2024-08-17T17:41:49.579916+0000","last_fullsized":"2024-08-17T17:41:49.579916+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":391,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.623291+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:41:41.623291+0000","last_peered":"2024-08-17T17:41:41.623291+0000","last_clean":"2024-08-17T17:41:41.623291+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:41:41.623291+0000","last_undegraded":"2024-08-17T17:41:41.623291+0000","last_fullsized":"2024-08-17T17:41:41.623291+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605804+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:41:41.605804+0000","last_peered":"2024-08-17T17:41:41.605804+0000","last_clean":"2024-08-17T17:41:41.605804+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:41:41.605804+0000","last_undegraded":"2024-08-17T17:41:41.605804+0000","last_fullsized":"2024-08-17T17:41:41.605804+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606243+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:41:41.606243+0000","last_peered":"2024-08-17T17:41:41.606243+0000","last_clean":"2024-08-17T17:41:41.606243+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:41:41.606243+0000","last_undegraded":"2024-08-17T17:41:41.606243+0000","last_fullsized":"2024-08-17T17:41:41.606243+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":384,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606158+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:41:41.606158+0000","last_peered":"2024-08-17T17:41:41.606158+0000","last_clean":"2024-08-17T17:41:41.606158+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:41:41.606158+0000","last_undegraded":"2024-08-17T17:41:41.606158+0000","last_fullsized":"2024-08-17T17:41:41.606158+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":616,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500170+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:41:49.500170+0000","last_peered":"2024-08-17T17:41:49.500170+0000","last_clean":"2024-08-17T17:41:49.500170+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:41:49.500170+0000","last_undegraded":"2024-08-17T17:41:49.500170+0000","last_fullsized":"2024-08-17T17:41:49.500170+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":156,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510672+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:41:49.510672+0000","last_peered":"2024-08-17T17:41:49.510672+0000","last_clean":"2024-08-17T17:41:49.510672+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:41:49.510672+0000","last_undegraded":"2024-08-17T17:41:49.510672+0000","last_fullsized":"2024-08-17T17:41:49.510672+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":291,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480335+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:41:41.480335+0000","last_peered":"2024-08-17T17:41:41.480335+0000","last_clean":"2024-08-17T17:41:41.480335+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:41:41.480335+0000","last_undegraded":"2024-08-17T17:41:41.480335+0000","last_fullsized":"2024-08-17T17:41:41.480335+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":574,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508546+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:41:49.508546+0000","last_peered":"2024-08-17T17:41:49.508546+0000","last_clean":"2024-08-17T17:41:49.508546+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:41:49.508546+0000","last_undegraded":"2024-08-17T17:41:49.508546+0000","last_fullsized":"2024-08-17T17:41:49.508546+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":435,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480611+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:41:41.480611+0000","last_peered":"2024-08-17T17:41:41.480611+0000","last_clean":"2024-08-17T17:41:41.480611+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:41:41.480611+0000","last_undegraded":"2024-08-17T17:41:41.480611+0000","last_fullsized":"2024-08-17T17:41:41.480611+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":568,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507560+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:41:49.507560+0000","last_peered":"2024-08-17T17:41:49.507560+0000","last_clean":"2024-08-17T17:41:49.507560+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:41:49.507560+0000","last_undegraded":"2024-08-17T17:41:49.507560+0000","last_fullsized":"2024-08-17T17:41:49.507560+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":358,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.623205+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:41:41.623205+0000","last_peered":"2024-08-17T17:41:41.623205+0000","last_clean":"2024-08-17T17:41:41.623205+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:41:41.623205+0000","last_undegraded":"2024-08-17T17:41:41.623205+0000","last_fullsized":"2024-08-17T17:41:41.623205+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":600,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510473+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:41:49.510473+0000","last_peered":"2024-08-17T17:41:49.510473+0000","last_clean":"2024-08-17T17:41:49.510473+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:41:49.510473+0000","last_undegraded":"2024-08-17T17:41:49.510473+0000","last_fullsized":"2024-08-17T17:41:49.510473+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":316,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507771+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:41:49.507771+0000","last_peered":"2024-08-17T17:41:49.507771+0000","last_clean":"2024-08-17T17:41:49.507771+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:41:49.507771+0000","last_undegraded":"2024-08-17T17:41:49.507771+0000","last_fullsized":"2024-08-17T17:41:49.507771+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":608,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.499784+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:41:49.499784+0000","last_peered":"2024-08-17T17:41:49.499784+0000","last_clean":"2024-08-17T17:41:49.499784+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:41:49.499784+0000","last_undegraded":"2024-08-17T17:41:49.499784+0000","last_fullsized":"2024-08-17T17:41:49.499784+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":567,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510435+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:41:49.510435+0000","last_peered":"2024-08-17T17:41:49.510435+0000","last_clean":"2024-08-17T17:41:49.510435+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:41:49.510435+0000","last_undegraded":"2024-08-17T17:41:49.510435+0000","last_fullsized":"2024-08-17T17:41:49.510435+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":370,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606663+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:41:41.606663+0000","last_peered":"2024-08-17T17:41:41.606663+0000","last_clean":"2024-08-17T17:41:41.606663+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:41:41.606663+0000","last_undegraded":"2024-08-17T17:41:41.606663+0000","last_fullsized":"2024-08-17T17:41:41.606663+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":256,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508302+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:41:49.508302+0000","last_peered":"2024-08-17T17:41:49.508302+0000","last_clean":"2024-08-17T17:41:49.508302+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:41:49.508302+0000","last_undegraded":"2024-08-17T17:41:49.508302+0000","last_fullsized":"2024-08-17T17:41:49.508302+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":312,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579469+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:41:49.579469+0000","last_peered":"2024-08-17T17:41:49.579469+0000","last_clean":"2024-08-17T17:41:49.579469+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:41:49.579469+0000","last_undegraded":"2024-08-17T17:41:49.579469+0000","last_fullsized":"2024-08-17T17:41:49.579469+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":378,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606173+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:41:41.606173+0000","last_peered":"2024-08-17T17:41:41.606173+0000","last_clean":"2024-08-17T17:41:41.606173+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:41:41.606173+0000","last_undegraded":"2024-08-17T17:41:41.606173+0000","last_fullsized":"2024-08-17T17:41:41.606173+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":241,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606071+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:41:41.606071+0000","last_peered":"2024-08-17T17:41:41.606071+0000","last_clean":"2024-08-17T17:41:41.606071+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:41:41.606071+0000","last_undegraded":"2024-08-17T17:41:41.606071+0000","last_fullsized":"2024-08-17T17:41:41.606071+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":378,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606552+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:41:41.606552+0000","last_peered":"2024-08-17T17:41:41.606552+0000","last_clean":"2024-08-17T17:41:41.606552+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:41:41.606552+0000","last_undegraded":"2024-08-17T17:41:41.606552+0000","last_fullsized":"2024-08-17T17:41:41.606552+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":241,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606160+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:41:41.606160+0000","last_peered":"2024-08-17T17:41:41.606160+0000","last_clean":"2024-08-17T17:41:41.606160+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:41:41.606160+0000","last_undegraded":"2024-08-17T17:41:41.606160+0000","last_fullsized":"2024-08-17T17:41:41.606160+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":620,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508914+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:41:49.508914+0000","last_peered":"2024-08-17T17:41:49.508914+0000","last_clean":"2024-08-17T17:41:49.508914+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:41:49.508914+0000","last_undegraded":"2024-08-17T17:41:49.508914+0000","last_fullsized":"2024-08-17T17:41:49.508914+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":593,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507948+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:41:49.507948+0000","last_peered":"2024-08-17T17:41:49.507948+0000","last_clean":"2024-08-17T17:41:49.507948+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:41:49.507948+0000","last_undegraded":"2024-08-17T17:41:49.507948+0000","last_fullsized":"2024-08-17T17:41:49.507948+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605689+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:41:41.605689+0000","last_peered":"2024-08-17T17:41:41.605689+0000","last_clean":"2024-08-17T17:41:41.605689+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:41:41.605689+0000","last_undegraded":"2024-08-17T17:41:41.605689+0000","last_fullsized":"2024-08-17T17:41:41.605689+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":315,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579871+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:41:49.579871+0000","last_peered":"2024-08-17T17:41:49.579871+0000","last_clean":"2024-08-17T17:41:49.579871+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:41:49.579871+0000","last_undegraded":"2024-08-17T17:41:49.579871+0000","last_fullsized":"2024-08-17T17:41:49.579871+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4225761304,"num_objects":64481,"num_object_clones":0,"num_object_copies":193443,"num_objects_missing_on_primary":4671,"num_objects_missing":4671,"num_objects_degraded":52431,"num_objects_misplaced":25790,"num_objects_unfound":0,"num_objects_dirty":64481,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171909,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30207,"num_bytes_recovered":1857093632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14523641856,"data_stored":14523629640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104628,"ondisk_log_size":104628,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170106,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463476,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2711944,"kb_used_data":1809920,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":91033208,"statfs":{"total":95995035648,"available":93218004992,"internally_reserved":0,"allocated":1853358080,"data_stored":1851241950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492150,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":913452,"kb_used_data":522276,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92831700,"statfs":{"total":95995035648,"available":95059660800,"internally_reserved":0,"allocated":534810624,"data_stored":532716794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986741,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2221628,"kb_used_data":1032308,"kb_used_omap":1,"kb_used_meta":1189310,"kb_avail":91523524,"statfs":{"total":95995035648,"available":93720088576,"internally_reserved":0,"allocated":1057083392,"data_stored":1054978202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1217853784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2,4,4],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225569,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2999596,"kb_used_data":2065828,"kb_used_omap":4,"kb_used_meta":933755,"kb_avail":90745556,"statfs":{"total":95995035648,"available":92923449344,"internally_reserved":0,"allocated":2115407872,"data_stored":2113314340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":956165568},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075894,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533787,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2871344,"kb_used_data":1815464,"kb_used_omap":5,"kb_used_meta":1055866,"kb_avail":90873808,"statfs":{"total":95995035648,"available":93054779392,"internally_reserved":0,"allocated":1859035136,"data_stored":1856936282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1081207024},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089764,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2567432,"kb_used_data":1558144,"kb_used_omap":5,"kb_used_meta":1009210,"kb_avail":91177720,"statfs":{"total":95995035648,"available":93365985280,"internally_reserved":0,"allocated":1595539456,"data_stored":1593423385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1033431099},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1588854784,"data_stored":1588854784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1853034496,"data_stored":1853030424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2108686336,"data_stored":2108686336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1049493504,"data_stored":1049493504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":529727488,"data_stored":529727488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1845362688,"data_stored":1845362688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2209222656,"data_stored":2209218584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:01.267 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:01.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:01.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:01.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:01.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:01.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:01.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:01.609 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:01.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:01.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:01.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:01.708 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:01 smithi142 ceph-mon[106729]: pgmap v1470: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52338/192531 objects degraded (27.184%); 25574/192531 objects misplaced (13.283%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:01.708 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52338/192531 objects degraded (27.184%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:01.708 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:01 smithi142 ceph-mon[103614]: pgmap v1470: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52338/192531 objects degraded (27.184%); 25574/192531 objects misplaced (13.283%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:01.708 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52338/192531 objects degraded (27.184%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:01.760 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:01.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:01 smithi152 ceph-mon[93456]: pgmap v1470: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52338/192531 objects degraded (27.184%); 25574/192531 objects misplaced (13.283%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52338/192531 objects degraded (27.184%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:01.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:01.907 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:01.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:02.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:02.025 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:42:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:42:01] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:42:02.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:02.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:02.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:02.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:02.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:02.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:02.402 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:02.504 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:02.525 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:02.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:02.649 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:02.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:02.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:02.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:02.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:02.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:02.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:03.002 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:03.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:03.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:03.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:03.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:03.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:03.373 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:03.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:03.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:03.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:03.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:03.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:03.724 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:03 smithi142 ceph-mon[103614]: pgmap v1471: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 52338/191736 objects degraded (27.297%); 25546/191736 objects misplaced (13.324%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:42:03.725 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:03 smithi142 ceph-mon[106729]: pgmap v1471: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 52338/191736 objects degraded (27.297%); 25546/191736 objects misplaced (13.324%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:42:03.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:03.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:03.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:03 smithi152 ceph-mon[93456]: pgmap v1471: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 52338/191736 objects degraded (27.297%); 25546/191736 objects misplaced (13.324%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:42:03.841 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:42:03.841 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:42:03.841 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:42:03.841 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:42:03.841 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:42:03.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:03.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:03.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:03.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:03.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:04.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:04.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:04.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:04.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:04.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:04.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:04.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:04.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:04.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:04.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:04.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:04.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:04.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:04.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:04.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:04.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:04.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:04.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:04.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:04.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:05.014 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:05.033 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:42:05.033 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:05.052 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:42:05.052 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:05.073 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:42:05.073 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:05.092 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:42:05.092 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:05.111 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:42:05.111 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:05.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:05.129 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:42:05.129 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:05.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:05.146 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:42:05.146 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:05.163 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:42:05.164 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:05.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:05.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:05.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:05.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:05.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:05.523 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:05.624 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:05.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:05.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:05.769 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:05 smithi152 ceph-mon[93456]: pgmap v1472: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52306/191544 objects degraded (27.308%); 25482/191544 objects misplaced (13.303%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:42:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3883527992' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:05.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:05 smithi142 ceph-mon[106729]: pgmap v1472: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52306/191544 objects degraded (27.308%); 25482/191544 objects misplaced (13.303%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:42:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3883527992' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:05 smithi142 ceph-mon[103614]: pgmap v1472: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 52306/191544 objects degraded (27.308%); 25482/191544 objects misplaced (13.303%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:42:05.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3883527992' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:05.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:05.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:05.997 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:06.018 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:06.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:06.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:06.142 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:06.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:06.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:06.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:06.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:06.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:06.443 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:42:06.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:42:06.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:06.521 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:06.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:06.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:06.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:06.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:06.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:06.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:06.780 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/276370192' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:06.780 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52306/191544 objects degraded (27.308%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:06.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/276370192' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:06.780 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52306/191544 objects degraded (27.308%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/276370192' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52306/191544 objects degraded (27.308%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:06.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:06.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:06.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:06.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:06.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:07.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:07.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:07.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:07.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:07.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:07.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:07.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:07.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:07.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:07.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:07.624 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:07.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:07.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:07.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:07.753 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[106729]: pgmap v1473: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52262/190599 objects degraded (27.420%); 25375/190599 objects misplaced (13.313%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:07.753 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:07.753 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/276370192' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:07.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[106729]: osdmap e388: 8 total, 8 up, 6 in 2024-08-17T17:42:07.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[103614]: pgmap v1473: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52262/190599 objects degraded (27.420%); 25375/190599 objects misplaced (13.313%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:07.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:07.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/276370192' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:07.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:07 smithi142 ceph-mon[103614]: osdmap e388: 8 total, 8 up, 6 in 2024-08-17T17:42:07.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:07.812 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:07 smithi152 ceph-mon[93456]: pgmap v1473: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52262/190599 objects degraded (27.420%); 25375/190599 objects misplaced (13.313%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:07 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/276370192' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:07 smithi152 ceph-mon[93456]: osdmap e388: 8 total, 8 up, 6 in 2024-08-17T17:42:07.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:07.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:07.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:07.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:07.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:08.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:08.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:08.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:08.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:08.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:08.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:08.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:08.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:08.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:08.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:08.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:08.658 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:08.682 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:08.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:08.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:08.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:08.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:08.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:09.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:09.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:09.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:09.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:42:09.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:09.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:09.281 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:09.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:09.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:09.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:09.428 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:09.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:09.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:09.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:09.675 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:09 smithi142 ceph-mon[106729]: pgmap v1475: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52262/189951 objects degraded (27.513%); 25327/189951 objects misplaced (13.333%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:09.676 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:09 smithi142 ceph-mon[103614]: pgmap v1475: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52262/189951 objects degraded (27.513%); 25327/189951 objects misplaced (13.333%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:09.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:09.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:09.713 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:09.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:09.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:42:09.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:09 smithi152 ceph-mon[93456]: pgmap v1475: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 52262/189951 objects degraded (27.513%); 25327/189951 objects misplaced (13.333%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:09.870 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:42:09.871 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:42:09.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:09.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:09.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:10.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:10.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:10.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:10.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:10.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:10.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:10.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:10.301 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:10.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:10.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:10.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:10.491 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:42:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:42:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:10.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:10.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:10.653 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:10.677 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:10.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:10.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:10.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:10.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:10.927 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:10.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:11.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:11.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:11.155 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:11.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.177 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:11.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:11.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:11.301 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:11.402 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:11.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:11.526 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:11.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-mon[106729]: from='client.47567 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-mon[106729]: pgmap v1476: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 52159/188805 objects degraded (27.626%); 25078/188805 objects misplaced (13.282%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:42:11.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52159/188805 objects degraded (27.626%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:11.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-mon[103614]: from='client.47567 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:11.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-mon[103614]: pgmap v1476: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 52159/188805 objects degraded (27.626%); 25078/188805 objects misplaced (13.282%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:42:11.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52159/188805 objects degraded (27.626%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:11.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:11.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:11.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:11.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:11.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:11.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:11 smithi152 ceph-mon[93456]: from='client.47567 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:11 smithi152 ceph-mon[93456]: pgmap v1476: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 52159/188805 objects degraded (27.626%); 25078/188805 objects misplaced (13.282%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:42:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52159/188805 objects degraded (27.626%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:11.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:11.914 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:42:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:42:11] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:42:11.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:11.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:12.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:12.046 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:12.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:12.151 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:12.172 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:12.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:12.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:12.294 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:12.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:12.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:12.424 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:12.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:12.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:12.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:12.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:12.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:12.758 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1475,"stamp":"2024-08-17T17:42:08.354472+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4148369381,"num_objects":63317,"num_object_clones":0,"num_object_copies":189951,"num_objects_missing_on_primary":4582,"num_objects_missing":4582,"num_objects_degraded":52262,"num_objects_misplaced":25327,"num_objects_unfound":0,"num_objects_dirty":63317,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":173377,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30484,"num_bytes_recovered":1874928270,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105025,"ondisk_log_size":105025,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14115816,"kb_used_data":8646900,"kb_used_omap":27,"kb_used_meta":5468836,"kb_avail":548355096,"statfs":{"total":575970213888,"available":561515618304,"internally_reserved":0,"allocated":8854425600,"data_stored":8841794200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5600088808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,1,0,1,0,1,1,0,4,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-77987840,"num_objects":-1190,"num_object_clones":0,"num_object_copies":-3570,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":-172,"num_objects_misplaced":-463,"num_objects_unfound":0,"num_objects_dirty":-1190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1190,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":236,"num_bytes_recovered":15466496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002974"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":617,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508227+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:41:49.508227+0000","last_peered":"2024-08-17T17:41:49.508227+0000","last_clean":"2024-08-17T17:41:49.508227+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:41:49.508227+0000","last_undegraded":"2024-08-17T17:41:49.508227+0000","last_fullsized":"2024-08-17T17:41:49.508227+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":96,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605990+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:41:41.605990+0000","last_peered":"2024-08-17T17:41:41.605990+0000","last_clean":"2024-08-17T17:41:41.605990+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:41:41.605990+0000","last_undegraded":"2024-08-17T17:41:41.605990+0000","last_fullsized":"2024-08-17T17:41:41.605990+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":290,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480323+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:41:41.480323+0000","last_peered":"2024-08-17T17:41:41.480323+0000","last_clean":"2024-08-17T17:41:41.480323+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:41:41.480323+0000","last_undegraded":"2024-08-17T17:41:41.480323+0000","last_fullsized":"2024-08-17T17:41:41.480323+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":315,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.511232+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:41:49.511232+0000","last_peered":"2024-08-17T17:41:49.511232+0000","last_clean":"2024-08-17T17:41:49.511232+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:41:49.511232+0000","last_undegraded":"2024-08-17T17:41:49.511232+0000","last_fullsized":"2024-08-17T17:41:49.511232+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":492,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507854+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:41:49.507854+0000","last_peered":"2024-08-17T17:41:49.507854+0000","last_clean":"2024-08-17T17:41:49.507854+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:41:49.507854+0000","last_undegraded":"2024-08-17T17:41:49.507854+0000","last_fullsized":"2024-08-17T17:41:49.507854+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":239,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605767+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:41:41.605767+0000","last_peered":"2024-08-17T17:41:41.605767+0000","last_clean":"2024-08-17T17:41:41.605767+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:41:41.605767+0000","last_undegraded":"2024-08-17T17:41:41.605767+0000","last_fullsized":"2024-08-17T17:41:41.605767+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":586,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507813+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:41:49.507813+0000","last_peered":"2024-08-17T17:41:49.507813+0000","last_clean":"2024-08-17T17:41:49.507813+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:41:49.507813+0000","last_undegraded":"2024-08-17T17:41:49.507813+0000","last_fullsized":"2024-08-17T17:41:49.507813+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":408,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.503689+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:41:49.503689+0000","last_peered":"2024-08-17T17:41:49.503689+0000","last_clean":"2024-08-17T17:41:49.503689+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:41:49.503689+0000","last_undegraded":"2024-08-17T17:41:49.503689+0000","last_fullsized":"2024-08-17T17:41:49.503689+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"387'10740","reported_seq":16158,"reported_epoch":387,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:42:04.417555+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:42:04.417555+0000","last_peered":"2024-08-17T17:42:04.417555+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:42:04.417555+0000","last_undegraded":"2024-08-17T17:42:04.417555+0000","last_fullsized":"2024-08-17T17:42:04.417555+0000","mapping_epoch":337,"log_start":"326'3863","ondisk_log_start":"326'3863","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6877,"log_dups_size":0,"ondisk_log_size":6877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259784704,"num_objects":3964,"num_object_clones":0,"num_object_copies":11892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3964,"num_objects_unfound":0,"num_objects_dirty":3964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10740,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":174260224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3964}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":92,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.661752+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:41:41.661752+0000","last_peered":"2024-08-17T17:41:41.661752+0000","last_clean":"2024-08-17T17:41:41.661752+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:41:41.661752+0000","last_undegraded":"2024-08-17T17:41:41.661752+0000","last_fullsized":"2024-08-17T17:41:41.661752+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":273,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.479940+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:41:41.479940+0000","last_peered":"2024-08-17T17:41:41.479940+0000","last_clean":"2024-08-17T17:41:41.479940+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:41:41.479940+0000","last_undegraded":"2024-08-17T17:41:41.479940+0000","last_fullsized":"2024-08-17T17:41:41.479940+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"388'10993","reported_seq":11079,"reported_epoch":388,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:07.137092+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:42:07.137092+0000","last_peered":"2024-08-17T17:42:07.137092+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:42:07.137092+0000","last_undegraded":"2024-08-17T17:42:07.137092+0000","last_fullsized":"2024-08-17T17:42:07.137092+0000","mapping_epoch":337,"log_start":"326'4382","ondisk_log_start":"326'4382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267452416,"num_objects":4081,"num_object_clones":0,"num_object_copies":12243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4081,"num_objects_unfound":0,"num_objects_dirty":4081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10993,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4081}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":94,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606330+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:41:41.606330+0000","last_peered":"2024-08-17T17:41:41.606330+0000","last_clean":"2024-08-17T17:41:41.606330+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:41:41.606330+0000","last_undegraded":"2024-08-17T17:41:41.606330+0000","last_fullsized":"2024-08-17T17:41:41.606330+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":358,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605732+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:41:41.605732+0000","last_peered":"2024-08-17T17:41:41.605732+0000","last_clean":"2024-08-17T17:41:41.605732+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:41:41.605732+0000","last_undegraded":"2024-08-17T17:41:41.605732+0000","last_fullsized":"2024-08-17T17:41:41.605732+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"387'10767","reported_seq":10855,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:04.186053+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:42:04.186053+0000","last_peered":"2024-08-17T17:42:04.186053+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:42:04.186053+0000","last_undegraded":"2024-08-17T17:42:04.186053+0000","last_fullsized":"2024-08-17T17:42:04.186053+0000","mapping_epoch":337,"log_start":"326'3853","ondisk_log_start":"326'3853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259981312,"num_objects":3967,"num_object_clones":0,"num_object_copies":11901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3967,"num_objects_unfound":0,"num_objects_dirty":3967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10767,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3967}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":90,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480385+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:41:41.480385+0000","last_peered":"2024-08-17T17:41:41.480385+0000","last_clean":"2024-08-17T17:41:41.480385+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:41:41.480385+0000","last_undegraded":"2024-08-17T17:41:41.480385+0000","last_fullsized":"2024-08-17T17:41:41.480385+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":369,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.504105+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:41:49.504105+0000","last_peered":"2024-08-17T17:41:49.504105+0000","last_clean":"2024-08-17T17:41:49.504105+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:41:49.504105+0000","last_undegraded":"2024-08-17T17:41:49.504105+0000","last_fullsized":"2024-08-17T17:41:49.504105+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"387'10992","reported_seq":11147,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:04.960937+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:42:04.960937+0000","last_peered":"2024-08-17T17:42:04.960937+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:42:04.960937+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'4013","ondisk_log_start":"326'4013","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259915776,"num_objects":3966,"num_object_clones":0,"num_object_copies":11898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10992,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2803},{"shards":"0,3,7","objects":3966}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":529,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508374+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:41:49.508374+0000","last_peered":"2024-08-17T17:41:49.508374+0000","last_clean":"2024-08-17T17:41:49.508374+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:41:49.508374+0000","last_undegraded":"2024-08-17T17:41:49.508374+0000","last_fullsized":"2024-08-17T17:41:49.508374+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":295,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579581+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:41:49.579581+0000","last_peered":"2024-08-17T17:41:49.579581+0000","last_clean":"2024-08-17T17:41:49.579581+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:41:49.579581+0000","last_undegraded":"2024-08-17T17:41:49.579581+0000","last_fullsized":"2024-08-17T17:41:49.579581+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"388'10657","reported_seq":10734,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.847848+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:42:06.847848+0000","last_peered":"2024-08-17T17:42:06.847848+0000","last_clean":"2024-08-17T17:42:06.847848+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:42:06.847848+0000","last_undegraded":"2024-08-17T17:42:06.847848+0000","last_fullsized":"2024-08-17T17:42:06.847848+0000","mapping_epoch":319,"log_start":"326'4074","ondisk_log_start":"326'4074","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258015232,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10657,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":619,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508489+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:41:49.508489+0000","last_peered":"2024-08-17T17:41:49.508489+0000","last_clean":"2024-08-17T17:41:49.508489+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:41:49.508489+0000","last_undegraded":"2024-08-17T17:41:49.508489+0000","last_fullsized":"2024-08-17T17:41:49.508489+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605984+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:41:41.605984+0000","last_peered":"2024-08-17T17:41:41.605984+0000","last_clean":"2024-08-17T17:41:41.605984+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:41:41.605984+0000","last_undegraded":"2024-08-17T17:41:41.605984+0000","last_fullsized":"2024-08-17T17:41:41.605984+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"388'10902","reported_seq":10989,"reported_epoch":388,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:07.036085+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:42:07.036085+0000","last_peered":"2024-08-17T17:42:07.036085+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:42:07.036085+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'4335","ondisk_log_start":"326'4335","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264110080,"num_objects":4030,"num_object_clones":0,"num_object_copies":12090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10902,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4030},{"shards":"1,6","objects":-3336}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":343,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.504225+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:41:49.504225+0000","last_peered":"2024-08-17T17:41:49.504225+0000","last_clean":"2024-08-17T17:41:49.504225+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:41:49.504225+0000","last_undegraded":"2024-08-17T17:41:49.504225+0000","last_fullsized":"2024-08-17T17:41:49.504225+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":368,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606517+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:41:41.606517+0000","last_peered":"2024-08-17T17:41:41.606517+0000","last_clean":"2024-08-17T17:41:41.606517+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:41:41.606517+0000","last_undegraded":"2024-08-17T17:41:41.606517+0000","last_fullsized":"2024-08-17T17:41:41.606517+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"387'10977","reported_seq":11132,"reported_epoch":387,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:03.449964+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:42:03.449964+0000","last_peered":"2024-08-17T17:42:03.449964+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:42:03.449964+0000","last_undegraded":"2024-08-17T17:42:03.449964+0000","last_fullsized":"2024-08-17T17:42:03.449964+0000","mapping_epoch":337,"log_start":"327'5361","ondisk_log_start":"327'5361","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264306688,"num_objects":4033,"num_object_clones":0,"num_object_copies":12099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4033,"num_objects_unfound":0,"num_objects_dirty":4033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10977,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":4033}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":379,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605851+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:41:41.605851+0000","last_peered":"2024-08-17T17:41:41.605851+0000","last_clean":"2024-08-17T17:41:41.605851+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:41:41.605851+0000","last_undegraded":"2024-08-17T17:41:41.605851+0000","last_fullsized":"2024-08-17T17:41:41.605851+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606403+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:41:41.606403+0000","last_peered":"2024-08-17T17:41:41.606403+0000","last_clean":"2024-08-17T17:41:41.606403+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:41:41.606403+0000","last_undegraded":"2024-08-17T17:41:41.606403+0000","last_fullsized":"2024-08-17T17:41:41.606403+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"387'10815","reported_seq":16434,"reported_epoch":387,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:04.557732+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:42:04.557732+0000","last_peered":"2024-08-17T17:42:04.557732+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:42:04.557732+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:42:04.557732+0000","mapping_epoch":337,"log_start":"326'3941","ondisk_log_start":"326'3941","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254869504,"num_objects":3889,"num_object_clones":0,"num_object_copies":11667,"num_objects_missing_on_primary":4582,"num_objects_missing":4582,"num_objects_degraded":3889,"num_objects_misplaced":4582,"num_objects_unfound":0,"num_objects_dirty":3889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10815,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2757,"num_bytes_recovered":180617216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":3876}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":588,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510577+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:41:49.510577+0000","last_peered":"2024-08-17T17:41:49.510577+0000","last_clean":"2024-08-17T17:41:49.510577+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:41:49.510577+0000","last_undegraded":"2024-08-17T17:41:49.510577+0000","last_fullsized":"2024-08-17T17:41:49.510577+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":525,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508067+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:41:49.508067+0000","last_peered":"2024-08-17T17:41:49.508067+0000","last_clean":"2024-08-17T17:41:49.508067+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:41:49.508067+0000","last_undegraded":"2024-08-17T17:41:49.508067+0000","last_fullsized":"2024-08-17T17:41:49.508067+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"387'10945","reported_seq":16585,"reported_epoch":387,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:03.995272+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:42:03.995272+0000","last_peered":"2024-08-17T17:42:03.995272+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:42:03.995272+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:42:03.995272+0000","mapping_epoch":337,"log_start":"327'5344","ondisk_log_start":"327'5344","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5601,"log_dups_size":0,"ondisk_log_size":5601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258867224,"num_objects":3951,"num_object_clones":0,"num_object_copies":11853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3951,"num_objects_misplaced":4700,"num_objects_unfound":0,"num_objects_dirty":3951,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10945,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2748,"num_bytes_recovered":180027392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3950},{"shards":"1,3,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":719,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:41:49.579302+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:41:49.579302+0000","last_peered":"2024-08-17T17:41:49.579302+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:41:49.579302+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":316,"reported_epoch":386,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:41:41.480208+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:41:41.480208+0000","last_peered":"2024-08-17T17:41:41.480208+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:41:41.480208+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:41:41.480208+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":568,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.511202+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:41:49.511202+0000","last_peered":"2024-08-17T17:41:49.511202+0000","last_clean":"2024-08-17T17:41:49.511202+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:41:49.511202+0000","last_undegraded":"2024-08-17T17:41:49.511202+0000","last_fullsized":"2024-08-17T17:41:49.511202+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":352,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.505560+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:41:49.505560+0000","last_peered":"2024-08-17T17:41:49.505560+0000","last_clean":"2024-08-17T17:41:49.505560+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:41:49.505560+0000","last_undegraded":"2024-08-17T17:41:49.505560+0000","last_fullsized":"2024-08-17T17:41:49.505560+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"387'10768","reported_seq":10925,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:04.637323+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:42:04.637323+0000","last_peered":"2024-08-17T17:42:04.637323+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:42:04.637323+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'3721","ondisk_log_start":"326'3721","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7047,"log_dups_size":0,"ondisk_log_size":7047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260571136,"num_objects":3976,"num_object_clones":0,"num_object_copies":11928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10768,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2671},{"shards":"0,4,7","objects":3976}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":384,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.499608+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:41:49.499608+0000","last_peered":"2024-08-17T17:41:49.499608+0000","last_clean":"2024-08-17T17:41:49.499608+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:41:49.499608+0000","last_undegraded":"2024-08-17T17:41:49.499608+0000","last_fullsized":"2024-08-17T17:41:49.499608+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":323,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605662+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:41:41.605662+0000","last_peered":"2024-08-17T17:41:41.605662+0000","last_clean":"2024-08-17T17:41:41.605662+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:41:41.605662+0000","last_undegraded":"2024-08-17T17:41:41.605662+0000","last_fullsized":"2024-08-17T17:41:41.605662+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":310,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579991+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:41:49.579991+0000","last_peered":"2024-08-17T17:41:49.579991+0000","last_clean":"2024-08-17T17:41:49.579991+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:41:49.579991+0000","last_undegraded":"2024-08-17T17:41:49.579991+0000","last_fullsized":"2024-08-17T17:41:49.579991+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"388'10830","reported_seq":10917,"reported_epoch":388,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:07.124639+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:42:07.124639+0000","last_peered":"2024-08-17T17:42:07.124639+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:42:07.124639+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4217","ondisk_log_start":"326'4217","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259915776,"num_objects":3966,"num_object_clones":0,"num_object_copies":11898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10830,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3966},{"shards":"5,6","objects":-3330}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":476,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508829+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:41:49.508829+0000","last_peered":"2024-08-17T17:41:49.508829+0000","last_clean":"2024-08-17T17:41:49.508829+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:41:49.508829+0000","last_undegraded":"2024-08-17T17:41:49.508829+0000","last_fullsized":"2024-08-17T17:41:49.508829+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":124,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.503537+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:41:49.503537+0000","last_peered":"2024-08-17T17:41:49.503537+0000","last_clean":"2024-08-17T17:41:49.503537+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:41:49.503537+0000","last_undegraded":"2024-08-17T17:41:49.503537+0000","last_fullsized":"2024-08-17T17:41:49.503537+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":660,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500436+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:41:49.500436+0000","last_peered":"2024-08-17T17:41:49.500436+0000","last_clean":"2024-08-17T17:41:49.500436+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:41:49.500436+0000","last_undegraded":"2024-08-17T17:41:49.500436+0000","last_fullsized":"2024-08-17T17:41:49.500436+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"387'10729","reported_seq":11610,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:04.930395+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:42:04.930395+0000","last_peered":"2024-08-17T17:42:04.930395+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:42:04.930395+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'3716","ondisk_log_start":"326'3716","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256704512,"num_objects":3917,"num_object_clones":0,"num_object_copies":11751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10729,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3917},{"shards":"0,5","objects":-2737}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":452,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500079+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:41:49.500079+0000","last_peered":"2024-08-17T17:41:49.500079+0000","last_clean":"2024-08-17T17:41:49.500079+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:41:49.500079+0000","last_undegraded":"2024-08-17T17:41:49.500079+0000","last_fullsized":"2024-08-17T17:41:49.500079+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":362,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606435+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:41:41.606435+0000","last_peered":"2024-08-17T17:41:41.606435+0000","last_clean":"2024-08-17T17:41:41.606435+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:41:41.606435+0000","last_undegraded":"2024-08-17T17:41:41.606435+0000","last_fullsized":"2024-08-17T17:41:41.606435+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508154+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:41:49.508154+0000","last_peered":"2024-08-17T17:41:49.508154+0000","last_clean":"2024-08-17T17:41:49.508154+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:41:49.508154+0000","last_undegraded":"2024-08-17T17:41:49.508154+0000","last_fullsized":"2024-08-17T17:41:49.508154+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"387'10937","reported_seq":25938,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.482800+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:42:05.482800+0000","last_peered":"2024-08-17T17:42:05.482800+0000","last_clean":"2024-08-17T17:42:05.482800+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:42:05.482800+0000","last_undegraded":"2024-08-17T17:42:05.482800+0000","last_fullsized":"2024-08-17T17:42:05.482800+0000","mapping_epoch":355,"log_start":"326'4630","ondisk_log_start":"326'4630","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524864,"num_objects":3899,"num_object_clones":0,"num_object_copies":11697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10937,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":301,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480202+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:41:41.480202+0000","last_peered":"2024-08-17T17:41:41.480202+0000","last_clean":"2024-08-17T17:41:41.480202+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:41:41.480202+0000","last_undegraded":"2024-08-17T17:41:41.480202+0000","last_fullsized":"2024-08-17T17:41:41.480202+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":116,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508112+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:41:49.508112+0000","last_peered":"2024-08-17T17:41:49.508112+0000","last_clean":"2024-08-17T17:41:49.508112+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:41:49.508112+0000","last_undegraded":"2024-08-17T17:41:49.508112+0000","last_fullsized":"2024-08-17T17:41:49.508112+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":611,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500399+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:41:49.500399+0000","last_peered":"2024-08-17T17:41:49.500399+0000","last_clean":"2024-08-17T17:41:49.500399+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:41:49.500399+0000","last_undegraded":"2024-08-17T17:41:49.500399+0000","last_fullsized":"2024-08-17T17:41:49.500399+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"387'10587","reported_seq":25155,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:42:05.409686+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:42:05.409686+0000","last_peered":"2024-08-17T17:42:05.409686+0000","last_clean":"2024-08-17T17:42:05.409686+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:42:05.409686+0000","last_undegraded":"2024-08-17T17:42:05.409686+0000","last_fullsized":"2024-08-17T17:42:05.409686+0000","mapping_epoch":345,"log_start":"326'4272","ondisk_log_start":"326'4272","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6315,"log_dups_size":0,"ondisk_log_size":6315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250150912,"num_objects":3817,"num_object_clones":0,"num_object_copies":11451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10587,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":297,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480291+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:41:41.480291+0000","last_peered":"2024-08-17T17:41:41.480291+0000","last_clean":"2024-08-17T17:41:41.480291+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:41:41.480291+0000","last_undegraded":"2024-08-17T17:41:41.480291+0000","last_fullsized":"2024-08-17T17:41:41.480291+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":367,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579732+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:41:49.579732+0000","last_peered":"2024-08-17T17:41:49.579732+0000","last_clean":"2024-08-17T17:41:49.579732+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:41:49.579732+0000","last_undegraded":"2024-08-17T17:41:49.579732+0000","last_fullsized":"2024-08-17T17:41:49.579732+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":439,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480095+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:41:41.480095+0000","last_peered":"2024-08-17T17:41:41.480095+0000","last_clean":"2024-08-17T17:41:41.480095+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:41:41.480095+0000","last_undegraded":"2024-08-17T17:41:41.480095+0000","last_fullsized":"2024-08-17T17:41:41.480095+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"387'10777","reported_seq":10953,"reported_epoch":387,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:05.529090+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:42:05.529090+0000","last_peered":"2024-08-17T17:42:05.529090+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:42:05.529090+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4487","ondisk_log_start":"326'4487","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6290,"log_dups_size":0,"ondisk_log_size":6290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262602752,"num_objects":4007,"num_object_clones":0,"num_object_copies":12021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10777,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":4007},{"shards":"0,3","objects":-3241}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":641,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507945+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:41:49.507945+0000","last_peered":"2024-08-17T17:41:49.507945+0000","last_clean":"2024-08-17T17:41:49.507945+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:41:49.507945+0000","last_undegraded":"2024-08-17T17:41:49.507945+0000","last_fullsized":"2024-08-17T17:41:49.507945+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":609,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500711+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:41:49.500711+0000","last_peered":"2024-08-17T17:41:49.500711+0000","last_clean":"2024-08-17T17:41:49.500711+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:41:49.500711+0000","last_undegraded":"2024-08-17T17:41:49.500711+0000","last_fullsized":"2024-08-17T17:41:49.500711+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":627,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510377+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:41:49.510377+0000","last_peered":"2024-08-17T17:41:49.510377+0000","last_clean":"2024-08-17T17:41:49.510377+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:41:49.510377+0000","last_undegraded":"2024-08-17T17:41:49.510377+0000","last_fullsized":"2024-08-17T17:41:49.510377+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"388'10683","reported_seq":25368,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:07.014543+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:42:07.014543+0000","last_peered":"2024-08-17T17:42:07.014543+0000","last_clean":"2024-08-17T17:42:07.014543+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:42:07.014543+0000","last_undegraded":"2024-08-17T17:42:07.014543+0000","last_fullsized":"2024-08-17T17:42:07.014543+0000","mapping_epoch":347,"log_start":"326'4072","ondisk_log_start":"326'4072","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255000576,"num_objects":3891,"num_object_clones":0,"num_object_copies":11673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10683,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":300,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579916+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:41:49.579916+0000","last_peered":"2024-08-17T17:41:49.579916+0000","last_clean":"2024-08-17T17:41:49.579916+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:41:49.579916+0000","last_undegraded":"2024-08-17T17:41:49.579916+0000","last_fullsized":"2024-08-17T17:41:49.579916+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":392,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.504056+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:41:49.504056+0000","last_peered":"2024-08-17T17:41:49.504056+0000","last_clean":"2024-08-17T17:41:49.504056+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:41:49.504056+0000","last_undegraded":"2024-08-17T17:41:49.504056+0000","last_fullsized":"2024-08-17T17:41:49.504056+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605804+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:41:41.605804+0000","last_peered":"2024-08-17T17:41:41.605804+0000","last_clean":"2024-08-17T17:41:41.605804+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:41:41.605804+0000","last_undegraded":"2024-08-17T17:41:41.605804+0000","last_fullsized":"2024-08-17T17:41:41.605804+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606243+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:41:41.606243+0000","last_peered":"2024-08-17T17:41:41.606243+0000","last_clean":"2024-08-17T17:41:41.606243+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:41:41.606243+0000","last_undegraded":"2024-08-17T17:41:41.606243+0000","last_fullsized":"2024-08-17T17:41:41.606243+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":384,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606158+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:41:41.606158+0000","last_peered":"2024-08-17T17:41:41.606158+0000","last_clean":"2024-08-17T17:41:41.606158+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:41:41.606158+0000","last_undegraded":"2024-08-17T17:41:41.606158+0000","last_fullsized":"2024-08-17T17:41:41.606158+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":616,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.500170+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:41:49.500170+0000","last_peered":"2024-08-17T17:41:49.500170+0000","last_clean":"2024-08-17T17:41:49.500170+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:41:49.500170+0000","last_undegraded":"2024-08-17T17:41:49.500170+0000","last_fullsized":"2024-08-17T17:41:49.500170+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":156,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510672+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:41:49.510672+0000","last_peered":"2024-08-17T17:41:49.510672+0000","last_clean":"2024-08-17T17:41:49.510672+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:41:49.510672+0000","last_undegraded":"2024-08-17T17:41:49.510672+0000","last_fullsized":"2024-08-17T17:41:49.510672+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":291,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480335+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:41:41.480335+0000","last_peered":"2024-08-17T17:41:41.480335+0000","last_clean":"2024-08-17T17:41:41.480335+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:41:41.480335+0000","last_undegraded":"2024-08-17T17:41:41.480335+0000","last_fullsized":"2024-08-17T17:41:41.480335+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":574,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508546+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:41:49.508546+0000","last_peered":"2024-08-17T17:41:49.508546+0000","last_clean":"2024-08-17T17:41:49.508546+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:41:49.508546+0000","last_undegraded":"2024-08-17T17:41:49.508546+0000","last_fullsized":"2024-08-17T17:41:49.508546+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":435,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.480611+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:41:41.480611+0000","last_peered":"2024-08-17T17:41:41.480611+0000","last_clean":"2024-08-17T17:41:41.480611+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:41:41.480611+0000","last_undegraded":"2024-08-17T17:41:41.480611+0000","last_fullsized":"2024-08-17T17:41:41.480611+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":568,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507560+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:41:49.507560+0000","last_peered":"2024-08-17T17:41:49.507560+0000","last_clean":"2024-08-17T17:41:49.507560+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:41:49.507560+0000","last_undegraded":"2024-08-17T17:41:49.507560+0000","last_fullsized":"2024-08-17T17:41:49.507560+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":359,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.503991+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:41:49.503991+0000","last_peered":"2024-08-17T17:41:49.503991+0000","last_clean":"2024-08-17T17:41:49.503991+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:41:49.503991+0000","last_undegraded":"2024-08-17T17:41:49.503991+0000","last_fullsized":"2024-08-17T17:41:49.503991+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":600,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510473+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:41:49.510473+0000","last_peered":"2024-08-17T17:41:49.510473+0000","last_clean":"2024-08-17T17:41:49.510473+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:41:49.510473+0000","last_undegraded":"2024-08-17T17:41:49.510473+0000","last_fullsized":"2024-08-17T17:41:49.510473+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":316,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507771+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:41:49.507771+0000","last_peered":"2024-08-17T17:41:49.507771+0000","last_clean":"2024-08-17T17:41:49.507771+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:41:49.507771+0000","last_undegraded":"2024-08-17T17:41:49.507771+0000","last_fullsized":"2024-08-17T17:41:49.507771+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":608,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.499784+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:41:49.499784+0000","last_peered":"2024-08-17T17:41:49.499784+0000","last_clean":"2024-08-17T17:41:49.499784+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:41:49.499784+0000","last_undegraded":"2024-08-17T17:41:49.499784+0000","last_fullsized":"2024-08-17T17:41:49.499784+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":567,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.510435+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:41:49.510435+0000","last_peered":"2024-08-17T17:41:49.510435+0000","last_clean":"2024-08-17T17:41:49.510435+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:41:49.510435+0000","last_undegraded":"2024-08-17T17:41:49.510435+0000","last_fullsized":"2024-08-17T17:41:49.510435+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":370,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606663+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:41:41.606663+0000","last_peered":"2024-08-17T17:41:41.606663+0000","last_clean":"2024-08-17T17:41:41.606663+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:41:41.606663+0000","last_undegraded":"2024-08-17T17:41:41.606663+0000","last_fullsized":"2024-08-17T17:41:41.606663+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":256,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508302+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:41:49.508302+0000","last_peered":"2024-08-17T17:41:49.508302+0000","last_clean":"2024-08-17T17:41:49.508302+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:41:49.508302+0000","last_undegraded":"2024-08-17T17:41:49.508302+0000","last_fullsized":"2024-08-17T17:41:49.508302+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":312,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579469+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:41:49.579469+0000","last_peered":"2024-08-17T17:41:49.579469+0000","last_clean":"2024-08-17T17:41:49.579469+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:41:49.579469+0000","last_undegraded":"2024-08-17T17:41:49.579469+0000","last_fullsized":"2024-08-17T17:41:49.579469+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":378,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606173+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:41:41.606173+0000","last_peered":"2024-08-17T17:41:41.606173+0000","last_clean":"2024-08-17T17:41:41.606173+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:41:41.606173+0000","last_undegraded":"2024-08-17T17:41:41.606173+0000","last_fullsized":"2024-08-17T17:41:41.606173+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":241,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606071+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:41:41.606071+0000","last_peered":"2024-08-17T17:41:41.606071+0000","last_clean":"2024-08-17T17:41:41.606071+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:41:41.606071+0000","last_undegraded":"2024-08-17T17:41:41.606071+0000","last_fullsized":"2024-08-17T17:41:41.606071+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":378,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606552+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:41:41.606552+0000","last_peered":"2024-08-17T17:41:41.606552+0000","last_clean":"2024-08-17T17:41:41.606552+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:41:41.606552+0000","last_undegraded":"2024-08-17T17:41:41.606552+0000","last_fullsized":"2024-08-17T17:41:41.606552+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":241,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.606160+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:41:41.606160+0000","last_peered":"2024-08-17T17:41:41.606160+0000","last_clean":"2024-08-17T17:41:41.606160+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:41:41.606160+0000","last_undegraded":"2024-08-17T17:41:41.606160+0000","last_fullsized":"2024-08-17T17:41:41.606160+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":620,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.508914+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:41:49.508914+0000","last_peered":"2024-08-17T17:41:49.508914+0000","last_clean":"2024-08-17T17:41:49.508914+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:41:49.508914+0000","last_undegraded":"2024-08-17T17:41:49.508914+0000","last_fullsized":"2024-08-17T17:41:49.508914+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":593,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.507948+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:41:49.507948+0000","last_peered":"2024-08-17T17:41:49.507948+0000","last_clean":"2024-08-17T17:41:49.507948+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:41:49.507948+0000","last_undegraded":"2024-08-17T17:41:49.507948+0000","last_fullsized":"2024-08-17T17:41:49.507948+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":341,"reported_epoch":386,"state":"active+clean","last_fresh":"2024-08-17T17:41:41.605689+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:41:41.605689+0000","last_peered":"2024-08-17T17:41:41.605689+0000","last_clean":"2024-08-17T17:41:41.605689+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:41:41.605689+0000","last_undegraded":"2024-08-17T17:41:41.605689+0000","last_fullsized":"2024-08-17T17:41:41.605689+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":315,"reported_epoch":387,"state":"active+clean","last_fresh":"2024-08-17T17:41:49.579871+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:41:49.579871+0000","last_peered":"2024-08-17T17:41:49.579871+0000","last_clean":"2024-08-17T17:41:49.579871+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:41:49.579871+0000","last_undegraded":"2024-08-17T17:41:49.579871+0000","last_fullsized":"2024-08-17T17:41:49.579871+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4147773464,"num_objects":63291,"num_object_clones":0,"num_object_copies":189873,"num_objects_missing_on_primary":4582,"num_objects_missing":4582,"num_objects_degraded":52259,"num_objects_misplaced":25327,"num_objects_unfound":0,"num_objects_dirty":63291,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173099,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30443,"num_bytes_recovered":1872560128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14332997632,"data_stored":14332985416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104818,"ondisk_log_size":104818,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170108,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463478,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2666652,"kb_used_data":1777104,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":91078500,"statfs":{"total":95995035648,"available":93264384000,"internally_reserved":0,"allocated":1819754496,"data_stored":1817630573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492152,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902828,"kb_used_data":511652,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92842324,"statfs":{"total":95995035648,"available":95070539776,"internally_reserved":0,"allocated":523931648,"data_stored":521837818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986743,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2191292,"kb_used_data":1014452,"kb_used_omap":1,"kb_used_meta":1176830,"kb_avail":91553860,"statfs":{"total":95995035648,"available":93751152640,"internally_reserved":0,"allocated":1038798848,"data_stored":1036693658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1205074264},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,0,0,0,1,1,0,4,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225571,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2968108,"kb_used_data":2030180,"kb_used_omap":4,"kb_used_meta":937915,"kb_avail":90777044,"statfs":{"total":95995035648,"available":92955693056,"internally_reserved":0,"allocated":2078904320,"data_stored":2076810788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":960425408},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075896,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533789,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2842672,"kb_used_data":1782632,"kb_used_omap":5,"kb_used_meta":1060026,"kb_avail":90902480,"statfs":{"total":95995035648,"available":93084139520,"internally_reserved":0,"allocated":1825415168,"data_stored":1823316314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1085466864},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089766,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2544264,"kb_used_data":1530880,"kb_used_omap":5,"kb_used_meta":1013370,"kb_avail":91200888,"statfs":{"total":95995035648,"available":93389709312,"internally_reserved":0,"allocated":1567621120,"data_stored":1565505049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1037690939},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1560477696,"data_stored":1560477696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1819086848,"data_stored":1819082776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2070151168,"data_stored":2070151168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1031864320,"data_stored":1031864320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":518389760,"data_stored":518389760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1811283968,"data_stored":1811283968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2182483968,"data_stored":2182479896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:12.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:42:12.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:12.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:12.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:12.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:12.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:12.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:13.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:13.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:13.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:13.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:13.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:13.293 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:13.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:13.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:13.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:13.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:13.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:13.668 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:13.711 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:13 smithi142 ceph-mon[106729]: pgmap v1477: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52159/187659 objects degraded (27.795%); 25036/187659 objects misplaced (13.341%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:42:13.711 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:13 smithi142 ceph-mon[103614]: pgmap v1477: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52159/187659 objects degraded (27.795%); 25036/187659 objects misplaced (13.341%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:42:13.723 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:13.723 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:13.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:13.792 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:13.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:13 smithi152 ceph-mon[93456]: pgmap v1477: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 52159/187659 objects degraded (27.795%); 25036/187659 objects misplaced (13.341%); 1.6 MiB/s, 26 objects/s recovering 2024-08-17T17:42:13.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:13.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:13.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:14.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:14.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:14.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:14.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:14.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:14.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:14.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:14.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:14.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:14.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:14.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:14.526 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:42:14.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:14.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:14.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:14.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:14.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:14.791 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:14.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:14.916 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:14.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:15.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:15.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:15.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:15.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:15.163 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:15.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:15.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:15.390 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:15.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:15.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:15.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:15.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:15.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:15.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:15.732 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:15 smithi142 ceph-mon[106729]: pgmap v1478: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 52121/187413 objects degraded (27.811%); 24954/187413 objects misplaced (13.315%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:15.733 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:15 smithi142 ceph-mon[103614]: pgmap v1478: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 52121/187413 objects degraded (27.811%); 24954/187413 objects misplaced (13.315%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:15.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:15.774 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:15.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:15.807 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:15 smithi152 ceph-mon[93456]: pgmap v1478: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 52121/187413 objects degraded (27.811%); 24954/187413 objects misplaced (13.315%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:15.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:15.914 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:15.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:15.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:16.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:16.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:16.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:16.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:16.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:16.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:16.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:16.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:16.457 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:42:16.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:16.539 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 52121/187413 objects degraded (27.811%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:16.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1474261060' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:16.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 52121/187413 objects degraded (27.811%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:16.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1474261060' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:16.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:16.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:16.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:16.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:16.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:16.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 52121/187413 objects degraded (27.811%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1474261060' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:42:16.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:16.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:16.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:16.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:17.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:17.034 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:17.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:17.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:17.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:17.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:17.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:17.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:17.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:17.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:17.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:17.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:17.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[106729]: pgmap v1479: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 52077/186195 objects degraded (27.969%); 24825/186195 objects misplaced (13.333%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1474261060' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:17.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[106729]: osdmap e389: 8 total, 8 up, 6 in 2024-08-17T17:42:17.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[103614]: pgmap v1479: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 52077/186195 objects degraded (27.969%); 24825/186195 objects misplaced (13.333%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1474261060' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:17.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:17 smithi142 ceph-mon[103614]: osdmap e389: 8 total, 8 up, 6 in 2024-08-17T17:42:17.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:17.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:17.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:17.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:17.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:17 smithi152 ceph-mon[93456]: pgmap v1479: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 52077/186195 objects degraded (27.969%); 24825/186195 objects misplaced (13.333%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:17 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1474261060' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:42:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:17 smithi152 ceph-mon[93456]: osdmap e389: 8 total, 8 up, 6 in 2024-08-17T17:42:17.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:17.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:17.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:18.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:18.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:18.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:18.162 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:18.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:18.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:18.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:18.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:18.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:18.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:18.545 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:18.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:18.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:18.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:18.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:18.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:18.924 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:18.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:18.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:19.026 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:19.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:19.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:19.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:19.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:19.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:19.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:19.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:19.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:19.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:19.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:19.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:19.681 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:42:19.681 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:19.692 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:42:19.692 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:19.710 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:42:19.710 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:19.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:19 smithi142 ceph-mon[103614]: pgmap v1481: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 52077/185547 objects degraded (28.067%); 24771/185547 objects misplaced (13.350%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:19.711 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:19 smithi142 ceph-mon[106729]: pgmap v1481: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 52077/185547 objects degraded (28.067%); 24771/185547 objects misplaced (13.350%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:19.726 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:42:19.726 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:19.744 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:42:19.744 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:19.762 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:42:19.763 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:19.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:19.779 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:42:19.779 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:19.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:19.799 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:42:19.799 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:19 smithi152 ceph-mon[93456]: pgmap v1481: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 52077/185547 objects degraded (28.067%); 24771/185547 objects misplaced (13.350%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:19.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:19.924 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:19.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:19.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:20.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:20.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:20.150 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:20.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:20.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:42:20.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:20.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:20.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:20.429 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:20.464 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:42:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:42:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:20.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:20.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:20.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1796745520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:20.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1796745520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:20.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:20.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:20.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:20.801 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:20.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1796745520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:20.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:20.924 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:20.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:21.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:21.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:21.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:21.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:21.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:21.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:21.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:21.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:21.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:21.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:21 smithi142 ceph-mon[106729]: pgmap v1482: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 51992/184245 objects degraded (28.219%); 24498/184245 objects misplaced (13.296%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51992/184245 objects degraded (28.219%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:21 smithi142 ceph-mon[103614]: pgmap v1482: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 51992/184245 objects degraded (28.219%); 24498/184245 objects misplaced (13.296%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:21.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51992/184245 objects degraded (28.219%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:21.651 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:42:21.651 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 51992/184245 objects degraded (28.219%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2222,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":389,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":61415,"data_bytes":4023719909,"bytes_used":14511972352,"bytes_avail":561458241536,"bytes_total":575970213888,"degraded_objects":51992,"degraded_total":184245,"degraded_ratio":0.28218947596949712,"misplaced_objects":24498,"misplaced_total":184245,"misplaced_ratio":0.13296425954571359,"recovering_objects_per_sec":26,"recovering_bytes_per_sec":1768945,"recovering_keys_per_sec":0,"num_objects_recovered":270,"num_bytes_recovered":17694720,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":151},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:42:21.660 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:21.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:21.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:21.805 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:21.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:21 smithi152 ceph-mon[93456]: pgmap v1482: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 51992/184245 objects degraded (28.219%); 24498/184245 objects misplaced (13.296%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51992/184245 objects degraded (28.219%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:21.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:21.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:21.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:21.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:21.985 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:42:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:42:21] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:42:22.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:22.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:22.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:22.177 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:22.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:22.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:22.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:22.423 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:42:22.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:22.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:22.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:22.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:22.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3996067624' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:22.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3996067624' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:22.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:22.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:22.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:22.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:22.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3996067624' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:22.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:22.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:22.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:22.957 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:42:22.957 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:42:22.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:175840 12.6 12:62362ce9:::benchmark_data_smithi142_261410_object57643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:22.952939+0000", 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.0024719780000000001, 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024945179999999998, 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:22.958 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 175840 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.952939+0000", 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.952939+0000", 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:22.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.952941+0000", 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.344e-06 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.952946+0000", 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4199999999999998e-06 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:22.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.952958+0000", 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1464000000000001e-05 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.952963+0000", 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8429999999999998e-06 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.953016+0000", 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2133999999999999e-05 2024-08-17T17:42:22.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.953109+0000", 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3213000000000004e-05 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6,7", 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.953194+0000", 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5554000000000001e-05 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:22.962 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:22.963 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:42:22.963 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 1 2024-08-17T17:42:22.963 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:42:22.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:42:23.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:23.055 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:23.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:23.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:23.180 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:23.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:23.190 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:23.226 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:23.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:23.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:23.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:23.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:23.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:23.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:23.582 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:23 smithi142 ceph-mon[106729]: pgmap v1483: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 51992/182856 objects degraded (28.433%); 24434/182856 objects misplaced (13.362%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:23.582 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:23 smithi142 ceph-mon[103614]: pgmap v1483: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 51992/182856 objects degraded (28.433%); 24434/182856 objects misplaced (13.362%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:23.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:23.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:23.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:23.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:23 smithi152 ceph-mon[93456]: pgmap v1483: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 51992/182856 objects degraded (28.433%); 24434/182856 objects misplaced (13.362%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:23.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:23.925 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:23.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:24.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:24.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:24.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:24.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:24.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:24.299 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:24.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:24.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:42:24.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:24.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:24.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:24.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:24.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:24.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:24.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:24.798 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:24.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:24.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:24.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:25.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:25.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:25.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:25.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:25.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:25.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:25.340 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:25.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:25.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:25.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:25.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:25.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:25.675 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:25.777 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:25.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:25 smithi152 ceph-mon[93456]: pgmap v1484: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 51948/182571 objects degraded (28.454%); 24339/182571 objects misplaced (13.331%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:25 smithi142 ceph-mon[106729]: pgmap v1484: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 51948/182571 objects degraded (28.454%); 24339/182571 objects misplaced (13.331%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:25 smithi142 ceph-mon[103614]: pgmap v1484: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 51948/182571 objects degraded (28.454%); 24339/182571 objects misplaced (13.331%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:25.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:25.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:25.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:26.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:26.049 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:26.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:26.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:26.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:26.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:26.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:26.428 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:26.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:26.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:26.653 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:26.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:26.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51948/182571 objects degraded (28.454%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:26.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:26.710 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:26.710 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51948/182571 objects degraded (28.454%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:26.711 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:26.711 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:26.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:26.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51948/182571 objects degraded (28.454%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:26.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:26.925 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:26.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:27.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:27.049 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:27.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:27.174 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:27.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:27.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:27.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:27.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:27.424 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:27.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:27.526 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:27.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:27.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:27.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:27.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:27.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:27 smithi152 ceph-mon[93456]: pgmap v1485: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 51902/181476 objects degraded (28.600%); 24200/181476 objects misplaced (13.335%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:27.842 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:27 smithi142 ceph-mon[106729]: pgmap v1485: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 51902/181476 objects degraded (28.600%); 24200/181476 objects misplaced (13.335%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:27.842 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:27 smithi142 ceph-mon[103614]: pgmap v1485: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 51902/181476 objects degraded (28.600%); 24200/181476 objects misplaced (13.335%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:27.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:27.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:27.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:28.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:28.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:28.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:28.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:28.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:28.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:28.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:28.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:28.417 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:28.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:28.462 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:42:28.463 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:42:28.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:28.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:28.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:28.670 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:28.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:28.772 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:28.794 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:28.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:28.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:28 smithi152 ceph-mon[93456]: from='client.37728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:28.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:28 smithi152 ceph-mon[93456]: pgmap v1486: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 51902/180843 objects degraded (28.700%); 24164/180843 objects misplaced (13.362%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:28.842 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:28 smithi142 ceph-mon[106729]: from='client.37728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:28.843 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:28 smithi142 ceph-mon[106729]: pgmap v1486: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 51902/180843 objects degraded (28.700%); 24164/180843 objects misplaced (13.362%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:28.843 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:28 smithi142 ceph-mon[103614]: from='client.37728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:28.843 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:28 smithi142 ceph-mon[103614]: pgmap v1486: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 51902/180843 objects degraded (28.700%); 24164/180843 objects misplaced (13.362%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:42:28.896 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:28.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:28.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:29.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:29.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:29.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:29.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:29.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:29.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:29.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:29.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:29.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:29.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:29.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:29.549 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:42:29.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:29.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:29.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:29.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:29.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4063532972' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:29.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4063532972' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:29.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4063532972' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:29.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:29.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:29.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:29.928 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1485,"stamp":"2024-08-17T17:42:26.359433+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3963230181,"num_objects":60492,"num_object_clones":0,"num_object_copies":181476,"num_objects_missing_on_primary":4411,"num_objects_missing":4411,"num_objects_degraded":51902,"num_objects_misplaced":24200,"num_objects_unfound":0,"num_objects_dirty":60492,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":176202,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31042,"num_bytes_recovered":1911497358,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104850,"ondisk_log_size":104850,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14038356,"kb_used_data":8245924,"kb_used_omap":27,"kb_used_meta":5792356,"kb_avail":548432556,"statfs":{"total":575970213888,"available":561594937344,"internally_reserved":0,"allocated":8443826176,"data_stored":8431109053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5931373288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,3,1,0,1,4,2,1,5,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-103088128,"num_objects":-1573,"num_object_clones":0,"num_object_copies":-4719,"num_objects_missing_on_primary":-88,"num_objects_missing":-88,"num_objects_degraded":-175,"num_objects_misplaced":-625,"num_objects_unfound":0,"num_objects_dirty":-1573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1573,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":284,"num_bytes_recovered":18612224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002768"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":621,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460399+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:42:16.460399+0000","last_peered":"2024-08-17T17:42:16.460399+0000","last_clean":"2024-08-17T17:42:16.460399+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:42:16.460399+0000","last_undegraded":"2024-08-17T17:42:16.460399+0000","last_fullsized":"2024-08-17T17:42:16.460399+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":98,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465204+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:42:06.465204+0000","last_peered":"2024-08-17T17:42:06.465204+0000","last_clean":"2024-08-17T17:42:06.465204+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:42:06.465204+0000","last_undegraded":"2024-08-17T17:42:06.465204+0000","last_fullsized":"2024-08-17T17:42:06.465204+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":292,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.481962+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:42:06.481962+0000","last_peered":"2024-08-17T17:42:06.481962+0000","last_clean":"2024-08-17T17:42:06.481962+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:42:06.481962+0000","last_undegraded":"2024-08-17T17:42:06.481962+0000","last_fullsized":"2024-08-17T17:42:06.481962+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":319,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459490+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:42:16.459490+0000","last_peered":"2024-08-17T17:42:16.459490+0000","last_clean":"2024-08-17T17:42:16.459490+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:42:16.459490+0000","last_undegraded":"2024-08-17T17:42:16.459490+0000","last_fullsized":"2024-08-17T17:42:16.459490+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":496,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461184+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:42:16.461184+0000","last_peered":"2024-08-17T17:42:16.461184+0000","last_clean":"2024-08-17T17:42:16.461184+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:42:16.461184+0000","last_undegraded":"2024-08-17T17:42:16.461184+0000","last_fullsized":"2024-08-17T17:42:16.461184+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":241,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465038+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:42:06.465038+0000","last_peered":"2024-08-17T17:42:06.465038+0000","last_clean":"2024-08-17T17:42:06.465038+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:42:06.465038+0000","last_undegraded":"2024-08-17T17:42:06.465038+0000","last_fullsized":"2024-08-17T17:42:06.465038+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":590,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459843+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:42:16.459843+0000","last_peered":"2024-08-17T17:42:16.459843+0000","last_clean":"2024-08-17T17:42:16.459843+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:42:16.459843+0000","last_undegraded":"2024-08-17T17:42:16.459843+0000","last_fullsized":"2024-08-17T17:42:16.459843+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":409,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.443207+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:42:06.443207+0000","last_peered":"2024-08-17T17:42:06.443207+0000","last_clean":"2024-08-17T17:42:06.443207+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:42:06.443207+0000","last_undegraded":"2024-08-17T17:42:06.443207+0000","last_fullsized":"2024-08-17T17:42:06.443207+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"389'10939","reported_seq":16756,"reported_epoch":389,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:42:24.673496+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:42:24.673496+0000","last_peered":"2024-08-17T17:42:24.673496+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:42:24.673496+0000","last_undegraded":"2024-08-17T17:42:24.673496+0000","last_fullsized":"2024-08-17T17:42:24.673496+0000","mapping_epoch":337,"log_start":"326'4063","ondisk_log_start":"326'4063","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246743040,"num_objects":3765,"num_object_clones":0,"num_object_copies":11295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3765,"num_objects_unfound":0,"num_objects_dirty":3765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10939,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2858,"num_bytes_recovered":187301888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3765}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":94,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.509409+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:42:06.509409+0000","last_peered":"2024-08-17T17:42:06.509409+0000","last_clean":"2024-08-17T17:42:06.509409+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:42:06.509409+0000","last_undegraded":"2024-08-17T17:42:06.509409+0000","last_fullsized":"2024-08-17T17:42:06.509409+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":275,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.481000+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:42:06.481000+0000","last_peered":"2024-08-17T17:42:06.481000+0000","last_clean":"2024-08-17T17:42:06.481000+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:42:06.481000+0000","last_undegraded":"2024-08-17T17:42:06.481000+0000","last_fullsized":"2024-08-17T17:42:06.481000+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"389'11153","reported_seq":11240,"reported_epoch":389,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:22.002311+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:42:22.002311+0000","last_peered":"2024-08-17T17:42:22.002311+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:42:22.002311+0000","last_undegraded":"2024-08-17T17:42:22.002311+0000","last_fullsized":"2024-08-17T17:42:22.002311+0000","mapping_epoch":337,"log_start":"326'4582","ondisk_log_start":"326'4582","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256966656,"num_objects":3921,"num_object_clones":0,"num_object_copies":11763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3921,"num_objects_unfound":0,"num_objects_dirty":3921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11153,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3921}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":96,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465406+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:42:06.465406+0000","last_peered":"2024-08-17T17:42:06.465406+0000","last_clean":"2024-08-17T17:42:06.465406+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:42:06.465406+0000","last_undegraded":"2024-08-17T17:42:06.465406+0000","last_fullsized":"2024-08-17T17:42:06.465406+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":360,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.464988+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:42:06.464988+0000","last_peered":"2024-08-17T17:42:06.464988+0000","last_clean":"2024-08-17T17:42:06.464988+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:42:06.464988+0000","last_undegraded":"2024-08-17T17:42:06.464988+0000","last_fullsized":"2024-08-17T17:42:06.464988+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"389'10980","reported_seq":11070,"reported_epoch":389,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:24.157756+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:42:24.157756+0000","last_peered":"2024-08-17T17:42:24.157756+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:42:24.157756+0000","last_undegraded":"2024-08-17T17:42:24.157756+0000","last_fullsized":"2024-08-17T17:42:24.157756+0000","mapping_epoch":337,"log_start":"326'4153","ondisk_log_start":"326'4153","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3754,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10980,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3754}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":92,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.481936+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:42:06.481936+0000","last_peered":"2024-08-17T17:42:06.481936+0000","last_clean":"2024-08-17T17:42:06.481936+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:42:06.481936+0000","last_undegraded":"2024-08-17T17:42:06.481936+0000","last_fullsized":"2024-08-17T17:42:06.481936+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":370,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.443253+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:42:06.443253+0000","last_peered":"2024-08-17T17:42:06.443253+0000","last_clean":"2024-08-17T17:42:06.443253+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:42:06.443253+0000","last_undegraded":"2024-08-17T17:42:06.443253+0000","last_fullsized":"2024-08-17T17:42:06.443253+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"389'11180","reported_seq":11339,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:24.882007+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:42:24.882007+0000","last_peered":"2024-08-17T17:42:24.882007+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:42:24.882007+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'4113","ondisk_log_start":"326'4113","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247595008,"num_objects":3778,"num_object_clones":0,"num_object_copies":11334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11180,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-2991},{"shards":"0,3,7","objects":3778}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":533,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460554+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:42:16.460554+0000","last_peered":"2024-08-17T17:42:16.460554+0000","last_clean":"2024-08-17T17:42:16.460554+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:42:16.460554+0000","last_undegraded":"2024-08-17T17:42:16.460554+0000","last_fullsized":"2024-08-17T17:42:16.460554+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":299,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.490457+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:42:16.490457+0000","last_peered":"2024-08-17T17:42:16.490457+0000","last_clean":"2024-08-17T17:42:16.490457+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:42:16.490457+0000","last_undegraded":"2024-08-17T17:42:16.490457+0000","last_fullsized":"2024-08-17T17:42:16.490457+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"389'10792","reported_seq":10870,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:21.997631+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:42:21.997631+0000","last_peered":"2024-08-17T17:42:21.997631+0000","last_clean":"2024-08-17T17:42:21.997631+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:42:21.997631+0000","last_undegraded":"2024-08-17T17:42:21.997631+0000","last_fullsized":"2024-08-17T17:42:21.997631+0000","mapping_epoch":319,"log_start":"326'4174","ondisk_log_start":"326'4174","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249167872,"num_objects":3802,"num_object_clones":0,"num_object_copies":11406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10792,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":623,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460202+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:42:16.460202+0000","last_peered":"2024-08-17T17:42:16.460202+0000","last_clean":"2024-08-17T17:42:16.460202+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:42:16.460202+0000","last_undegraded":"2024-08-17T17:42:16.460202+0000","last_fullsized":"2024-08-17T17:42:16.460202+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":343,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465262+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:42:06.465262+0000","last_peered":"2024-08-17T17:42:06.465262+0000","last_clean":"2024-08-17T17:42:06.465262+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:42:06.465262+0000","last_undegraded":"2024-08-17T17:42:06.465262+0000","last_fullsized":"2024-08-17T17:42:06.465262+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"389'11038","reported_seq":11126,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:22.026339+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:42:22.026339+0000","last_peered":"2024-08-17T17:42:22.026339+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:42:22.026339+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'4435","ondisk_log_start":"326'4435","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6603,"log_dups_size":0,"ondisk_log_size":6603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255197184,"num_objects":3894,"num_object_clones":0,"num_object_copies":11682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11038,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3894},{"shards":"1,6","objects":-3472}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":344,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.443390+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:42:06.443390+0000","last_peered":"2024-08-17T17:42:06.443390+0000","last_clean":"2024-08-17T17:42:06.443390+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:42:06.443390+0000","last_undegraded":"2024-08-17T17:42:06.443390+0000","last_fullsized":"2024-08-17T17:42:06.443390+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":370,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465545+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:42:06.465545+0000","last_peered":"2024-08-17T17:42:06.465545+0000","last_clean":"2024-08-17T17:42:06.465545+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:42:06.465545+0000","last_undegraded":"2024-08-17T17:42:06.465545+0000","last_fullsized":"2024-08-17T17:42:06.465545+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"389'11174","reported_seq":11334,"reported_epoch":389,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:23.947294+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:42:23.947294+0000","last_peered":"2024-08-17T17:42:23.947294+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:42:23.947294+0000","last_undegraded":"2024-08-17T17:42:23.947294+0000","last_fullsized":"2024-08-17T17:42:23.947294+0000","mapping_epoch":337,"log_start":"327'5561","ondisk_log_start":"327'5561","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251396096,"num_objects":3836,"num_object_clones":0,"num_object_copies":11508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3836,"num_objects_unfound":0,"num_objects_dirty":3836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11174,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3836}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":381,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465127+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:42:06.465127+0000","last_peered":"2024-08-17T17:42:06.465127+0000","last_clean":"2024-08-17T17:42:06.465127+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:42:06.465127+0000","last_undegraded":"2024-08-17T17:42:06.465127+0000","last_fullsized":"2024-08-17T17:42:06.465127+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":343,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465563+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:42:06.465563+0000","last_peered":"2024-08-17T17:42:06.465563+0000","last_clean":"2024-08-17T17:42:06.465563+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:42:06.465563+0000","last_undegraded":"2024-08-17T17:42:06.465563+0000","last_fullsized":"2024-08-17T17:42:06.465563+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"389'10987","reported_seq":16951,"reported_epoch":389,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:24.612138+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:42:24.612138+0000","last_peered":"2024-08-17T17:42:24.612138+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:42:24.612138+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:42:24.612138+0000","mapping_epoch":337,"log_start":"326'4141","ondisk_log_start":"326'4141","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243597312,"num_objects":3717,"num_object_clones":0,"num_object_copies":11151,"num_objects_missing_on_primary":4411,"num_objects_missing":4411,"num_objects_degraded":3717,"num_objects_misplaced":4411,"num_objects_unfound":0,"num_objects_dirty":3717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10987,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2929,"num_bytes_recovered":191889408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":3705}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":592,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459182+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:42:16.459182+0000","last_peered":"2024-08-17T17:42:16.459182+0000","last_clean":"2024-08-17T17:42:16.459182+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:42:16.459182+0000","last_undegraded":"2024-08-17T17:42:16.459182+0000","last_fullsized":"2024-08-17T17:42:16.459182+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":529,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459931+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:42:16.459931+0000","last_peered":"2024-08-17T17:42:16.459931+0000","last_clean":"2024-08-17T17:42:16.459931+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:42:16.459931+0000","last_undegraded":"2024-08-17T17:42:16.459931+0000","last_fullsized":"2024-08-17T17:42:16.459931+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"389'11133","reported_seq":17151,"reported_epoch":389,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:23.998123+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:42:23.998123+0000","last_peered":"2024-08-17T17:42:23.998123+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:42:23.998123+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:42:23.998123+0000","mapping_epoch":337,"log_start":"327'5544","ondisk_log_start":"327'5544","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246546456,"num_objects":3763,"num_object_clones":0,"num_object_copies":11289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3763,"num_objects_misplaced":4513,"num_objects_unfound":0,"num_objects_dirty":3763,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11133,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2935,"num_bytes_recovered":192282624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3763}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":723,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:16.491946+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:42:16.491946+0000","last_peered":"2024-08-17T17:42:16.491946+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:42:16.491946+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":318,"reported_epoch":388,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:42:06.481758+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:42:06.481758+0000","last_peered":"2024-08-17T17:42:06.481758+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:42:06.481758+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:42:06.481758+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":572,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459413+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:42:16.459413+0000","last_peered":"2024-08-17T17:42:16.459413+0000","last_clean":"2024-08-17T17:42:16.459413+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:42:16.459413+0000","last_undegraded":"2024-08-17T17:42:16.459413+0000","last_fullsized":"2024-08-17T17:42:16.459413+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":353,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.443752+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:42:06.443752+0000","last_peered":"2024-08-17T17:42:06.443752+0000","last_clean":"2024-08-17T17:42:06.443752+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:42:06.443752+0000","last_undegraded":"2024-08-17T17:42:06.443752+0000","last_fullsized":"2024-08-17T17:42:06.443752+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"389'10952","reported_seq":11113,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:24.900033+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:42:24.900033+0000","last_peered":"2024-08-17T17:42:24.900033+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:42:24.900033+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'3921","ondisk_log_start":"326'3921","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248512512,"num_objects":3792,"num_object_clones":0,"num_object_copies":11376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10952,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2855},{"shards":"0,4,7","objects":3792}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":388,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459746+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:42:16.459746+0000","last_peered":"2024-08-17T17:42:16.459746+0000","last_clean":"2024-08-17T17:42:16.459746+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:42:16.459746+0000","last_undegraded":"2024-08-17T17:42:16.459746+0000","last_fullsized":"2024-08-17T17:42:16.459746+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":325,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.464901+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:42:06.464901+0000","last_peered":"2024-08-17T17:42:06.464901+0000","last_clean":"2024-08-17T17:42:06.464901+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:42:06.464901+0000","last_undegraded":"2024-08-17T17:42:06.464901+0000","last_fullsized":"2024-08-17T17:42:06.464901+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":314,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.490364+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:42:16.490364+0000","last_peered":"2024-08-17T17:42:16.490364+0000","last_clean":"2024-08-17T17:42:16.490364+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:42:16.490364+0000","last_undegraded":"2024-08-17T17:42:16.490364+0000","last_fullsized":"2024-08-17T17:42:16.490364+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"389'10987","reported_seq":11075,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:21.912746+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:42:21.912746+0000","last_peered":"2024-08-17T17:42:21.912746+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:42:21.912746+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4417","ondisk_log_start":"326'4417","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249626624,"num_objects":3809,"num_object_clones":0,"num_object_copies":11427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10987,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3809},{"shards":"5,6","objects":-3487}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":480,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459414+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:42:16.459414+0000","last_peered":"2024-08-17T17:42:16.459414+0000","last_clean":"2024-08-17T17:42:16.459414+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:42:16.459414+0000","last_undegraded":"2024-08-17T17:42:16.459414+0000","last_fullsized":"2024-08-17T17:42:16.459414+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":125,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.443080+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:42:06.443080+0000","last_peered":"2024-08-17T17:42:06.443080+0000","last_clean":"2024-08-17T17:42:06.443080+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:42:06.443080+0000","last_undegraded":"2024-08-17T17:42:06.443080+0000","last_fullsized":"2024-08-17T17:42:06.443080+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":664,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461637+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:42:16.461637+0000","last_peered":"2024-08-17T17:42:16.461637+0000","last_clean":"2024-08-17T17:42:16.461637+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:42:16.461637+0000","last_undegraded":"2024-08-17T17:42:16.461637+0000","last_fullsized":"2024-08-17T17:42:16.461637+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"389'10910","reported_seq":11795,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:24.644769+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:42:24.644769+0000","last_peered":"2024-08-17T17:42:24.644769+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:42:24.644769+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'3916","ondisk_log_start":"326'3916","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244842496,"num_objects":3736,"num_object_clones":0,"num_object_copies":11208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10910,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3736},{"shards":"0,5","objects":-2918}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":456,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460676+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:42:16.460676+0000","last_peered":"2024-08-17T17:42:16.460676+0000","last_clean":"2024-08-17T17:42:16.460676+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:42:16.460676+0000","last_undegraded":"2024-08-17T17:42:16.460676+0000","last_fullsized":"2024-08-17T17:42:16.460676+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":364,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465460+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:42:06.465460+0000","last_peered":"2024-08-17T17:42:06.465460+0000","last_clean":"2024-08-17T17:42:06.465460+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:42:06.465460+0000","last_undegraded":"2024-08-17T17:42:06.465460+0000","last_fullsized":"2024-08-17T17:42:06.465460+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":251,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460322+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:42:16.460322+0000","last_peered":"2024-08-17T17:42:16.460322+0000","last_clean":"2024-08-17T17:42:16.460322+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:42:16.460322+0000","last_undegraded":"2024-08-17T17:42:16.460322+0000","last_fullsized":"2024-08-17T17:42:16.460322+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"389'11109","reported_seq":26114,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:25.267103+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:42:25.267103+0000","last_peered":"2024-08-17T17:42:25.267103+0000","last_clean":"2024-08-17T17:42:25.267103+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:42:25.267103+0000","last_undegraded":"2024-08-17T17:42:25.267103+0000","last_fullsized":"2024-08-17T17:42:25.267103+0000","mapping_epoch":355,"log_start":"327'4830","ondisk_log_start":"327'4830","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244252672,"num_objects":3727,"num_object_clones":0,"num_object_copies":11181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11109,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":303,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.481888+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:42:06.481888+0000","last_peered":"2024-08-17T17:42:06.481888+0000","last_clean":"2024-08-17T17:42:06.481888+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:42:06.481888+0000","last_undegraded":"2024-08-17T17:42:06.481888+0000","last_fullsized":"2024-08-17T17:42:06.481888+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460240+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:42:16.460240+0000","last_peered":"2024-08-17T17:42:16.460240+0000","last_clean":"2024-08-17T17:42:16.460240+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:42:16.460240+0000","last_undegraded":"2024-08-17T17:42:16.460240+0000","last_fullsized":"2024-08-17T17:42:16.460240+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":615,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460940+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:42:16.460940+0000","last_peered":"2024-08-17T17:42:16.460940+0000","last_clean":"2024-08-17T17:42:16.460940+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:42:16.460940+0000","last_undegraded":"2024-08-17T17:42:16.460940+0000","last_fullsized":"2024-08-17T17:42:16.460940+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"389'10769","reported_seq":25341,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:25.096729+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:42:25.096729+0000","last_peered":"2024-08-17T17:42:25.096729+0000","last_clean":"2024-08-17T17:42:25.096729+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:42:25.096729+0000","last_undegraded":"2024-08-17T17:42:25.096729+0000","last_fullsized":"2024-08-17T17:42:25.096729+0000","mapping_epoch":345,"log_start":"326'4472","ondisk_log_start":"326'4472","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6297,"log_dups_size":0,"ondisk_log_size":6297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238223360,"num_objects":3635,"num_object_clones":0,"num_object_copies":10905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10769,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":299,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.481556+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:42:06.481556+0000","last_peered":"2024-08-17T17:42:06.481556+0000","last_clean":"2024-08-17T17:42:06.481556+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:42:06.481556+0000","last_undegraded":"2024-08-17T17:42:06.481556+0000","last_fullsized":"2024-08-17T17:42:06.481556+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":371,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.489692+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:42:16.489692+0000","last_peered":"2024-08-17T17:42:16.489692+0000","last_clean":"2024-08-17T17:42:16.489692+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:42:16.489692+0000","last_undegraded":"2024-08-17T17:42:16.489692+0000","last_fullsized":"2024-08-17T17:42:16.489692+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":441,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.481656+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:42:06.481656+0000","last_peered":"2024-08-17T17:42:06.481656+0000","last_clean":"2024-08-17T17:42:06.481656+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:42:06.481656+0000","last_undegraded":"2024-08-17T17:42:06.481656+0000","last_fullsized":"2024-08-17T17:42:06.481656+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"389'10990","reported_seq":11170,"reported_epoch":389,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:25.267807+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:42:25.267807+0000","last_peered":"2024-08-17T17:42:25.267807+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:42:25.267807+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"326'4687","ondisk_log_start":"326'4687","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248643584,"num_objects":3794,"num_object_clones":0,"num_object_copies":11382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10990,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":3794},{"shards":"0,3","objects":-3454}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":645,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460012+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:42:16.460012+0000","last_peered":"2024-08-17T17:42:16.460012+0000","last_clean":"2024-08-17T17:42:16.460012+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:42:16.460012+0000","last_undegraded":"2024-08-17T17:42:16.460012+0000","last_fullsized":"2024-08-17T17:42:16.460012+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":613,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461889+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:42:16.461889+0000","last_peered":"2024-08-17T17:42:16.461889+0000","last_clean":"2024-08-17T17:42:16.461889+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:42:16.461889+0000","last_undegraded":"2024-08-17T17:42:16.461889+0000","last_fullsized":"2024-08-17T17:42:16.461889+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":631,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459037+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:42:16.459037+0000","last_peered":"2024-08-17T17:42:16.459037+0000","last_clean":"2024-08-17T17:42:16.459037+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:42:16.459037+0000","last_undegraded":"2024-08-17T17:42:16.459037+0000","last_fullsized":"2024-08-17T17:42:16.459037+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"389'10831","reported_seq":25517,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:21.834325+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:42:21.834325+0000","last_peered":"2024-08-17T17:42:21.834325+0000","last_clean":"2024-08-17T17:42:21.834325+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:42:21.834325+0000","last_undegraded":"2024-08-17T17:42:21.834325+0000","last_fullsized":"2024-08-17T17:42:21.834325+0000","mapping_epoch":347,"log_start":"326'4272","ondisk_log_start":"326'4272","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245301248,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10831,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":304,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.489957+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:42:16.489957+0000","last_peered":"2024-08-17T17:42:16.489957+0000","last_clean":"2024-08-17T17:42:16.489957+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:42:16.489957+0000","last_undegraded":"2024-08-17T17:42:16.489957+0000","last_fullsized":"2024-08-17T17:42:16.489957+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":393,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.445784+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:42:06.445784+0000","last_peered":"2024-08-17T17:42:06.445784+0000","last_clean":"2024-08-17T17:42:06.445784+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:42:06.445784+0000","last_undegraded":"2024-08-17T17:42:06.445784+0000","last_fullsized":"2024-08-17T17:42:06.445784+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":343,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465058+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:42:06.465058+0000","last_peered":"2024-08-17T17:42:06.465058+0000","last_clean":"2024-08-17T17:42:06.465058+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:42:06.465058+0000","last_undegraded":"2024-08-17T17:42:06.465058+0000","last_fullsized":"2024-08-17T17:42:06.465058+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":343,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465356+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:42:06.465356+0000","last_peered":"2024-08-17T17:42:06.465356+0000","last_clean":"2024-08-17T17:42:06.465356+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:42:06.465356+0000","last_undegraded":"2024-08-17T17:42:06.465356+0000","last_fullsized":"2024-08-17T17:42:06.465356+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":386,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465445+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:42:06.465445+0000","last_peered":"2024-08-17T17:42:06.465445+0000","last_clean":"2024-08-17T17:42:06.465445+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:42:06.465445+0000","last_undegraded":"2024-08-17T17:42:06.465445+0000","last_fullsized":"2024-08-17T17:42:06.465445+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":620,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460759+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:42:16.460759+0000","last_peered":"2024-08-17T17:42:16.460759+0000","last_clean":"2024-08-17T17:42:16.460759+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:42:16.460759+0000","last_undegraded":"2024-08-17T17:42:16.460759+0000","last_fullsized":"2024-08-17T17:42:16.460759+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":160,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461269+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:42:16.461269+0000","last_peered":"2024-08-17T17:42:16.461269+0000","last_clean":"2024-08-17T17:42:16.461269+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:42:16.461269+0000","last_undegraded":"2024-08-17T17:42:16.461269+0000","last_fullsized":"2024-08-17T17:42:16.461269+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":293,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.482072+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:42:06.482072+0000","last_peered":"2024-08-17T17:42:06.482072+0000","last_clean":"2024-08-17T17:42:06.482072+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:42:06.482072+0000","last_undegraded":"2024-08-17T17:42:06.482072+0000","last_fullsized":"2024-08-17T17:42:06.482072+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":578,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459109+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:42:16.459109+0000","last_peered":"2024-08-17T17:42:16.459109+0000","last_clean":"2024-08-17T17:42:16.459109+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:42:16.459109+0000","last_undegraded":"2024-08-17T17:42:16.459109+0000","last_fullsized":"2024-08-17T17:42:16.459109+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":437,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.483466+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:42:06.483466+0000","last_peered":"2024-08-17T17:42:06.483466+0000","last_clean":"2024-08-17T17:42:06.483466+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:42:06.483466+0000","last_undegraded":"2024-08-17T17:42:06.483466+0000","last_fullsized":"2024-08-17T17:42:06.483466+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":572,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.458941+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:42:16.458941+0000","last_peered":"2024-08-17T17:42:16.458941+0000","last_clean":"2024-08-17T17:42:16.458941+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:42:16.458941+0000","last_undegraded":"2024-08-17T17:42:16.458941+0000","last_fullsized":"2024-08-17T17:42:16.458941+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":360,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.445703+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:42:06.445703+0000","last_peered":"2024-08-17T17:42:06.445703+0000","last_clean":"2024-08-17T17:42:06.445703+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:42:06.445703+0000","last_undegraded":"2024-08-17T17:42:06.445703+0000","last_fullsized":"2024-08-17T17:42:06.445703+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":604,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459104+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:42:16.459104+0000","last_peered":"2024-08-17T17:42:16.459104+0000","last_clean":"2024-08-17T17:42:16.459104+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:42:16.459104+0000","last_undegraded":"2024-08-17T17:42:16.459104+0000","last_fullsized":"2024-08-17T17:42:16.459104+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":320,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461105+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:42:16.461105+0000","last_peered":"2024-08-17T17:42:16.461105+0000","last_clean":"2024-08-17T17:42:16.461105+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:42:16.461105+0000","last_undegraded":"2024-08-17T17:42:16.461105+0000","last_fullsized":"2024-08-17T17:42:16.461105+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":612,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459946+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:42:16.459946+0000","last_peered":"2024-08-17T17:42:16.459946+0000","last_clean":"2024-08-17T17:42:16.459946+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:42:16.459946+0000","last_undegraded":"2024-08-17T17:42:16.459946+0000","last_fullsized":"2024-08-17T17:42:16.459946+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":571,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461073+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:42:16.461073+0000","last_peered":"2024-08-17T17:42:16.461073+0000","last_clean":"2024-08-17T17:42:16.461073+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:42:16.461073+0000","last_undegraded":"2024-08-17T17:42:16.461073+0000","last_fullsized":"2024-08-17T17:42:16.461073+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":372,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465185+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:42:06.465185+0000","last_peered":"2024-08-17T17:42:06.465185+0000","last_clean":"2024-08-17T17:42:06.465185+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:42:06.465185+0000","last_undegraded":"2024-08-17T17:42:06.465185+0000","last_fullsized":"2024-08-17T17:42:06.465185+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":260,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.460477+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:42:16.460477+0000","last_peered":"2024-08-17T17:42:16.460477+0000","last_clean":"2024-08-17T17:42:16.460477+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:42:16.460477+0000","last_undegraded":"2024-08-17T17:42:16.460477+0000","last_fullsized":"2024-08-17T17:42:16.460477+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":317,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.489549+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:42:16.489549+0000","last_peered":"2024-08-17T17:42:16.489549+0000","last_clean":"2024-08-17T17:42:16.489549+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:42:16.489549+0000","last_undegraded":"2024-08-17T17:42:16.489549+0000","last_fullsized":"2024-08-17T17:42:16.489549+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":380,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465307+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:42:06.465307+0000","last_peered":"2024-08-17T17:42:06.465307+0000","last_clean":"2024-08-17T17:42:06.465307+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:42:06.465307+0000","last_undegraded":"2024-08-17T17:42:06.465307+0000","last_fullsized":"2024-08-17T17:42:06.465307+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":243,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465360+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:42:06.465360+0000","last_peered":"2024-08-17T17:42:06.465360+0000","last_clean":"2024-08-17T17:42:06.465360+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:42:06.465360+0000","last_undegraded":"2024-08-17T17:42:06.465360+0000","last_fullsized":"2024-08-17T17:42:06.465360+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":380,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465005+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:42:06.465005+0000","last_peered":"2024-08-17T17:42:06.465005+0000","last_clean":"2024-08-17T17:42:06.465005+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:42:06.465005+0000","last_undegraded":"2024-08-17T17:42:06.465005+0000","last_fullsized":"2024-08-17T17:42:06.465005+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":243,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465412+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:42:06.465412+0000","last_peered":"2024-08-17T17:42:06.465412+0000","last_clean":"2024-08-17T17:42:06.465412+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:42:06.465412+0000","last_undegraded":"2024-08-17T17:42:06.465412+0000","last_fullsized":"2024-08-17T17:42:06.465412+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":624,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.459506+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:42:16.459506+0000","last_peered":"2024-08-17T17:42:16.459506+0000","last_clean":"2024-08-17T17:42:16.459506+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:42:16.459506+0000","last_undegraded":"2024-08-17T17:42:16.459506+0000","last_fullsized":"2024-08-17T17:42:16.459506+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":597,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.461266+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:42:16.461266+0000","last_peered":"2024-08-17T17:42:16.461266+0000","last_clean":"2024-08-17T17:42:16.461266+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:42:16.461266+0000","last_undegraded":"2024-08-17T17:42:16.461266+0000","last_fullsized":"2024-08-17T17:42:16.461266+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":343,"reported_epoch":388,"state":"active+clean","last_fresh":"2024-08-17T17:42:06.465469+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:42:06.465469+0000","last_peered":"2024-08-17T17:42:06.465469+0000","last_clean":"2024-08-17T17:42:06.465469+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:42:06.465469+0000","last_undegraded":"2024-08-17T17:42:06.465469+0000","last_fullsized":"2024-08-17T17:42:06.465469+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":319,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.490002+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:42:16.490002+0000","last_peered":"2024-08-17T17:42:16.490002+0000","last_clean":"2024-08-17T17:42:16.490002+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:42:16.490002+0000","last_undegraded":"2024-08-17T17:42:16.490002+0000","last_fullsized":"2024-08-17T17:42:16.490002+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3962634264,"num_objects":60466,"num_object_clones":0,"num_object_copies":181398,"num_objects_missing_on_primary":4411,"num_objects_missing":4411,"num_objects_degraded":51899,"num_objects_misplaced":24200,"num_objects_unfound":0,"num_objects_dirty":60466,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175924,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31001,"num_bytes_recovered":1909129216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13863825408,"data_stored":13863813192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104643,"ondisk_log_size":104643,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170112,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463481,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2607720,"kb_used_data":1709856,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":91137432,"statfs":{"total":95995035648,"available":93324730368,"internally_reserved":0,"allocated":1750892544,"data_stored":1748760828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492156,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":882700,"kb_used_data":487364,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92862452,"statfs":{"total":95995035648,"available":95091150848,"internally_reserved":0,"allocated":499060736,"data_stored":496951320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986747,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2467804,"kb_used_data":963284,"kb_used_omap":1,"kb_used_meta":1504510,"kb_avail":91277348,"statfs":{"total":95995035648,"available":93468004352,"internally_reserved":0,"allocated":986402816,"data_stored":984282040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1540618584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,0,0,2,2,1,5,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225575,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2859020,"kb_used_data":1929412,"kb_used_omap":4,"kb_used_meta":929595,"kb_avail":90886132,"statfs":{"total":95995035648,"available":93067399168,"internally_reserved":0,"allocated":1975717888,"data_stored":1973608770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":951905728},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075900,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533793,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2762576,"kb_used_data":1698376,"kb_used_omap":5,"kb_used_meta":1064186,"kb_avail":90982576,"statfs":{"total":95995035648,"available":93166157824,"internally_reserved":0,"allocated":1739137024,"data_stored":1737022584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1089726704},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089770,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2458536,"kb_used_data":1457632,"kb_used_omap":5,"kb_used_meta":1000890,"kb_avail":91286616,"statfs":{"total":95995035648,"available":93477494784,"internally_reserved":0,"allocated":1492615168,"data_stored":1490483511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1024911419},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1485111296,"data_stored":1485111296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1732579328,"data_stored":1732575256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1969487872,"data_stored":1969487872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":981139456,"data_stored":981139456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":493223936,"data_stored":493223936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1743519744,"data_stored":1743519744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2119503872,"data_stored":2119499800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:30.021 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:30.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:30.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:30.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:30.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:30.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:30.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:30.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:30.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:30.426 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:30.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:30.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:30.542 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:42:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:42:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:30.640 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:30.662 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:30.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:30.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:30.789 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:30 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4063532972' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:30 smithi152 ceph-mon[93456]: osdmap e390: 8 total, 8 up, 6 in 2024-08-17T17:42:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:30 smithi152 ceph-mon[93456]: pgmap v1488: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51828/179880 objects degraded (28.813%); 23960/179880 objects misplaced (13.320%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:42:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4063532972' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[106729]: osdmap e390: 8 total, 8 up, 6 in 2024-08-17T17:42:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[106729]: pgmap v1488: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51828/179880 objects degraded (28.813%); 23960/179880 objects misplaced (13.320%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:42:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4063532972' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[103614]: osdmap e390: 8 total, 8 up, 6 in 2024-08-17T17:42:30.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:30 smithi142 ceph-mon[103614]: pgmap v1488: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51828/179880 objects degraded (28.813%); 23960/179880 objects misplaced (13.320%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:42:30.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:30.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:30.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:31.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:31.037 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:31.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:31.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:31.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:31.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:31.285 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:31.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:31.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:31.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:31.455 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:42:31.455 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:31.471 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:42:31.471 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:31.487 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:42:31.487 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:31.504 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:42:31.504 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:31.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:31.522 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:42:31.523 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:31.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:31.540 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:42:31.540 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:31.559 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:42:31.559 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:31.582 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:42:31.582 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:31.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:31.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:31.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:31.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51828/179880 objects degraded (28.813%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:31.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/36759824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:31.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51828/179880 objects degraded (28.813%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:31.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/36759824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:31.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51828/179880 objects degraded (28.813%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/36759824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:31.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:31.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:31.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:32.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:32.040 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:42:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:42:31] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:42:32.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:32.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:32.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:32.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:32.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:32.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:32.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:32.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:32.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:32.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:32.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:32.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:32.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:32.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:32 smithi152 ceph-mon[93456]: pgmap v1489: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 51828/178737 objects degraded (28.997%); 23910/178737 objects misplaced (13.377%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:42:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:32 smithi142 ceph-mon[106729]: pgmap v1489: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 51828/178737 objects degraded (28.997%); 23910/178737 objects misplaced (13.377%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:42:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:32 smithi142 ceph-mon[103614]: pgmap v1489: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 51828/178737 objects degraded (28.997%); 23910/178737 objects misplaced (13.377%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:42:32.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:32.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:32.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:32.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:32.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:42:33.015 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:33.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:33.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:33.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:33.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:33.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:33.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:33.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:33.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:33.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:33.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:33.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:33.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:33.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:33.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:33.947 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:33.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:34.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:34.069 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:34.069 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.071 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:34.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:34.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:34.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:34.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:34.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:34.349 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:34.393 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:34.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:34.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:34.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:34.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:34.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:34.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:34.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:34.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:34.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:34.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:34.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:35.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:35.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:35.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:35.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:35.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:42:35.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:35.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:35.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:35.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:35.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:35.576 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:35 smithi142 ceph-mon[106729]: pgmap v1490: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 51771/178422 objects degraded (29.016%); 23805/178422 objects misplaced (13.342%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:35 smithi142 ceph-mon[103614]: pgmap v1490: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 51771/178422 objects degraded (29.016%); 23805/178422 objects misplaced (13.342%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:35.678 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:35.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:35.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:35 smithi152 ceph-mon[93456]: pgmap v1490: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 51771/178422 objects degraded (29.016%); 23805/178422 objects misplaced (13.342%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:42:35.825 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:35.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:35.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:35.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:36.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:36.073 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:36.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:36.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:36.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:36.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:36.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:36.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:36.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:36.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:36.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:36.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:36.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:36.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:36.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:36.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51771/178422 objects degraded (29.016%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:36.855 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51771/178422 objects degraded (29.016%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:36.855 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51771/178422 objects degraded (29.016%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:36.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:36.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:36.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:37.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:37.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:37.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:37.193 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:42:37.193 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:42:37.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:37.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:37.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:37.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:37.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:37.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:37.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:37.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:37.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:37.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:37.799 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:37.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:37 smithi152 ceph-mon[93456]: pgmap v1491: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 51727/176967 objects degraded (29.230%); 23659/176967 objects misplaced (13.369%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:37.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:37 smithi142 ceph-mon[103614]: pgmap v1491: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 51727/176967 objects degraded (29.230%); 23659/176967 objects misplaced (13.369%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:37.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:37 smithi142 ceph-mon[106729]: pgmap v1491: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 51727/176967 objects degraded (29.230%); 23659/176967 objects misplaced (13.369%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:37.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:37.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:42:37.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:37.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:37.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:38.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:38.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:38.140 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:38.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:38.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:38.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:38.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:38.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:38.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:38.485 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:38.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:38.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:38.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:38 smithi142 ceph-mon[103614]: from='client.37746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:38.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:38 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:38 smithi142 ceph-mon[106729]: from='client.37746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:38 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:38.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:38.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:38.759 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1491,"stamp":"2024-08-17T17:42:36.362245+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3864729573,"num_objects":58989,"num_object_clones":0,"num_object_copies":176967,"num_objects_missing_on_primary":4322,"num_objects_missing":4322,"num_objects_degraded":51727,"num_objects_misplaced":23659,"num_objects_unfound":0,"num_objects_dirty":58989,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":177705,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31320,"num_bytes_recovered":1929716366,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104853,"ondisk_log_size":104853,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13851572,"kb_used_data":8041092,"kb_used_omap":27,"kb_used_meta":5810404,"kb_avail":548619340,"statfs":{"total":575970213888,"available":561786204160,"internally_reserved":0,"allocated":8234078208,"data_stored":8221314327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5949854440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-98500608,"num_objects":-1503,"num_object_clones":0,"num_object_copies":-4509,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":-175,"num_objects_misplaced":-541,"num_objects_unfound":0,"num_objects_dirty":-1503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1503,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":278,"num_bytes_recovered":18219008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002811"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":623,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559338+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:42:29.559338+0000","last_peered":"2024-08-17T17:42:29.559338+0000","last_clean":"2024-08-17T17:42:29.559338+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:42:29.559338+0000","last_undegraded":"2024-08-17T17:42:29.559338+0000","last_fullsized":"2024-08-17T17:42:29.559338+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":99,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499356+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:42:16.499356+0000","last_peered":"2024-08-17T17:42:16.499356+0000","last_clean":"2024-08-17T17:42:16.499356+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:42:16.499356+0000","last_undegraded":"2024-08-17T17:42:16.499356+0000","last_fullsized":"2024-08-17T17:42:16.499356+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":293,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.520296+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:42:16.520296+0000","last_peered":"2024-08-17T17:42:16.520296+0000","last_clean":"2024-08-17T17:42:16.520296+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:42:16.520296+0000","last_undegraded":"2024-08-17T17:42:16.520296+0000","last_fullsized":"2024-08-17T17:42:16.520296+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":321,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559582+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:42:29.559582+0000","last_peered":"2024-08-17T17:42:29.559582+0000","last_clean":"2024-08-17T17:42:29.559582+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:42:29.559582+0000","last_undegraded":"2024-08-17T17:42:29.559582+0000","last_fullsized":"2024-08-17T17:42:29.559582+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":498,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559777+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:42:29.559777+0000","last_peered":"2024-08-17T17:42:29.559777+0000","last_clean":"2024-08-17T17:42:29.559777+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:42:29.559777+0000","last_undegraded":"2024-08-17T17:42:29.559777+0000","last_fullsized":"2024-08-17T17:42:29.559777+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":242,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499235+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:42:16.499235+0000","last_peered":"2024-08-17T17:42:16.499235+0000","last_clean":"2024-08-17T17:42:16.499235+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:42:16.499235+0000","last_undegraded":"2024-08-17T17:42:16.499235+0000","last_fullsized":"2024-08-17T17:42:16.499235+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":592,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559293+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:42:29.559293+0000","last_peered":"2024-08-17T17:42:29.559293+0000","last_clean":"2024-08-17T17:42:29.559293+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:42:29.559293+0000","last_undegraded":"2024-08-17T17:42:29.559293+0000","last_fullsized":"2024-08-17T17:42:29.559293+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":410,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.497183+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:42:16.497183+0000","last_peered":"2024-08-17T17:42:16.497183+0000","last_clean":"2024-08-17T17:42:16.497183+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:42:16.497183+0000","last_undegraded":"2024-08-17T17:42:16.497183+0000","last_fullsized":"2024-08-17T17:42:16.497183+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"390'11042","reported_seq":17066,"reported_epoch":390,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:42:34.557541+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:42:34.557541+0000","last_peered":"2024-08-17T17:42:34.557541+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:42:34.557541+0000","last_undegraded":"2024-08-17T17:42:34.557541+0000","last_fullsized":"2024-08-17T17:42:34.557541+0000","mapping_epoch":337,"log_start":"326'4163","ondisk_log_start":"326'4163","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239992832,"num_objects":3662,"num_object_clones":0,"num_object_copies":10986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3662,"num_objects_unfound":0,"num_objects_dirty":3662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11042,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2961,"num_bytes_recovered":194052096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3662}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":95,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.538759+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:42:16.538759+0000","last_peered":"2024-08-17T17:42:16.538759+0000","last_clean":"2024-08-17T17:42:16.538759+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:42:16.538759+0000","last_undegraded":"2024-08-17T17:42:16.538759+0000","last_fullsized":"2024-08-17T17:42:16.538759+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.519021+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:42:16.519021+0000","last_peered":"2024-08-17T17:42:16.519021+0000","last_clean":"2024-08-17T17:42:16.519021+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:42:16.519021+0000","last_undegraded":"2024-08-17T17:42:16.519021+0000","last_fullsized":"2024-08-17T17:42:16.519021+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"390'11239","reported_seq":11327,"reported_epoch":390,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:31.682791+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:42:31.682791+0000","last_peered":"2024-08-17T17:42:31.682791+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:42:31.682791+0000","last_undegraded":"2024-08-17T17:42:31.682791+0000","last_fullsized":"2024-08-17T17:42:31.682791+0000","mapping_epoch":337,"log_start":"326'4682","ondisk_log_start":"326'4682","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6557,"log_dups_size":0,"ondisk_log_size":6557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251330560,"num_objects":3835,"num_object_clones":0,"num_object_copies":11505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3835,"num_objects_unfound":0,"num_objects_dirty":3835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11239,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3835}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":97,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499788+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:42:16.499788+0000","last_peered":"2024-08-17T17:42:16.499788+0000","last_clean":"2024-08-17T17:42:16.499788+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:42:16.499788+0000","last_undegraded":"2024-08-17T17:42:16.499788+0000","last_fullsized":"2024-08-17T17:42:16.499788+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":361,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499209+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:42:16.499209+0000","last_peered":"2024-08-17T17:42:16.499209+0000","last_clean":"2024-08-17T17:42:16.499209+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:42:16.499209+0000","last_undegraded":"2024-08-17T17:42:16.499209+0000","last_fullsized":"2024-08-17T17:42:16.499209+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"390'11061","reported_seq":11152,"reported_epoch":390,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:34.449133+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:42:34.449133+0000","last_peered":"2024-08-17T17:42:34.449133+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:42:34.449133+0000","last_undegraded":"2024-08-17T17:42:34.449133+0000","last_fullsized":"2024-08-17T17:42:34.449133+0000","mapping_epoch":337,"log_start":"326'4153","ondisk_log_start":"326'4153","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6908,"log_dups_size":0,"ondisk_log_size":6908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240713728,"num_objects":3673,"num_object_clones":0,"num_object_copies":11019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3673,"num_objects_unfound":0,"num_objects_dirty":3673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11061,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3673}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":93,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.520296+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:42:16.520296+0000","last_peered":"2024-08-17T17:42:16.520296+0000","last_clean":"2024-08-17T17:42:16.520296+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:42:16.520296+0000","last_undegraded":"2024-08-17T17:42:16.520296+0000","last_fullsized":"2024-08-17T17:42:16.520296+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":371,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.489632+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:42:16.489632+0000","last_peered":"2024-08-17T17:42:16.489632+0000","last_clean":"2024-08-17T17:42:16.489632+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:42:16.489632+0000","last_undegraded":"2024-08-17T17:42:16.489632+0000","last_fullsized":"2024-08-17T17:42:16.489632+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"390'11278","reported_seq":11439,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:34.855306+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:42:34.855306+0000","last_peered":"2024-08-17T17:42:34.855306+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:42:34.855306+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'4213","ondisk_log_start":"326'4213","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241172480,"num_objects":3680,"num_object_clones":0,"num_object_copies":11040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11278,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-3089},{"shards":"0,3,7","objects":3680}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":535,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559493+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:42:29.559493+0000","last_peered":"2024-08-17T17:42:29.559493+0000","last_clean":"2024-08-17T17:42:29.559493+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:42:29.559493+0000","last_undegraded":"2024-08-17T17:42:29.559493+0000","last_fullsized":"2024-08-17T17:42:29.559493+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":302,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.554495+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:42:29.554495+0000","last_peered":"2024-08-17T17:42:29.554495+0000","last_clean":"2024-08-17T17:42:29.554495+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:42:29.554495+0000","last_undegraded":"2024-08-17T17:42:29.554495+0000","last_fullsized":"2024-08-17T17:42:29.554495+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"390'10889","reported_seq":10968,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.918010+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:42:31.918010+0000","last_peered":"2024-08-17T17:42:31.918010+0000","last_clean":"2024-08-17T17:42:31.918010+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:42:31.918010+0000","last_undegraded":"2024-08-17T17:42:31.918010+0000","last_fullsized":"2024-08-17T17:42:31.918010+0000","mapping_epoch":319,"log_start":"326'4274","ondisk_log_start":"326'4274","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242810880,"num_objects":3705,"num_object_clones":0,"num_object_copies":11115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10889,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":625,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.560159+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:42:29.560159+0000","last_peered":"2024-08-17T17:42:29.560159+0000","last_clean":"2024-08-17T17:42:29.560159+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:42:29.560159+0000","last_undegraded":"2024-08-17T17:42:29.560159+0000","last_fullsized":"2024-08-17T17:42:29.560159+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":344,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499502+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:42:16.499502+0000","last_peered":"2024-08-17T17:42:16.499502+0000","last_clean":"2024-08-17T17:42:16.499502+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:42:16.499502+0000","last_undegraded":"2024-08-17T17:42:16.499502+0000","last_fullsized":"2024-08-17T17:42:16.499502+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"390'11129","reported_seq":11218,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:32.046433+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:42:32.046433+0000","last_peered":"2024-08-17T17:42:32.046433+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:42:32.046433+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"326'4535","ondisk_log_start":"326'4535","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249233408,"num_objects":3803,"num_object_clones":0,"num_object_copies":11409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11129,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3803},{"shards":"1,6","objects":-3563}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":345,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.489691+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:42:16.489691+0000","last_peered":"2024-08-17T17:42:16.489691+0000","last_clean":"2024-08-17T17:42:16.489691+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:42:16.489691+0000","last_undegraded":"2024-08-17T17:42:16.489691+0000","last_fullsized":"2024-08-17T17:42:16.489691+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":371,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499946+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:42:16.499946+0000","last_peered":"2024-08-17T17:42:16.499946+0000","last_clean":"2024-08-17T17:42:16.499946+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:42:16.499946+0000","last_undegraded":"2024-08-17T17:42:16.499946+0000","last_fullsized":"2024-08-17T17:42:16.499946+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"390'11270","reported_seq":11432,"reported_epoch":390,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:33.497460+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:42:33.497460+0000","last_peered":"2024-08-17T17:42:33.497460+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:42:33.497460+0000","last_undegraded":"2024-08-17T17:42:33.497460+0000","last_fullsized":"2024-08-17T17:42:33.497460+0000","mapping_epoch":337,"log_start":"327'5661","ondisk_log_start":"327'5661","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245104640,"num_objects":3740,"num_object_clones":0,"num_object_copies":11220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3740,"num_objects_unfound":0,"num_objects_dirty":3740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11270,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3740}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":382,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499344+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:42:16.499344+0000","last_peered":"2024-08-17T17:42:16.499344+0000","last_clean":"2024-08-17T17:42:16.499344+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:42:16.499344+0000","last_undegraded":"2024-08-17T17:42:16.499344+0000","last_fullsized":"2024-08-17T17:42:16.499344+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":344,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499672+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:42:16.499672+0000","last_peered":"2024-08-17T17:42:16.499672+0000","last_clean":"2024-08-17T17:42:16.499672+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:42:16.499672+0000","last_undegraded":"2024-08-17T17:42:16.499672+0000","last_fullsized":"2024-08-17T17:42:16.499672+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"390'11076","reported_seq":17219,"reported_epoch":390,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:34.701620+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:42:34.701620+0000","last_peered":"2024-08-17T17:42:34.701620+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:42:34.701620+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:42:34.701620+0000","mapping_epoch":337,"log_start":"326'4241","ondisk_log_start":"326'4241","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237764608,"num_objects":3628,"num_object_clones":0,"num_object_copies":10884,"num_objects_missing_on_primary":4322,"num_objects_missing":4322,"num_objects_degraded":3628,"num_objects_misplaced":4322,"num_objects_unfound":0,"num_objects_dirty":3628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11076,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3018,"num_bytes_recovered":197722112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":3616}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":594,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.558668+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:42:29.558668+0000","last_peered":"2024-08-17T17:42:29.558668+0000","last_clean":"2024-08-17T17:42:29.558668+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:42:29.558668+0000","last_undegraded":"2024-08-17T17:42:29.558668+0000","last_fullsized":"2024-08-17T17:42:29.558668+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":531,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559373+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:42:29.559373+0000","last_peered":"2024-08-17T17:42:29.559373+0000","last_clean":"2024-08-17T17:42:29.559373+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:42:29.559373+0000","last_undegraded":"2024-08-17T17:42:29.559373+0000","last_fullsized":"2024-08-17T17:42:29.559373+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"390'11219","reported_seq":17411,"reported_epoch":390,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:33.827020+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:42:33.827020+0000","last_peered":"2024-08-17T17:42:33.827020+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:42:33.827020+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:42:33.827020+0000","mapping_epoch":337,"log_start":"327'5644","ondisk_log_start":"327'5644","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240910360,"num_objects":3677,"num_object_clones":0,"num_object_copies":11031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3677,"num_objects_misplaced":4427,"num_objects_unfound":0,"num_objects_dirty":3677,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11219,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3021,"num_bytes_recovered":197918720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3677}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":725,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:29.553495+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:42:29.553495+0000","last_peered":"2024-08-17T17:42:29.553495+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:42:29.553495+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":319,"reported_epoch":389,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:42:16.520025+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:42:16.520025+0000","last_peered":"2024-08-17T17:42:16.520025+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:42:16.520025+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:42:16.520025+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":574,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559489+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:42:29.559489+0000","last_peered":"2024-08-17T17:42:29.559489+0000","last_clean":"2024-08-17T17:42:29.559489+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:42:29.559489+0000","last_undegraded":"2024-08-17T17:42:29.559489+0000","last_fullsized":"2024-08-17T17:42:29.559489+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":354,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.490313+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:42:16.490313+0000","last_peered":"2024-08-17T17:42:16.490313+0000","last_clean":"2024-08-17T17:42:16.490313+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:42:16.490313+0000","last_undegraded":"2024-08-17T17:42:16.490313+0000","last_fullsized":"2024-08-17T17:42:16.490313+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"390'11052","reported_seq":11215,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:34.818512+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:42:34.818512+0000","last_peered":"2024-08-17T17:42:34.818512+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:42:34.818512+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'4021","ondisk_log_start":"326'4021","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241958912,"num_objects":3692,"num_object_clones":0,"num_object_copies":11076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11052,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-2955},{"shards":"0,4,7","objects":3692}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":390,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.549808+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:42:29.549808+0000","last_peered":"2024-08-17T17:42:29.549808+0000","last_clean":"2024-08-17T17:42:29.549808+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:42:29.549808+0000","last_undegraded":"2024-08-17T17:42:29.549808+0000","last_fullsized":"2024-08-17T17:42:29.549808+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":326,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499125+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:42:16.499125+0000","last_peered":"2024-08-17T17:42:16.499125+0000","last_clean":"2024-08-17T17:42:16.499125+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:42:16.499125+0000","last_undegraded":"2024-08-17T17:42:16.499125+0000","last_fullsized":"2024-08-17T17:42:16.499125+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":316,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.557830+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:42:29.557830+0000","last_peered":"2024-08-17T17:42:29.557830+0000","last_clean":"2024-08-17T17:42:29.557830+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:42:29.557830+0000","last_undegraded":"2024-08-17T17:42:29.557830+0000","last_fullsized":"2024-08-17T17:42:29.557830+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"390'11077","reported_seq":11166,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:31.887253+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:42:31.887253+0000","last_peered":"2024-08-17T17:42:31.887253+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:42:31.887253+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4517","ondisk_log_start":"326'4517","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243728384,"num_objects":3719,"num_object_clones":0,"num_object_copies":11157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11077,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3719},{"shards":"5,6","objects":-3577}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":482,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559341+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:42:29.559341+0000","last_peered":"2024-08-17T17:42:29.559341+0000","last_clean":"2024-08-17T17:42:29.559341+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:42:29.559341+0000","last_undegraded":"2024-08-17T17:42:29.559341+0000","last_fullsized":"2024-08-17T17:42:29.559341+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":126,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.497041+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:42:16.497041+0000","last_peered":"2024-08-17T17:42:16.497041+0000","last_clean":"2024-08-17T17:42:16.497041+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:42:16.497041+0000","last_undegraded":"2024-08-17T17:42:16.497041+0000","last_fullsized":"2024-08-17T17:42:16.497041+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":667,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.550723+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:42:29.550723+0000","last_peered":"2024-08-17T17:42:29.550723+0000","last_clean":"2024-08-17T17:42:29.550723+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:42:29.550723+0000","last_undegraded":"2024-08-17T17:42:29.550723+0000","last_fullsized":"2024-08-17T17:42:29.550723+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"390'11014","reported_seq":11901,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:34.796458+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:42:34.796458+0000","last_peered":"2024-08-17T17:42:34.796458+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:42:34.796458+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'4016","ondisk_log_start":"326'4016","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238026752,"num_objects":3632,"num_object_clones":0,"num_object_copies":10896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11014,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3632},{"shards":"0,5","objects":-3022}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":458,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.550152+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:42:29.550152+0000","last_peered":"2024-08-17T17:42:29.550152+0000","last_clean":"2024-08-17T17:42:29.550152+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:42:29.550152+0000","last_undegraded":"2024-08-17T17:42:29.550152+0000","last_fullsized":"2024-08-17T17:42:29.550152+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":365,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499623+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:42:16.499623+0000","last_peered":"2024-08-17T17:42:16.499623+0000","last_clean":"2024-08-17T17:42:16.499623+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:42:16.499623+0000","last_undegraded":"2024-08-17T17:42:16.499623+0000","last_fullsized":"2024-08-17T17:42:16.499623+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":253,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559248+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:42:29.559248+0000","last_peered":"2024-08-17T17:42:29.559248+0000","last_clean":"2024-08-17T17:42:29.559248+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:42:29.559248+0000","last_undegraded":"2024-08-17T17:42:29.559248+0000","last_fullsized":"2024-08-17T17:42:29.559248+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"390'11196","reported_seq":26203,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:35.081937+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:42:35.081937+0000","last_peered":"2024-08-17T17:42:35.081937+0000","last_clean":"2024-08-17T17:42:35.081937+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:42:35.081937+0000","last_undegraded":"2024-08-17T17:42:35.081937+0000","last_fullsized":"2024-08-17T17:42:35.081937+0000","mapping_epoch":355,"log_start":"327'4930","ondisk_log_start":"327'4930","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6266,"log_dups_size":0,"ondisk_log_size":6266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238551040,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11196,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.520212+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:42:16.520212+0000","last_peered":"2024-08-17T17:42:16.520212+0000","last_clean":"2024-08-17T17:42:16.520212+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:42:16.520212+0000","last_undegraded":"2024-08-17T17:42:16.520212+0000","last_fullsized":"2024-08-17T17:42:16.520212+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559161+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:42:29.559161+0000","last_peered":"2024-08-17T17:42:29.559161+0000","last_clean":"2024-08-17T17:42:29.559161+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:42:29.559161+0000","last_undegraded":"2024-08-17T17:42:29.559161+0000","last_fullsized":"2024-08-17T17:42:29.559161+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":617,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.550423+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:42:29.550423+0000","last_peered":"2024-08-17T17:42:29.550423+0000","last_clean":"2024-08-17T17:42:29.550423+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:42:29.550423+0000","last_undegraded":"2024-08-17T17:42:29.550423+0000","last_fullsized":"2024-08-17T17:42:29.550423+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"390'10870","reported_seq":25444,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:35.003642+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:42:35.003642+0000","last_peered":"2024-08-17T17:42:35.003642+0000","last_clean":"2024-08-17T17:42:35.003642+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:42:35.003642+0000","last_undegraded":"2024-08-17T17:42:35.003642+0000","last_fullsized":"2024-08-17T17:42:35.003642+0000","mapping_epoch":345,"log_start":"326'4572","ondisk_log_start":"326'4572","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6298,"log_dups_size":0,"ondisk_log_size":6298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10870,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":300,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.519316+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:42:16.519316+0000","last_peered":"2024-08-17T17:42:16.519316+0000","last_clean":"2024-08-17T17:42:16.519316+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:42:16.519316+0000","last_undegraded":"2024-08-17T17:42:16.519316+0000","last_fullsized":"2024-08-17T17:42:16.519316+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":373,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.554142+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:42:29.554142+0000","last_peered":"2024-08-17T17:42:29.554142+0000","last_clean":"2024-08-17T17:42:29.554142+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:42:29.554142+0000","last_undegraded":"2024-08-17T17:42:29.554142+0000","last_fullsized":"2024-08-17T17:42:29.554142+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":442,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.519925+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:42:16.519925+0000","last_peered":"2024-08-17T17:42:16.519925+0000","last_clean":"2024-08-17T17:42:16.519925+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:42:16.519925+0000","last_undegraded":"2024-08-17T17:42:16.519925+0000","last_fullsized":"2024-08-17T17:42:16.519925+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"390'11093","reported_seq":11276,"reported_epoch":390,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:35.279191+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:42:35.279191+0000","last_peered":"2024-08-17T17:42:35.279191+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:42:35.279191+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'4787","ondisk_log_start":"327'4787","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6306,"log_dups_size":0,"ondisk_log_size":6306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241893376,"num_objects":3691,"num_object_clones":0,"num_object_copies":11073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11093,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":3691},{"shards":"0,3","objects":-3557}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":647,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559462+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:42:29.559462+0000","last_peered":"2024-08-17T17:42:29.559462+0000","last_clean":"2024-08-17T17:42:29.559462+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:42:29.559462+0000","last_undegraded":"2024-08-17T17:42:29.559462+0000","last_fullsized":"2024-08-17T17:42:29.559462+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":616,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.550724+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:42:29.550724+0000","last_peered":"2024-08-17T17:42:29.550724+0000","last_clean":"2024-08-17T17:42:29.550724+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:42:29.550724+0000","last_undegraded":"2024-08-17T17:42:29.550724+0000","last_fullsized":"2024-08-17T17:42:29.550724+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":633,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.558510+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:42:29.558510+0000","last_peered":"2024-08-17T17:42:29.558510+0000","last_clean":"2024-08-17T17:42:29.558510+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:42:29.558510+0000","last_undegraded":"2024-08-17T17:42:29.558510+0000","last_fullsized":"2024-08-17T17:42:29.558510+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"390'10922","reported_seq":25609,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:31.687038+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:42:31.687038+0000","last_peered":"2024-08-17T17:42:31.687038+0000","last_clean":"2024-08-17T17:42:31.687038+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:42:31.687038+0000","last_undegraded":"2024-08-17T17:42:31.687038+0000","last_fullsized":"2024-08-17T17:42:31.687038+0000","mapping_epoch":347,"log_start":"326'4372","ondisk_log_start":"326'4372","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239337472,"num_objects":3652,"num_object_clones":0,"num_object_copies":10956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10922,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":306,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.554808+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:42:29.554808+0000","last_peered":"2024-08-17T17:42:29.554808+0000","last_clean":"2024-08-17T17:42:29.554808+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:42:29.554808+0000","last_undegraded":"2024-08-17T17:42:29.554808+0000","last_fullsized":"2024-08-17T17:42:29.554808+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":394,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.490607+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:42:16.490607+0000","last_peered":"2024-08-17T17:42:16.490607+0000","last_clean":"2024-08-17T17:42:16.490607+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:42:16.490607+0000","last_undegraded":"2024-08-17T17:42:16.490607+0000","last_fullsized":"2024-08-17T17:42:16.490607+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":344,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499293+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:42:16.499293+0000","last_peered":"2024-08-17T17:42:16.499293+0000","last_clean":"2024-08-17T17:42:16.499293+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:42:16.499293+0000","last_undegraded":"2024-08-17T17:42:16.499293+0000","last_fullsized":"2024-08-17T17:42:16.499293+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":344,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499532+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:42:16.499532+0000","last_peered":"2024-08-17T17:42:16.499532+0000","last_clean":"2024-08-17T17:42:16.499532+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:42:16.499532+0000","last_undegraded":"2024-08-17T17:42:16.499532+0000","last_fullsized":"2024-08-17T17:42:16.499532+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":387,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499657+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:42:16.499657+0000","last_peered":"2024-08-17T17:42:16.499657+0000","last_clean":"2024-08-17T17:42:16.499657+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:42:16.499657+0000","last_undegraded":"2024-08-17T17:42:16.499657+0000","last_fullsized":"2024-08-17T17:42:16.499657+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":622,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.550247+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:42:29.550247+0000","last_peered":"2024-08-17T17:42:29.550247+0000","last_clean":"2024-08-17T17:42:29.550247+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:42:29.550247+0000","last_undegraded":"2024-08-17T17:42:29.550247+0000","last_fullsized":"2024-08-17T17:42:29.550247+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":162,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.560212+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:42:29.560212+0000","last_peered":"2024-08-17T17:42:29.560212+0000","last_clean":"2024-08-17T17:42:29.560212+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:42:29.560212+0000","last_undegraded":"2024-08-17T17:42:29.560212+0000","last_fullsized":"2024-08-17T17:42:29.560212+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":294,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.520498+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:42:16.520498+0000","last_peered":"2024-08-17T17:42:16.520498+0000","last_clean":"2024-08-17T17:42:16.520498+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:42:16.520498+0000","last_undegraded":"2024-08-17T17:42:16.520498+0000","last_fullsized":"2024-08-17T17:42:16.520498+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":580,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559022+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:42:29.559022+0000","last_peered":"2024-08-17T17:42:29.559022+0000","last_clean":"2024-08-17T17:42:29.559022+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:42:29.559022+0000","last_undegraded":"2024-08-17T17:42:29.559022+0000","last_fullsized":"2024-08-17T17:42:29.559022+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":438,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.521299+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:42:16.521299+0000","last_peered":"2024-08-17T17:42:16.521299+0000","last_clean":"2024-08-17T17:42:16.521299+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:42:16.521299+0000","last_undegraded":"2024-08-17T17:42:16.521299+0000","last_fullsized":"2024-08-17T17:42:16.521299+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":574,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.558406+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:42:29.558406+0000","last_peered":"2024-08-17T17:42:29.558406+0000","last_clean":"2024-08-17T17:42:29.558406+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:42:29.558406+0000","last_undegraded":"2024-08-17T17:42:29.558406+0000","last_fullsized":"2024-08-17T17:42:29.558406+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":361,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.490538+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:42:16.490538+0000","last_peered":"2024-08-17T17:42:16.490538+0000","last_clean":"2024-08-17T17:42:16.490538+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:42:16.490538+0000","last_undegraded":"2024-08-17T17:42:16.490538+0000","last_fullsized":"2024-08-17T17:42:16.490538+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":606,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.558585+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:42:29.558585+0000","last_peered":"2024-08-17T17:42:29.558585+0000","last_clean":"2024-08-17T17:42:29.558585+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:42:29.558585+0000","last_undegraded":"2024-08-17T17:42:29.558585+0000","last_fullsized":"2024-08-17T17:42:29.558585+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":322,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559698+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:42:29.559698+0000","last_peered":"2024-08-17T17:42:29.559698+0000","last_clean":"2024-08-17T17:42:29.559698+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:42:29.559698+0000","last_undegraded":"2024-08-17T17:42:29.559698+0000","last_fullsized":"2024-08-17T17:42:29.559698+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":614,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.550053+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:42:29.550053+0000","last_peered":"2024-08-17T17:42:29.550053+0000","last_clean":"2024-08-17T17:42:29.550053+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:42:29.550053+0000","last_undegraded":"2024-08-17T17:42:29.550053+0000","last_fullsized":"2024-08-17T17:42:29.550053+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":573,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.560238+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:42:29.560238+0000","last_peered":"2024-08-17T17:42:29.560238+0000","last_clean":"2024-08-17T17:42:29.560238+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:42:29.560238+0000","last_undegraded":"2024-08-17T17:42:29.560238+0000","last_fullsized":"2024-08-17T17:42:29.560238+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":373,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499466+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:42:16.499466+0000","last_peered":"2024-08-17T17:42:16.499466+0000","last_clean":"2024-08-17T17:42:16.499466+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:42:16.499466+0000","last_undegraded":"2024-08-17T17:42:16.499466+0000","last_fullsized":"2024-08-17T17:42:16.499466+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":262,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559417+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:42:29.559417+0000","last_peered":"2024-08-17T17:42:29.559417+0000","last_clean":"2024-08-17T17:42:29.559417+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:42:29.559417+0000","last_undegraded":"2024-08-17T17:42:29.559417+0000","last_fullsized":"2024-08-17T17:42:29.559417+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":319,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.554046+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:42:29.554046+0000","last_peered":"2024-08-17T17:42:29.554046+0000","last_clean":"2024-08-17T17:42:29.554046+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:42:29.554046+0000","last_undegraded":"2024-08-17T17:42:29.554046+0000","last_fullsized":"2024-08-17T17:42:29.554046+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":381,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499503+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:42:16.499503+0000","last_peered":"2024-08-17T17:42:16.499503+0000","last_clean":"2024-08-17T17:42:16.499503+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:42:16.499503+0000","last_undegraded":"2024-08-17T17:42:16.499503+0000","last_fullsized":"2024-08-17T17:42:16.499503+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":244,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499587+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:42:16.499587+0000","last_peered":"2024-08-17T17:42:16.499587+0000","last_clean":"2024-08-17T17:42:16.499587+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:42:16.499587+0000","last_undegraded":"2024-08-17T17:42:16.499587+0000","last_fullsized":"2024-08-17T17:42:16.499587+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":381,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499330+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:42:16.499330+0000","last_peered":"2024-08-17T17:42:16.499330+0000","last_clean":"2024-08-17T17:42:16.499330+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:42:16.499330+0000","last_undegraded":"2024-08-17T17:42:16.499330+0000","last_fullsized":"2024-08-17T17:42:16.499330+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":244,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499885+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:42:16.499885+0000","last_peered":"2024-08-17T17:42:16.499885+0000","last_clean":"2024-08-17T17:42:16.499885+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:42:16.499885+0000","last_undegraded":"2024-08-17T17:42:16.499885+0000","last_fullsized":"2024-08-17T17:42:16.499885+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":626,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559663+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:42:29.559663+0000","last_peered":"2024-08-17T17:42:29.559663+0000","last_clean":"2024-08-17T17:42:29.559663+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:42:29.559663+0000","last_undegraded":"2024-08-17T17:42:29.559663+0000","last_fullsized":"2024-08-17T17:42:29.559663+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":599,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.559865+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:42:29.559865+0000","last_peered":"2024-08-17T17:42:29.559865+0000","last_clean":"2024-08-17T17:42:29.559865+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:42:29.559865+0000","last_undegraded":"2024-08-17T17:42:29.559865+0000","last_fullsized":"2024-08-17T17:42:29.559865+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":344,"reported_epoch":389,"state":"active+clean","last_fresh":"2024-08-17T17:42:16.499139+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:42:16.499139+0000","last_peered":"2024-08-17T17:42:16.499139+0000","last_clean":"2024-08-17T17:42:16.499139+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:42:16.499139+0000","last_undegraded":"2024-08-17T17:42:16.499139+0000","last_fullsized":"2024-08-17T17:42:16.499139+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":321,"reported_epoch":390,"state":"active+clean","last_fresh":"2024-08-17T17:42:29.554419+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:42:29.554419+0000","last_peered":"2024-08-17T17:42:29.554419+0000","last_clean":"2024-08-17T17:42:29.554419+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:42:29.554419+0000","last_undegraded":"2024-08-17T17:42:29.554419+0000","last_fullsized":"2024-08-17T17:42:29.554419+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3864133656,"num_objects":58963,"num_object_clones":0,"num_object_copies":176889,"num_objects_missing_on_primary":4322,"num_objects_missing":4322,"num_objects_degraded":51724,"num_objects_misplaced":23659,"num_objects_unfound":0,"num_objects_dirty":58963,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":177427,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31279,"num_bytes_recovered":1927348224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13624225792,"data_stored":13624213576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104646,"ondisk_log_size":104646,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170114,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463483,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2566904,"kb_used_data":1669040,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":91178248,"statfs":{"total":95995035648,"available":93366525952,"internally_reserved":0,"allocated":1709096960,"data_stored":1706957451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492158,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":869340,"kb_used_data":474004,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92875812,"statfs":{"total":95995035648,"available":95104831488,"internally_reserved":0,"allocated":485380096,"data_stored":483262887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986749,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2449004,"kb_used_data":940324,"kb_used_omap":1,"kb_used_meta":1508670,"kb_avail":91296148,"statfs":{"total":95995035648,"available":93487255552,"internally_reserved":0,"allocated":962891776,"data_stored":960763207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1544878424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225577,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2832796,"kb_used_data":1880980,"kb_used_omap":4,"kb_used_meta":951803,"kb_avail":90912356,"statfs":{"total":95995035648,"available":93094252544,"internally_reserved":0,"allocated":1926123520,"data_stored":1924006609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":974646720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075902,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533795,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2708640,"kb_used_data":1656920,"kb_used_omap":5,"kb_used_meta":1051706,"kb_avail":91036512,"statfs":{"total":95995035648,"available":93221388288,"internally_reserved":0,"allocated":1696686080,"data_stored":1694563847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1076947184},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089772,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2424888,"kb_used_data":1419824,"kb_used_omap":5,"kb_used_meta":1005050,"kb_avail":91320264,"statfs":{"total":95995035648,"available":93511950336,"internally_reserved":0,"allocated":1453899776,"data_stored":1451760326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1029171259},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1446510592,"data_stored":1446510592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1691226112,"data_stored":1691222040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1919811584,"data_stored":1919811584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":955318272,"data_stored":955318272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":479657984,"data_stored":479657984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1702166528,"data_stored":1702166528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2090274816,"data_stored":2090270744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:42:38.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:42:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:38 smithi152 ceph-mon[93456]: from='client.37746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:42:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:38 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:38.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:38.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:38.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:38.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:38.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:38.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:39.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:39.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:39.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.241 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:39.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:39.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:39.365 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:39.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:39.476 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:39 smithi142 ceph-mon[106729]: pgmap v1492: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 51727/176013 objects degraded (29.388%); 23614/176013 objects misplaced (13.416%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:39.476 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:39 smithi142 ceph-mon[103614]: pgmap v1492: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 51727/176013 objects degraded (29.388%); 23614/176013 objects misplaced (13.416%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:39.490 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:39.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:39.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:39.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:39.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:39.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:39 smithi152 ceph-mon[93456]: pgmap v1492: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 51727/176013 objects degraded (29.388%); 23614/176013 objects misplaced (13.416%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:42:39.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:39.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:39.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:39.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:39.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:39.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:40.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:40.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:40.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:40.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:40.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:40.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:40.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:40.439 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:42:40.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:40.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:40.522 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2557723902' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:40.522 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:40 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:40.522 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:42:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:42:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:40.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:40.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:40.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:40.748 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:40.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:40.862 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2557723902' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:40.862 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:40 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:40.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2557723902' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:40.862 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:40 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:42:40.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:40.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:40.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:41.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:41.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:41.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:41.113 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:41.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:41.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:41.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:41.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:41.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:41.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:41.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:41.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:41.532 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:41 smithi152 ceph-mon[93456]: pgmap v1493: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 51631/174846 objects degraded (29.529%); 23367/174846 objects misplaced (13.364%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:42:41.532 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:41 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:41.532 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:41 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:42:41.533 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:41 smithi152 ceph-mon[93456]: osdmap e391: 8 total, 8 up, 6 in 2024-08-17T17:42:41.533 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51631/174846 objects degraded (29.529%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:41.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[103614]: pgmap v1493: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 51631/174846 objects degraded (29.529%); 23367/174846 objects misplaced (13.364%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:42:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:42:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[103614]: osdmap e391: 8 total, 8 up, 6 in 2024-08-17T17:42:41.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51631/174846 objects degraded (29.529%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[106729]: pgmap v1493: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 51631/174846 objects degraded (29.529%); 23367/174846 objects misplaced (13.364%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:42:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:42:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:42:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[106729]: osdmap e391: 8 total, 8 up, 6 in 2024-08-17T17:42:41.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51631/174846 objects degraded (29.529%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:41.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:41.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:41.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:41.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:41.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:41.880 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:41.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:41.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:41.947 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:42:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:42:41] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:42:41.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:42.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:42.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:42.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.129 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:42.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:42.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:42.251 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:42.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:42.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:42.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:42.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:42.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:42.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:42.728 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:42.752 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:42.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:42.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:42.796 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:42.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:42.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:42.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:42.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:42.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:43.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:43.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:43.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:43.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:43.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:43.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:43.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:43.256 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:42:43.256 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:42:43.257 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:43.257 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:43.264 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:42:43.265 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:43.280 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:42:43.280 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:43.296 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:42:43.296 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:43.313 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:42:43.313 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:43.328 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:42:43.329 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:43.344 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:42:43.344 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:43.356 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:43.377 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:43.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:43.479 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:43.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:43.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:43.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:43.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:42:43.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:43.741 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:43 smithi142 ceph-mon[106729]: pgmap v1495: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 51631/174156 objects degraded (29.646%); 23309/174156 objects misplaced (13.384%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:43.741 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3604481649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:43.741 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:43 smithi142 ceph-mon[103614]: pgmap v1495: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 51631/174156 objects degraded (29.646%); 23309/174156 objects misplaced (13.384%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:43.742 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3604481649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:43.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:43 smithi152 ceph-mon[93456]: pgmap v1495: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 51631/174156 objects degraded (29.646%); 23309/174156 objects misplaced (13.384%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3604481649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:43.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:43.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:43.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:43.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:44.008 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:44.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:44.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:44.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:44.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:44.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:44.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:44.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:44.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:44.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:44.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:44.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:44.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:44.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:44.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:44 smithi152 ceph-mon[93456]: pgmap v1496: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 51580/173556 objects degraded (29.720%); 23230/173556 objects misplaced (13.385%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:44.852 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:44 smithi142 ceph-mon[106729]: pgmap v1496: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 51580/173556 objects degraded (29.720%); 23230/173556 objects misplaced (13.385%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:44 smithi142 ceph-mon[103614]: pgmap v1496: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 51580/173556 objects degraded (29.720%); 23230/173556 objects misplaced (13.385%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:44.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:44.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:44.884 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:44.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:44.998 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:44.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:44.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:45.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:45.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:45.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:45.226 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:45.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:45.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:45.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:45.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:45.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:45.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:45.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:45.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:45.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:45.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:45.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:45.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:45.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:45.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:45.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:45.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:45.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:45.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:46.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51580/173556 objects degraded (29.720%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:46.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:46.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51580/173556 objects degraded (29.720%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:46.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51580/173556 objects degraded (29.720%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:46.123 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:46.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:46.245 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:42:46.245 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:42:46.245 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:42:46.245 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:112415 12.4 e330/319 12:226a051b:::benchmark_data_smithi142_261410_object112414:head v 330'6931, mlcod=330'6928)", 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.971707+0000", 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.27178154000001, 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34399211899999999, 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.246 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971707+0000", 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971707+0000", 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971708+0000", 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0950000000000001e-06 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.247 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971781+0000", 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2865000000000001e-05 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971793+0000", 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1956999999999999e-05 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.248 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971800+0000", 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1509999999999998e-06 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971860+0000", 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9917999999999998e-05 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.971883+0000", 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2957000000000001e-05 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.249 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315643+0000", 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34376013399999999 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315699+0000", 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6042000000000003e-05 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.250 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112420 12.6 12:61ca06b2:::benchmark_data_smithi142_261410_object112419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.974860+0000", 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26862915200002, 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34479163299999999, 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.251 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112420 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974860+0000", 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974860+0000", 2024-08-17T17:42:46.252 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974862+0000", 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.582e-06 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974895+0000", 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2369999999999997e-05 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.253 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974911+0000", 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6665000000000001e-05 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.974921+0000", 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7599999999999997e-06 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976156+0000", 2024-08-17T17:42:46.254 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001235014 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976372+0000", 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000216145 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976513+0000", 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014044799999999999 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.255 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319515+0000", 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34300181600000001 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319525+0000", 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0505e-05 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319558+0000", 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2926000000000003e-05 2024-08-17T17:42:46.256 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319607+0000", 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9308999999999998e-05 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319651+0000", 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4092999999999999e-05 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.257 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:112418 12.7 e330/319 12:e0d90dce:::benchmark_data_smithi142_261410_object112417:head v 330'7009, mlcod=330'7006)", 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.975051+0000", 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26843816899998, 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33773811199999998, 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975051+0000", 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.258 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975051+0000", 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975052+0000", 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.082e-06 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.259 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975138+0000", 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6225999999999997e-05 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975153+0000", 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4523e-05 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975160+0000", 2024-08-17T17:42:46.260 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2710000000000003e-06 2024-08-17T17:42:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975262+0000", 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010202800000000001 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.262 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975283+0000", 2024-08-17T17:42:46.263 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0791000000000001e-05 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312735+0000", 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33745272399999998 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312789+0000", 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3467000000000002e-05 2024-08-17T17:42:46.264 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:112419 12.4 e330/319 12:213f767a:::benchmark_data_smithi142_261410_object112418:head v 330'6932, mlcod=330'6928)", 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.975189+0000", 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26829983699997, 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34409234900000002, 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.265 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975189+0000", 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975189+0000", 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.266 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975190+0000", 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9999999999999997e-07 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975245+0000", 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5454000000000001e-05 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975256+0000", 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0832000000000001e-05 2024-08-17T17:42:46.267 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975263+0000", 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312755+0000", 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33749242299999999 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.268 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312789+0000", 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3392999999999997e-05 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319234+0000", 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064450030000000004 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319281+0000", 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7772000000000003e-05 2024-08-17T17:42:46.269 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112421 12.6 12:6aa59c45:::benchmark_data_smithi142_261410_object112420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.975254+0000", 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26823500900002, 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35309264099999998, 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.270 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112421 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975254+0000", 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.271 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975254+0000", 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975255+0000", 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.5199999999999995e-07 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975276+0000", 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1350999999999999e-05 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975288+0000", 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1435e-05 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.975294+0000", 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7320000000000001e-06 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315710+0000", 2024-08-17T17:42:46.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34041555600000001 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.315859+0000", 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149598 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.316011+0000", 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000151278 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328095+0000", 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012084594000000001 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328130+0000", 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5156e-05 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328248+0000", 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011713 2024-08-17T17:42:46.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328300+0000", 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2113000000000001e-05 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328347+0000", 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6845999999999998e-05 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112424 12.6 12:645c2f85:::benchmark_data_smithi142_261410_object112423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.976252+0000", 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26723695199996, 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35217241199999999, 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112424 2024-08-17T17:42:46.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976252+0000", 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976252+0000", 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976253+0000", 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.09e-07 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976272+0000", 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9545e-05 2024-08-17T17:42:46.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976281+0000", 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6240000000000008e-06 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.976287+0000", 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0290000000000002e-06 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319683+0000", 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34339546700000001 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319831+0000", 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014813599999999999 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.320195+0000", 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00036470099999999999 2024-08-17T17:42:46.281 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328189+0000", 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079938449999999994 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328238+0000", 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8869999999999998e-05 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328354+0000", 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116296 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328395+0000", 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0342000000000001e-05 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.328424+0000", 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9648000000000001e-05 2024-08-17T17:42:46.283 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:112426 12.6 12:6a0dc15b:::benchmark_data_smithi142_261410_object112425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.979459+0000", 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26403016400002, 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36174886099999998, 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.284 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 112426 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979459+0000", 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979459+0000", 2024-08-17T17:42:46.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979460+0000", 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.765e-06 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.979485+0000", 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4813000000000001e-05 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312692+0000", 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33320643500000002 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312710+0000", 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7818000000000001e-05 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.323866+0000", 2024-08-17T17:42:46.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011156919 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.324047+0000", 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000180514 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.324182+0000", 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013546000000000001 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.340995+0000", 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016812420000000002 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341058+0000", 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3223000000000001e-05 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341106+0000", 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7602000000000003e-05 2024-08-17T17:42:46.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341164+0000", 2024-08-17T17:42:46.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8125999999999999e-05 2024-08-17T17:42:46.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.341208+0000", 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3766000000000001e-05 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:112420 12.6 e330/319 ondisk, result = 0)", 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:09.980350+0000", 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 576.26313857499997, 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.33917538400000002, 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980350+0000", 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980350+0000", 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980351+0000", 2024-08-17T17:42:46.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.032e-06 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:09.980356+0000", 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9729999999999997e-06 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312695+0000", 2024-08-17T17:42:46.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.332338945 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.312708+0000", 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2778e-05 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319502+0000", 2024-08-17T17:42:46.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067940639999999998 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319509+0000", 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4869999999999999e-06 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:10.319526+0000", 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7105e-05 2024-08-17T17:42:46.296 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124351 12.6 12:637584c3:::benchmark_data_smithi142_261410_object6154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.812962+0000", 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 556.43052696699999, 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36987189399999998, 2024-08-17T17:42:46.297 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124351 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812962+0000", 2024-08-17T17:42:46.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812962+0000", 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812963+0000", 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.39e-06 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812968+0000", 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.561e-06 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812978+0000", 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0335e-05 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.812984+0000", 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7740000000000001e-06 2024-08-17T17:42:46.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.813573+0000", 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058854999999999997 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.813702+0000", 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012960900000000001 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.813820+0000", 2024-08-17T17:42:46.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117624 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818616+0000", 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0047958339999999997 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.822350+0000", 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037340160000000002 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182730+0000", 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36038021100000001 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182784+0000", 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4246999999999997e-05 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182834+0000", 2024-08-17T17:42:46.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9743000000000001e-05 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124355 12.6 12:6a89de93:::benchmark_data_smithi142_261410_object6158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.818633+0000", 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 556.42485541300005, 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36461828400000001, 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.304 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124355 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818633+0000", 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818633+0000", 2024-08-17T17:42:46.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818636+0000", 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1849999999999998e-06 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818641+0000", 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4960000000000004e-06 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818654+0000", 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3348e-05 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818663+0000", 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6689999999999995e-06 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818718+0000", 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4656e-05 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818847+0000", 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012894499999999999 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.818974+0000", 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000127284 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.822364+0000", 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003390311 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.826234+0000", 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038695719999999999 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182872+0000", 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35663769899999997 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183211+0000", 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00033976399999999999 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183252+0000", 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0355000000000002e-05 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.310 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPush(12.a 335/332 [PushOp(12:540a96b7:::benchmark_data_smithi142_261410_object6169:head, version: 323'409, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:540a96b7:::benchmark_data_smithi142_261410_object6169:head@323'409, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.830075+0000", 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 556.41341382899998, 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35255843799999997, 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830075+0000", 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830075+0000", 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830076+0000", 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2410000000000001e-06 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830106+0000", 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9629e-05 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830126+0000", 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9961000000000001e-05 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.830140+0000", 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4003e-05 2024-08-17T17:42:46.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.831209+0000", 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010689160000000001 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.831220+0000", 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0869e-05 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.314 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182633+0000", 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35141381900000002 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:124358 12.7 e335/332 12:ee40e7c7:::benchmark_data_smithi142_261410_object6161:head v 335'7760, mlcod=322'128)", 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.834295+0000", 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 556.409193695, 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.36059230599999997, 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.315 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834295+0000", 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834295+0000", 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834296+0000", 2024-08-17T17:42:46.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.145e-06 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834306+0000", 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1819999999999995e-06 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834321+0000", 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5634999999999999e-05 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.834330+0000", 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2949999999999999e-06 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182530+0000", 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.34819973700000001 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182567+0000", 2024-08-17T17:42:46.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7032000000000001e-05 2024-08-17T17:42:46.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194840+0000", 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012272857 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194887+0000", 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7423000000000003e-05 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:124351 12.6 e335/319 ondisk, result = 0)", 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.839700+0000", 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 556.40378906599994, 2024-08-17T17:42:46.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.343139741, 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839700+0000", 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839700+0000", 2024-08-17T17:42:46.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839701+0000", 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.06e-06 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.839707+0000", 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8340000000000004e-06 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182642+0000", 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.342935406 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182655+0000", 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3193999999999999e-05 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182716+0000", 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0647999999999999e-05 2024-08-17T17:42:46.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182724+0000", 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8290000000000008e-06 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182840+0000", 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011577 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.329 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:124369 12.6 12:6623c19c:::benchmark_data_smithi142_261410_object6172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-08-17T17:42:46.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:46.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:46.330 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:46.330 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:33:29.843595+0000", 2024-08-17T17:42:46.330 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 556.39989424600003, 2024-08-17T17:42:46.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.35119447599999998, 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124369 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843595+0000", 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843595+0000", 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843596+0000", 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:29.843604+0000", 2024-08-17T17:42:46.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3780000000000003e-06 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182660+0000", 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.339056248 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.182673+0000", 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2995e-05 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183281+0000", 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00060766499999999996 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183414+0000", 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013326000000000001 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.183536+0000", 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012200499999999999 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.191414+0000", 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078783220000000001 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.192455+0000", 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010408139999999999 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194685+0000", 2024-08-17T17:42:46.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0022300499999999999 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194741+0000", 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6051000000000003e-05 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:33:30.194789+0000", 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7958999999999999e-05 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146414 12.d 12:bfc2c874:::benchmark_data_smithi142_261410_object28217:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:59.816471+0000", 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 286.42701821899999, 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.062425194000000003, 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.337 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146414 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.816471+0000", 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.816471+0000", 2024-08-17T17:42:46.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.816473+0000", 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.962e-06 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.816479+0000", 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0360000000000003e-06 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.816495+0000", 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6773999999999999e-05 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.816504+0000", 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.191e-06 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.832690+0000", 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016186850999999999 2024-08-17T17:42:46.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.833313+0000", 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00062268499999999999 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.844554+0000", 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011241007000000001 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.844658+0000", 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103794 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.844822+0000", 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000163585 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.862814+0000", 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017992241999999999 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.875019+0000", 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012205524000000001 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.875085+0000", 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5717999999999997e-05 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.875128+0000", 2024-08-17T17:42:46.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2636999999999999e-05 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.875169+0000", 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1431999999999999e-05 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.878896+0000", 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037267559999999999 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.344 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146415 12.d 12:b23fb380:::benchmark_data_smithi142_261410_object28218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:59.825116+0000", 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 286.41837317599999, 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.058409067000000002, 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.345 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146415 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.825116+0000", 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.825116+0000", 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.825117+0000", 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.519e-06 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.825123+0000", 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9900000000000002e-06 2024-08-17T17:42:46.347 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.825137+0000", 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3929e-05 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.825145+0000", 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6669999999999996e-06 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.833355+0000", 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0082098569999999992 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.833885+0000", 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00052990600000000002 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.853639+0000", 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019754236000000001 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.853751+0000", 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0001126 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.853923+0000", 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000171674 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.871578+0000", 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017654982999999999 2024-08-17T17:42:46.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.878962+0000", 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0073839489999999999 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.879005+0000", 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2672000000000001e-05 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.879063+0000", 2024-08-17T17:42:46.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8297000000000001e-05 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.879132+0000", 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9018999999999998e-05 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.883525+0000", 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0043927690000000004 2024-08-17T17:42:46.352 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146417 12.d 12:b1f3b9ed:::benchmark_data_smithi142_261410_object28220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:59.828680+0000", 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 286.41480924299998, 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.058915744999999999, 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.353 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146417 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.828680+0000", 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.828680+0000", 2024-08-17T17:42:46.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.828681+0000", 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3790000000000001e-06 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.828686+0000", 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9439999999999996e-06 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.355 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.828700+0000", 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4346e-05 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.828708+0000", 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4749999999999996e-06 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.833928+0000", 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052205760000000002 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.356 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.834522+0000", 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059342100000000001 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.862860+0000", 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.028338341999999999 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.862986+0000", 2024-08-17T17:42:46.357 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000126303 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.863156+0000", 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016952800000000001 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.875030+0000", 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011874146 2024-08-17T17:42:46.358 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.883611+0000", 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0085812820000000008 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.883665+0000", 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3273999999999998e-05 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.883709+0000", 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4255000000000002e-05 2024-08-17T17:42:46.359 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.883755+0000", 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5841999999999999e-05 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.887595+0000", 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038406320000000001 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.360 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146418 12.d 12:bec73b36:::benchmark_data_smithi142_261410_object28221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:59.836038+0000", 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 286.40745106700001, 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.056141901000000001, 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146418 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.361 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836038+0000", 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836038+0000", 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836040+0000", 2024-08-17T17:42:46.362 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7600000000000001e-06 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836046+0000", 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1800000000000001e-06 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836063+0000", 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6991e-05 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.363 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836076+0000", 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3542e-05 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836134+0000", 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8208999999999998e-05 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.836778+0000", 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00064338000000000004 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.364 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.871631+0000", 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034852718999999997 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.871743+0000", 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112323 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.871946+0000", 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020342599999999999 2024-08-17T17:42:46.365 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.879018+0000", 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0070716310000000001 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.887652+0000", 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0086344439999999998 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.366 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.887745+0000", 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2736000000000001e-05 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.887851+0000", 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000106325 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.367 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.887911+0000", 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9111999999999999e-05 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:59.892180+0000", 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042691229999999997 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.368 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:175744 12.d 12:b6b60f07:::benchmark_data_smithi142_261410_object57547:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:22.442551+0000", 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 23.800938207000002, 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.034629578000000001, 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 175744 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.369 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.442551+0000", 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.442551+0000", 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.370 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.442551+0000", 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0800000000000005e-07 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.442554+0000", 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3769999999999999e-06 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.442560+0000", 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.144e-06 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.371 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.442564+0000", 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5220000000000001e-06 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.447308+0000", 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0047435079999999996 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.447973+0000", 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00066511800000000002 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.372 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.463643+0000", 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015670396 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.463710+0000", 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7065999999999998e-05 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.463817+0000", 2024-08-17T17:42:46.373 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010655899999999999 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.469055+0000", 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005237813 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.469118+0000", 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2990999999999994e-05 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.374 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.469186+0000", 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8740999999999998e-05 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.473791+0000", 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046045330000000001 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.375 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.473854+0000", 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3373000000000005e-05 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.477180+0000", 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003325929 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176636 12.d 12:b0a12c4e:::benchmark_data_smithi142_261410_object58439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:42:46.376 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:29.338257+0000", 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 16.905232045999998, 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027734411, 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176636 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.377 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.338257+0000", 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.338257+0000", 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.338258+0000", 2024-08-17T17:42:46.378 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.556e-06 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.338264+0000", 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9109999999999998e-06 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.338278+0000", 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3431000000000001e-05 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.379 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.338285+0000", 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8959999999999997e-06 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.342703+0000", 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044187740000000003 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.343267+0000", 2024-08-17T17:42:46.380 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000563646 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.356063+0000", 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012795992000000001 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.356176+0000", 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112973 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:42:46.381 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.356339+0000", 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016332299999999999 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.361405+0000", 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005065328 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.361424+0000", 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9239999999999999e-05 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.382 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.361470+0000", 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6357999999999999e-05 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.362628+0000", 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011579660000000001 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.362693+0000", 2024-08-17T17:42:46.383 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4819000000000004e-05 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.365991+0000", 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003298198 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:42:46.384 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:42:46.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:46.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:46.557 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:46.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:42:46.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:46.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:46.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:46.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:46.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:46.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:46.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:46.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:47.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:46 smithi152 ceph-mon[93456]: pgmap v1497: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 51537/172797 objects degraded (29.825%); 23096/172797 objects misplaced (13.366%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:47.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:47.082 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:46 smithi142 ceph-mon[106729]: pgmap v1497: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 51537/172797 objects degraded (29.825%); 23096/172797 objects misplaced (13.366%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:47.083 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:46 smithi142 ceph-mon[103614]: pgmap v1497: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 51537/172797 objects degraded (29.825%); 23096/172797 objects misplaced (13.366%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:47.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:47.205 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:47.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:47.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:47.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:47.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:47.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:47.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:47.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:47.710 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:47.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:47.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:47.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:47.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:47.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:47.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:48.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:48.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:48.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:48.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:48.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:48.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:48.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:48.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:48.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:48.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:42:48.467 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:48.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:48.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:48.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:48.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:48.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:48.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:48.853 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:48.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:48.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:48.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:49.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:49.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:49.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:49.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:49.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:49.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:49.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:49.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:49.372 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:42:49.372 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 51537/171633 objects degraded (30.027%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2249,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":391,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":57211,"data_bytes":3748206565,"bytes_used":13963055104,"bytes_avail":562007158784,"bytes_total":575970213888,"degraded_objects":51537,"degraded_total":171633,"degraded_ratio":0.30027442275086957,"misplaced_objects":23059,"misplaced_total":171633,"misplaced_ratio":0.13435062021872252,"recovering_objects_per_sec":28,"recovering_bytes_per_sec":1867260,"recovering_keys_per_sec":0,"num_objects_recovered":285,"num_bytes_recovered":18677760,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":145},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:42:49.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:49.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:49.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:49.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:49 smithi142 ceph-mon[106729]: pgmap v1498: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51537/171633 objects degraded (30.027%); 23059/171633 objects misplaced (13.435%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:49.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/903989976' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:49.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:49 smithi142 ceph-mon[103614]: pgmap v1498: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51537/171633 objects degraded (30.027%); 23059/171633 objects misplaced (13.435%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:49.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/903989976' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:49.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:49.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:49 smithi152 ceph-mon[93456]: pgmap v1498: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51537/171633 objects degraded (30.027%); 23059/171633 objects misplaced (13.435%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:42:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/903989976' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:42:49.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:49.847 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:49.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:49.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:49.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:50.022 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:50.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:50.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:50.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:50.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:50.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:50.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:50.453 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:50.477 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:50.531 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:42:50.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:42:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:42:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:42:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:50.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:50.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:50.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:50.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:50.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:50.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:50.950 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:50.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:50.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:51.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:51.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:51.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:51.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:51.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:51.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:51.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:51.482 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:51.584 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:51.605 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:51 smithi142 ceph-mon[106729]: pgmap v1499: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 51438/170304 objects degraded (30.204%); 22791/170304 objects misplaced (13.383%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:42:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51438/170304 objects degraded (30.204%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:51 smithi142 ceph-mon[103614]: pgmap v1499: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 51438/170304 objects degraded (30.204%); 22791/170304 objects misplaced (13.383%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:42:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51438/170304 objects degraded (30.204%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:51.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:51.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:51 smithi152 ceph-mon[93456]: pgmap v1499: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 51438/170304 objects degraded (30.204%); 22791/170304 objects misplaced (13.383%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:42:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51438/170304 objects degraded (30.204%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:51.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:51.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:51.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:51.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:51.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:52.064 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:42:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:42:51] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:42:52.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:52.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:52.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:52.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:52.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:52.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:52.356 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:52.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:52.480 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:52.582 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:52.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:52.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:52.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:52.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:52.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:52.737 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:52.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:52.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:52.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:52.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:52.977 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:52.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:53.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:53.100 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:53.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:53.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:53.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:53.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:53.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:53.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:53.430 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:42:53.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:53.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:53.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:53.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:53.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:53.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:53.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:53 smithi152 ceph-mon[93456]: pgmap v1500: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 51438/169374 objects degraded (30.369%); 22736/169374 objects misplaced (13.424%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:42:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2701515233' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:53 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:53.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:53.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:53 smithi142 ceph-mon[106729]: pgmap v1500: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 51438/169374 objects degraded (30.369%); 22736/169374 objects misplaced (13.424%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:42:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2701515233' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:53 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:53 smithi142 ceph-mon[103614]: pgmap v1500: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 51438/169374 objects degraded (30.369%); 22736/169374 objects misplaced (13.424%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:42:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2701515233' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:53 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:42:53.950 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:53.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:53.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:53.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:54.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:54.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:54.197 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:54.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:54.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:54.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:54.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:54.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:54.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:54.513 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:54 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:54.513 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:54 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:54.513 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:54 smithi142 ceph-mon[106729]: osdmap e392: 8 total, 8 up, 6 in 2024-08-17T17:42:54.513 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:54 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:54.514 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:54 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:54.514 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:54 smithi142 ceph-mon[103614]: osdmap e392: 8 total, 8 up, 6 in 2024-08-17T17:42:54.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:54.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:54.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:54.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:54.821 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:54 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:42:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:54 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:42:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:54 smithi152 ceph-mon[93456]: osdmap e392: 8 total, 8 up, 6 in 2024-08-17T17:42:54.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:54.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:54.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:54.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:54.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:54.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:55.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:55.072 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:55.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:55.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:55.214 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:55.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:55.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:55.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:55.462 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:55.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:55.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.586 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:55.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:55.639 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:55 smithi142 ceph-mon[106729]: pgmap v1502: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 51383/168570 objects degraded (30.482%); 22632/168570 objects misplaced (13.426%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:42:55.639 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:55 smithi142 ceph-mon[103614]: pgmap v1502: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 51383/168570 objects degraded (30.482%); 22632/168570 objects misplaced (13.426%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:42:55.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:55.711 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:55.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:55 smithi152 ceph-mon[93456]: pgmap v1502: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 51383/168570 objects degraded (30.482%); 22632/168570 objects misplaced (13.426%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:42:55.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:55.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:55.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:55.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:55.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:56.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:56.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:56.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:56.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:56.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:56.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:56.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:56.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:56.418 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:42:56.418 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:42:56.418 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:42:56.418 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:42:56.418 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:42:56.419 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:42:56.420 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:42:56.427 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:42:56.427 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:42:56.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:56.461 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:56.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:56.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51383/168570 objects degraded (30.482%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:56.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/252815486' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51383/168570 objects degraded (30.482%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:56.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/252815486' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:56.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:56.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:56.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51383/168570 objects degraded (30.482%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:42:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/252815486' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:42:56.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:56.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:56.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:56.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:57.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:57.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:57.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:57.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:57.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:57.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:57.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:57.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.462 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:57.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:57.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:57.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:57.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:57.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:57.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:57.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:42:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:57 smithi152 ceph-mon[93456]: pgmap v1503: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 51338/167601 objects degraded (30.631%); 22482/167601 objects misplaced (13.414%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:57.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:42:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:57 smithi142 ceph-mon[106729]: pgmap v1503: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 51338/167601 objects degraded (30.631%); 22482/167601 objects misplaced (13.414%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:57 smithi142 ceph-mon[103614]: pgmap v1503: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 51338/167601 objects degraded (30.631%); 22482/167601 objects misplaced (13.414%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:42:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:57.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:57.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:57.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:58.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:42:58.090 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:42:58.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:58.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:58.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:58.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:58.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:58.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:58.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:58.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:58.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:58.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:58.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:58.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:58.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:58.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:58.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:42:58.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:58.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:42:59.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:59.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:42:59.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:42:59.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:42:59.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:59.342 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:59.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:59.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:59.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:59.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:42:59.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:59.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:42:59.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:42:59.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:42:59 smithi152 ceph-mon[93456]: pgmap v1504: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 51338/166524 objects degraded (30.829%); 22429/166524 objects misplaced (13.469%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:59.716 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:42:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:42:59.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:42:59.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:42:59 smithi142 ceph-mon[103614]: pgmap v1504: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 51338/166524 objects degraded (30.829%); 22429/166524 objects misplaced (13.469%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:59.860 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:42:59 smithi142 ceph-mon[106729]: pgmap v1504: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 51338/166524 objects degraded (30.829%); 22429/166524 objects misplaced (13.469%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:42:59.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:42:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:42:59.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:42:59.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:42:59.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:43:00.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:00.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:00.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:00.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:00.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:00.322 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:43:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:43:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:00.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:00.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:00.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:00.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:00.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:00.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:00.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:00.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:00.716 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:00.716 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:00.716 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:00 smithi142 ceph-mon[103614]: pgmap v1505: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 51240/165435 objects degraded (30.973%); 22199/165435 objects misplaced (13.419%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:00.716 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:00.716 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:00.716 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:00 smithi142 ceph-mon[106729]: pgmap v1505: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 51240/165435 objects degraded (30.973%); 22199/165435 objects misplaced (13.419%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:00.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:00.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:00 smithi152 ceph-mon[93456]: pgmap v1505: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 51240/165435 objects degraded (30.973%); 22199/165435 objects misplaced (13.419%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:00.824 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:00.824 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:00.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:00.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:00.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:00.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:01.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:01.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:01.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:01.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:01.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:01.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:01.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:01.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:01.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:01.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:43:01.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:01.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:01.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:01.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:01.759 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:01.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:01.864 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:01.864 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:01.865 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51240/165435 objects degraded (30.973%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:01.865 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:01.865 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:01.865 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51240/165435 objects degraded (30.973%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:01.865 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:43:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:43:01] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:43:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:01.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:01.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:01.922 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:01.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.007 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:02.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51240/165435 objects degraded (30.973%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:02.087 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:43:02.088 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:43:02.108 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:02.131 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:02.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:02.255 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:02.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:43:02.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:02.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:02.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:02.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:02.521 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:02.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:02.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:02.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:02.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:02.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:02.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:02 smithi142 ceph-mon[106729]: from='client.37782 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:02 smithi142 ceph-mon[106729]: pgmap v1506: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 51240/164739 objects degraded (31.104%); 22162/164739 objects misplaced (13.453%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:02 smithi142 ceph-mon[103614]: from='client.37782 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:02 smithi142 ceph-mon[103614]: pgmap v1506: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 51240/164739 objects degraded (31.104%); 22162/164739 objects misplaced (13.453%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:02.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:02.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:02.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:03.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:02 smithi152 ceph-mon[93456]: from='client.37782 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:02 smithi152 ceph-mon[93456]: pgmap v1506: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 51240/164739 objects degraded (31.104%); 22162/164739 objects misplaced (13.453%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:03.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:03.136 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:03.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:03.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:03.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:03.383 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1505,"stamp":"2024-08-17T17:43:00.368900+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3612809189,"num_objects":55145,"num_object_clones":0,"num_object_copies":165435,"num_objects_missing_on_primary":4089,"num_objects_missing":4089,"num_objects_degraded":51240,"num_objects_misplaced":22199,"num_objects_unfound":0,"num_objects_dirty":55145,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":181549,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32043,"num_bytes_recovered":1977098894,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104997,"ondisk_log_size":104997,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13328436,"kb_used_data":7517956,"kb_used_omap":27,"kb_used_meta":5810404,"kb_avail":549142476,"statfs":{"total":575970213888,"available":562321895424,"internally_reserved":0,"allocated":7698386944,"data_stored":7685529547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5949854440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-106364928,"num_objects":-1623,"num_object_clones":0,"num_object_copies":-4869,"num_objects_missing_on_primary":-98,"num_objects_missing":-98,"num_objects_degraded":-198,"num_objects_misplaced":-592,"num_objects_unfound":0,"num_objects_dirty":-1623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1623,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":284,"num_bytes_recovered":18612224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002713"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":627,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437248+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:42:53.437248+0000","last_peered":"2024-08-17T17:42:53.437248+0000","last_clean":"2024-08-17T17:42:53.437248+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:42:53.437248+0000","last_undegraded":"2024-08-17T17:42:53.437248+0000","last_fullsized":"2024-08-17T17:42:53.437248+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":101,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440798+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:42:40.440798+0000","last_peered":"2024-08-17T17:42:40.440798+0000","last_clean":"2024-08-17T17:42:40.440798+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:42:40.440798+0000","last_undegraded":"2024-08-17T17:42:40.440798+0000","last_fullsized":"2024-08-17T17:42:40.440798+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":295,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491659+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:42:40.491659+0000","last_peered":"2024-08-17T17:42:40.491659+0000","last_clean":"2024-08-17T17:42:40.491659+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:42:40.491659+0000","last_undegraded":"2024-08-17T17:42:40.491659+0000","last_fullsized":"2024-08-17T17:42:40.491659+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":325,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435688+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:42:53.435688+0000","last_peered":"2024-08-17T17:42:53.435688+0000","last_clean":"2024-08-17T17:42:53.435688+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:42:53.435688+0000","last_undegraded":"2024-08-17T17:42:53.435688+0000","last_fullsized":"2024-08-17T17:42:53.435688+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":502,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435384+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:42:53.435384+0000","last_peered":"2024-08-17T17:42:53.435384+0000","last_clean":"2024-08-17T17:42:53.435384+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:42:53.435384+0000","last_undegraded":"2024-08-17T17:42:53.435384+0000","last_fullsized":"2024-08-17T17:42:53.435384+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":244,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440824+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:42:40.440824+0000","last_peered":"2024-08-17T17:42:40.440824+0000","last_clean":"2024-08-17T17:42:40.440824+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:42:40.440824+0000","last_undegraded":"2024-08-17T17:42:40.440824+0000","last_fullsized":"2024-08-17T17:42:40.440824+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":596,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435084+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:42:53.435084+0000","last_peered":"2024-08-17T17:42:53.435084+0000","last_clean":"2024-08-17T17:42:53.435084+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:42:53.435084+0000","last_undegraded":"2024-08-17T17:42:53.435084+0000","last_fullsized":"2024-08-17T17:42:53.435084+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":412,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.435543+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:42:40.435543+0000","last_peered":"2024-08-17T17:42:40.435543+0000","last_clean":"2024-08-17T17:42:40.435543+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:42:40.435543+0000","last_undegraded":"2024-08-17T17:42:40.435543+0000","last_fullsized":"2024-08-17T17:42:40.435543+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"392'11277","reported_seq":17773,"reported_epoch":392,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:42:59.690174+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:42:59.690174+0000","last_peered":"2024-08-17T17:42:59.690174+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:42:59.690174+0000","last_undegraded":"2024-08-17T17:42:59.690174+0000","last_fullsized":"2024-08-17T17:42:59.690174+0000","mapping_epoch":337,"log_start":"326'4363","ondisk_log_start":"326'4363","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224591872,"num_objects":3427,"num_object_clones":0,"num_object_copies":10281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3427,"num_objects_unfound":0,"num_objects_dirty":3427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11277,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3196,"num_bytes_recovered":209453056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3427}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":97,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.456350+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:42:40.456350+0000","last_peered":"2024-08-17T17:42:40.456350+0000","last_clean":"2024-08-17T17:42:40.456350+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:42:40.456350+0000","last_undegraded":"2024-08-17T17:42:40.456350+0000","last_fullsized":"2024-08-17T17:42:40.456350+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":278,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491334+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:42:40.491334+0000","last_peered":"2024-08-17T17:42:40.491334+0000","last_clean":"2024-08-17T17:42:40.491334+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:42:40.491334+0000","last_undegraded":"2024-08-17T17:42:40.491334+0000","last_fullsized":"2024-08-17T17:42:40.491334+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"392'11487","reported_seq":11577,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:57.021178+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:42:57.021178+0000","last_peered":"2024-08-17T17:42:57.021178+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:42:57.021178+0000","last_undegraded":"2024-08-17T17:42:57.021178+0000","last_fullsized":"2024-08-17T17:42:57.021178+0000","mapping_epoch":337,"log_start":"327'4882","ondisk_log_start":"327'4882","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235077632,"num_objects":3587,"num_object_clones":0,"num_object_copies":10761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3587,"num_objects_unfound":0,"num_objects_dirty":3587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11487,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3587}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":99,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441580+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:42:40.441580+0000","last_peered":"2024-08-17T17:42:40.441580+0000","last_clean":"2024-08-17T17:42:40.441580+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:42:40.441580+0000","last_undegraded":"2024-08-17T17:42:40.441580+0000","last_fullsized":"2024-08-17T17:42:40.441580+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":363,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440754+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:42:40.440754+0000","last_peered":"2024-08-17T17:42:40.440754+0000","last_clean":"2024-08-17T17:42:40.440754+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:42:40.440754+0000","last_undegraded":"2024-08-17T17:42:40.440754+0000","last_fullsized":"2024-08-17T17:42:40.440754+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"392'11313","reported_seq":11406,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:59.553542+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:42:59.553542+0000","last_peered":"2024-08-17T17:42:59.553542+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:42:59.553542+0000","last_undegraded":"2024-08-17T17:42:59.553542+0000","last_fullsized":"2024-08-17T17:42:59.553542+0000","mapping_epoch":337,"log_start":"326'4453","ondisk_log_start":"326'4453","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224198656,"num_objects":3421,"num_object_clones":0,"num_object_copies":10263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3421,"num_objects_unfound":0,"num_objects_dirty":3421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11313,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3421}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":95,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491588+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:42:40.491588+0000","last_peered":"2024-08-17T17:42:40.491588+0000","last_clean":"2024-08-17T17:42:40.491588+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:42:40.491588+0000","last_undegraded":"2024-08-17T17:42:40.491588+0000","last_fullsized":"2024-08-17T17:42:40.491588+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":373,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.435596+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:42:40.435596+0000","last_peered":"2024-08-17T17:42:40.435596+0000","last_clean":"2024-08-17T17:42:40.435596+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:42:40.435596+0000","last_undegraded":"2024-08-17T17:42:40.435596+0000","last_fullsized":"2024-08-17T17:42:40.435596+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"392'11519","reported_seq":11684,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:59.526602+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:42:59.526602+0000","last_peered":"2024-08-17T17:42:59.526602+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:42:59.526602+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'4513","ondisk_log_start":"326'4513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225378304,"num_objects":3439,"num_object_clones":0,"num_object_copies":10317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11519,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-3330},{"shards":"0,3,7","objects":3439}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":539,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437389+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:42:53.437389+0000","last_peered":"2024-08-17T17:42:53.437389+0000","last_clean":"2024-08-17T17:42:53.437389+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:42:53.437389+0000","last_undegraded":"2024-08-17T17:42:53.437389+0000","last_fullsized":"2024-08-17T17:42:53.437389+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":306,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.426572+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:42:53.426572+0000","last_peered":"2024-08-17T17:42:53.426572+0000","last_clean":"2024-08-17T17:42:53.426572+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:42:53.426572+0000","last_undegraded":"2024-08-17T17:42:53.426572+0000","last_fullsized":"2024-08-17T17:42:53.426572+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"392'11129","reported_seq":11210,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:57.114770+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:42:57.114770+0000","last_peered":"2024-08-17T17:42:57.114770+0000","last_clean":"2024-08-17T17:42:57.114770+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:42:57.114770+0000","last_undegraded":"2024-08-17T17:42:57.114770+0000","last_fullsized":"2024-08-17T17:42:57.114770+0000","mapping_epoch":319,"log_start":"326'4574","ondisk_log_start":"326'4574","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11129,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":629,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435298+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:42:53.435298+0000","last_peered":"2024-08-17T17:42:53.435298+0000","last_clean":"2024-08-17T17:42:53.435298+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:42:53.435298+0000","last_undegraded":"2024-08-17T17:42:53.435298+0000","last_fullsized":"2024-08-17T17:42:53.435298+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":346,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441298+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:42:40.441298+0000","last_peered":"2024-08-17T17:42:40.441298+0000","last_clean":"2024-08-17T17:42:40.441298+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:42:40.441298+0000","last_undegraded":"2024-08-17T17:42:40.441298+0000","last_fullsized":"2024-08-17T17:42:40.441298+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"392'11375","reported_seq":11466,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:57.086200+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:42:57.086200+0000","last_peered":"2024-08-17T17:42:57.086200+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:42:57.086200+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'4835","ondisk_log_start":"327'4835","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233111552,"num_objects":3557,"num_object_clones":0,"num_object_copies":10671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11375,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3557},{"shards":"1,6","objects":-3809}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":347,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.435731+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:42:40.435731+0000","last_peered":"2024-08-17T17:42:40.435731+0000","last_clean":"2024-08-17T17:42:40.435731+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:42:40.435731+0000","last_undegraded":"2024-08-17T17:42:40.435731+0000","last_fullsized":"2024-08-17T17:42:40.435731+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":373,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.442054+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:42:40.442054+0000","last_peered":"2024-08-17T17:42:40.442054+0000","last_clean":"2024-08-17T17:42:40.442054+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:42:40.442054+0000","last_undegraded":"2024-08-17T17:42:40.442054+0000","last_fullsized":"2024-08-17T17:42:40.442054+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"392'11503","reported_seq":11669,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:42:59.000899+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:42:59.000899+0000","last_peered":"2024-08-17T17:42:59.000899+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:42:59.000899+0000","last_undegraded":"2024-08-17T17:42:59.000899+0000","last_fullsized":"2024-08-17T17:42:59.000899+0000","mapping_epoch":337,"log_start":"327'5861","ondisk_log_start":"327'5861","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229834752,"num_objects":3507,"num_object_clones":0,"num_object_copies":10521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3507,"num_objects_unfound":0,"num_objects_dirty":3507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11503,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3507}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":384,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440959+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:42:40.440959+0000","last_peered":"2024-08-17T17:42:40.440959+0000","last_clean":"2024-08-17T17:42:40.440959+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:42:40.440959+0000","last_undegraded":"2024-08-17T17:42:40.440959+0000","last_fullsized":"2024-08-17T17:42:40.440959+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":346,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441692+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:42:40.441692+0000","last_peered":"2024-08-17T17:42:40.441692+0000","last_clean":"2024-08-17T17:42:40.441692+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:42:40.441692+0000","last_undegraded":"2024-08-17T17:42:40.441692+0000","last_fullsized":"2024-08-17T17:42:40.441692+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"392'11305","reported_seq":17913,"reported_epoch":392,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:59.873973+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:42:59.873973+0000","last_peered":"2024-08-17T17:42:59.873973+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:42:59.873973+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:42:59.873973+0000","mapping_epoch":337,"log_start":"326'4441","ondisk_log_start":"326'4441","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6864,"log_dups_size":0,"ondisk_log_size":6864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222756864,"num_objects":3399,"num_object_clones":0,"num_object_copies":10197,"num_objects_missing_on_primary":4089,"num_objects_missing":4089,"num_objects_degraded":3399,"num_objects_misplaced":4089,"num_objects_unfound":0,"num_objects_dirty":3399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11305,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3247,"num_bytes_recovered":212729856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":16},{"shards":"1,2,7","objects":3383}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":598,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435947+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:42:53.435947+0000","last_peered":"2024-08-17T17:42:53.435947+0000","last_clean":"2024-08-17T17:42:53.435947+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:42:53.435947+0000","last_undegraded":"2024-08-17T17:42:53.435947+0000","last_fullsized":"2024-08-17T17:42:53.435947+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":535,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435094+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:42:53.435094+0000","last_peered":"2024-08-17T17:42:53.435094+0000","last_clean":"2024-08-17T17:42:53.435094+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:42:53.435094+0000","last_undegraded":"2024-08-17T17:42:53.435094+0000","last_fullsized":"2024-08-17T17:42:53.435094+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"392'11477","reported_seq":18192,"reported_epoch":392,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:42:59.022068+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:42:59.022068+0000","last_peered":"2024-08-17T17:42:59.022068+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:42:59.022068+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:42:59.022068+0000","mapping_epoch":337,"log_start":"327'5844","ondisk_log_start":"327'5844","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5633,"log_dups_size":0,"ondisk_log_size":5633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224002072,"num_objects":3419,"num_object_clones":0,"num_object_copies":10257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3419,"num_objects_misplaced":4168,"num_objects_unfound":0,"num_objects_dirty":3419,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11477,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3280,"num_bytes_recovered":214892544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3418},{"shards":"1,3,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":729,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:53.426295+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:42:53.426295+0000","last_peered":"2024-08-17T17:42:53.426295+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:42:53.426295+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":321,"reported_epoch":391,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:42:40.491519+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:42:40.491519+0000","last_peered":"2024-08-17T17:42:40.491519+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:42:40.491519+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:42:40.491519+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":578,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435615+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:42:53.435615+0000","last_peered":"2024-08-17T17:42:53.435615+0000","last_clean":"2024-08-17T17:42:53.435615+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:42:53.435615+0000","last_undegraded":"2024-08-17T17:42:53.435615+0000","last_fullsized":"2024-08-17T17:42:53.435615+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":356,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.436372+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:42:40.436372+0000","last_peered":"2024-08-17T17:42:40.436372+0000","last_clean":"2024-08-17T17:42:40.436372+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:42:40.436372+0000","last_undegraded":"2024-08-17T17:42:40.436372+0000","last_fullsized":"2024-08-17T17:42:40.436372+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"392'11300","reported_seq":11468,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:59.455640+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:42:59.455640+0000","last_peered":"2024-08-17T17:42:59.455640+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:42:59.455640+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'4321","ondisk_log_start":"326'4321","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225705984,"num_objects":3444,"num_object_clones":0,"num_object_copies":10332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11300,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-3203},{"shards":"0,4,7","objects":3444}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":394,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427650+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:42:53.427650+0000","last_peered":"2024-08-17T17:42:53.427650+0000","last_clean":"2024-08-17T17:42:53.427650+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:42:53.427650+0000","last_undegraded":"2024-08-17T17:42:53.427650+0000","last_fullsized":"2024-08-17T17:42:53.427650+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":328,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440636+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:42:40.440636+0000","last_peered":"2024-08-17T17:42:40.440636+0000","last_clean":"2024-08-17T17:42:40.440636+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:42:40.440636+0000","last_undegraded":"2024-08-17T17:42:40.440636+0000","last_fullsized":"2024-08-17T17:42:40.440636+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":320,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427632+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:42:53.427632+0000","last_peered":"2024-08-17T17:42:53.427632+0000","last_clean":"2024-08-17T17:42:53.427632+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:42:53.427632+0000","last_undegraded":"2024-08-17T17:42:53.427632+0000","last_fullsized":"2024-08-17T17:42:53.427632+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"392'11331","reported_seq":11422,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:57.007323+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:42:57.007323+0000","last_peered":"2024-08-17T17:42:57.007323+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:42:57.007323+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"326'4717","ondisk_log_start":"326'4717","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11331,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3465},{"shards":"5,6","objects":-3831}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":486,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435689+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:42:53.435689+0000","last_peered":"2024-08-17T17:42:53.435689+0000","last_clean":"2024-08-17T17:42:53.435689+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:42:53.435689+0000","last_undegraded":"2024-08-17T17:42:53.435689+0000","last_fullsized":"2024-08-17T17:42:53.435689+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":128,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.435617+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:42:40.435617+0000","last_peered":"2024-08-17T17:42:40.435617+0000","last_clean":"2024-08-17T17:42:40.435617+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:42:40.435617+0000","last_undegraded":"2024-08-17T17:42:40.435617+0000","last_fullsized":"2024-08-17T17:42:40.435617+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":671,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428395+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:42:53.428395+0000","last_peered":"2024-08-17T17:42:53.428395+0000","last_clean":"2024-08-17T17:42:53.428395+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:42:53.428395+0000","last_undegraded":"2024-08-17T17:42:53.428395+0000","last_fullsized":"2024-08-17T17:42:53.428395+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"392'11272","reported_seq":12164,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:59.623187+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:42:59.623187+0000","last_peered":"2024-08-17T17:42:59.623187+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:42:59.623187+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'4216","ondisk_log_start":"326'4216","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221118464,"num_objects":3374,"num_object_clones":0,"num_object_copies":10122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11272,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3374},{"shards":"0,5","objects":-3280}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":462,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428114+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:42:53.428114+0000","last_peered":"2024-08-17T17:42:53.428114+0000","last_clean":"2024-08-17T17:42:53.428114+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:42:53.428114+0000","last_undegraded":"2024-08-17T17:42:53.428114+0000","last_fullsized":"2024-08-17T17:42:53.428114+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":367,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440977+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:42:40.440977+0000","last_peered":"2024-08-17T17:42:40.440977+0000","last_clean":"2024-08-17T17:42:40.440977+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:42:40.440977+0000","last_undegraded":"2024-08-17T17:42:40.440977+0000","last_fullsized":"2024-08-17T17:42:40.440977+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":257,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437176+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:42:53.437176+0000","last_peered":"2024-08-17T17:42:53.437176+0000","last_clean":"2024-08-17T17:42:53.437176+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:42:53.437176+0000","last_undegraded":"2024-08-17T17:42:53.437176+0000","last_fullsized":"2024-08-17T17:42:53.437176+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"392'11418","reported_seq":26429,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:57.753079+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:42:57.753079+0000","last_peered":"2024-08-17T17:42:57.753079+0000","last_clean":"2024-08-17T17:42:57.753079+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:42:57.753079+0000","last_undegraded":"2024-08-17T17:42:57.753079+0000","last_fullsized":"2024-08-17T17:42:57.753079+0000","mapping_epoch":355,"log_start":"327'5130","ondisk_log_start":"327'5130","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6288,"log_dups_size":0,"ondisk_log_size":6288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224002048,"num_objects":3418,"num_object_clones":0,"num_object_copies":10254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11418,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491467+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:42:40.491467+0000","last_peered":"2024-08-17T17:42:40.491467+0000","last_clean":"2024-08-17T17:42:40.491467+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:42:40.491467+0000","last_undegraded":"2024-08-17T17:42:40.491467+0000","last_fullsized":"2024-08-17T17:42:40.491467+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":126,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437083+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:42:53.437083+0000","last_peered":"2024-08-17T17:42:53.437083+0000","last_clean":"2024-08-17T17:42:53.437083+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:42:53.437083+0000","last_undegraded":"2024-08-17T17:42:53.437083+0000","last_fullsized":"2024-08-17T17:42:53.437083+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":621,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428695+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:42:53.428695+0000","last_peered":"2024-08-17T17:42:53.428695+0000","last_clean":"2024-08-17T17:42:53.428695+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:42:53.428695+0000","last_undegraded":"2024-08-17T17:42:53.428695+0000","last_fullsized":"2024-08-17T17:42:53.428695+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"392'11080","reported_seq":25659,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:57.695729+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:42:57.695729+0000","last_peered":"2024-08-17T17:42:57.695729+0000","last_clean":"2024-08-17T17:42:57.695729+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:42:57.695729+0000","last_undegraded":"2024-08-17T17:42:57.695729+0000","last_fullsized":"2024-08-17T17:42:57.695729+0000","mapping_epoch":345,"log_start":"327'4772","ondisk_log_start":"327'4772","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217841664,"num_objects":3324,"num_object_clones":0,"num_object_copies":9972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11080,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":302,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491496+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:42:40.491496+0000","last_peered":"2024-08-17T17:42:40.491496+0000","last_clean":"2024-08-17T17:42:40.491496+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:42:40.491496+0000","last_undegraded":"2024-08-17T17:42:40.491496+0000","last_fullsized":"2024-08-17T17:42:40.491496+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":377,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.426965+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:42:53.426965+0000","last_peered":"2024-08-17T17:42:53.426965+0000","last_clean":"2024-08-17T17:42:53.426965+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:42:53.426965+0000","last_undegraded":"2024-08-17T17:42:53.426965+0000","last_fullsized":"2024-08-17T17:42:53.426965+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":444,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491655+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:42:40.491655+0000","last_peered":"2024-08-17T17:42:40.491655+0000","last_clean":"2024-08-17T17:42:40.491655+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:42:40.491655+0000","last_undegraded":"2024-08-17T17:42:40.491655+0000","last_fullsized":"2024-08-17T17:42:40.491655+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"392'11297","reported_seq":11484,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:42:57.673575+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:42:57.673575+0000","last_peered":"2024-08-17T17:42:57.673575+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:42:57.673575+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'4987","ondisk_log_start":"327'4987","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6310,"log_dups_size":0,"ondisk_log_size":6310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228524032,"num_objects":3487,"num_object_clones":0,"num_object_copies":10461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11297,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":3487},{"shards":"0,3","objects":-3761}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":652,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435256+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:42:53.435256+0000","last_peered":"2024-08-17T17:42:53.435256+0000","last_clean":"2024-08-17T17:42:53.435256+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:42:53.435256+0000","last_undegraded":"2024-08-17T17:42:53.435256+0000","last_fullsized":"2024-08-17T17:42:53.435256+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":620,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.429547+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:42:53.429547+0000","last_peered":"2024-08-17T17:42:53.429547+0000","last_clean":"2024-08-17T17:42:53.429547+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:42:53.429547+0000","last_undegraded":"2024-08-17T17:42:53.429547+0000","last_fullsized":"2024-08-17T17:42:53.429547+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":638,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435403+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:42:53.435403+0000","last_peered":"2024-08-17T17:42:53.435403+0000","last_clean":"2024-08-17T17:42:53.435403+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:42:53.435403+0000","last_undegraded":"2024-08-17T17:42:53.435403+0000","last_fullsized":"2024-08-17T17:42:53.435403+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"392'11188","reported_seq":25877,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:56.887247+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:42:56.887247+0000","last_peered":"2024-08-17T17:42:56.887247+0000","last_clean":"2024-08-17T17:42:56.887247+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:42:56.887247+0000","last_undegraded":"2024-08-17T17:42:56.887247+0000","last_fullsized":"2024-08-17T17:42:56.887247+0000","mapping_epoch":347,"log_start":"326'4572","ondisk_log_start":"326'4572","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221904896,"num_objects":3386,"num_object_clones":0,"num_object_copies":10158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11188,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":310,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.426841+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:42:53.426841+0000","last_peered":"2024-08-17T17:42:53.426841+0000","last_clean":"2024-08-17T17:42:53.426841+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:42:53.426841+0000","last_undegraded":"2024-08-17T17:42:53.426841+0000","last_fullsized":"2024-08-17T17:42:53.426841+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":396,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.436087+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:42:40.436087+0000","last_peered":"2024-08-17T17:42:40.436087+0000","last_clean":"2024-08-17T17:42:40.436087+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:42:40.436087+0000","last_undegraded":"2024-08-17T17:42:40.436087+0000","last_fullsized":"2024-08-17T17:42:40.436087+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":346,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440894+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:42:40.440894+0000","last_peered":"2024-08-17T17:42:40.440894+0000","last_clean":"2024-08-17T17:42:40.440894+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:42:40.440894+0000","last_undegraded":"2024-08-17T17:42:40.440894+0000","last_fullsized":"2024-08-17T17:42:40.440894+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":346,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440906+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:42:40.440906+0000","last_peered":"2024-08-17T17:42:40.440906+0000","last_clean":"2024-08-17T17:42:40.440906+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:42:40.440906+0000","last_undegraded":"2024-08-17T17:42:40.440906+0000","last_fullsized":"2024-08-17T17:42:40.440906+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":389,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441447+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:42:40.441447+0000","last_peered":"2024-08-17T17:42:40.441447+0000","last_clean":"2024-08-17T17:42:40.441447+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:42:40.441447+0000","last_undegraded":"2024-08-17T17:42:40.441447+0000","last_fullsized":"2024-08-17T17:42:40.441447+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":627,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428539+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:42:53.428539+0000","last_peered":"2024-08-17T17:42:53.428539+0000","last_clean":"2024-08-17T17:42:53.428539+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:42:53.428539+0000","last_undegraded":"2024-08-17T17:42:53.428539+0000","last_fullsized":"2024-08-17T17:42:53.428539+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":166,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437914+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:42:53.437914+0000","last_peered":"2024-08-17T17:42:53.437914+0000","last_clean":"2024-08-17T17:42:53.437914+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:42:53.437914+0000","last_undegraded":"2024-08-17T17:42:53.437914+0000","last_fullsized":"2024-08-17T17:42:53.437914+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":296,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491659+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:42:40.491659+0000","last_peered":"2024-08-17T17:42:40.491659+0000","last_clean":"2024-08-17T17:42:40.491659+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:42:40.491659+0000","last_undegraded":"2024-08-17T17:42:40.491659+0000","last_fullsized":"2024-08-17T17:42:40.491659+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":585,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435445+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:42:53.435445+0000","last_peered":"2024-08-17T17:42:53.435445+0000","last_clean":"2024-08-17T17:42:53.435445+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:42:53.435445+0000","last_undegraded":"2024-08-17T17:42:53.435445+0000","last_fullsized":"2024-08-17T17:42:53.435445+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":440,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.491868+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:42:40.491868+0000","last_peered":"2024-08-17T17:42:40.491868+0000","last_clean":"2024-08-17T17:42:40.491868+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:42:40.491868+0000","last_undegraded":"2024-08-17T17:42:40.491868+0000","last_fullsized":"2024-08-17T17:42:40.491868+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":579,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435008+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:42:53.435008+0000","last_peered":"2024-08-17T17:42:53.435008+0000","last_clean":"2024-08-17T17:42:53.435008+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:42:53.435008+0000","last_undegraded":"2024-08-17T17:42:53.435008+0000","last_fullsized":"2024-08-17T17:42:53.435008+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":363,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.436014+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:42:40.436014+0000","last_peered":"2024-08-17T17:42:40.436014+0000","last_clean":"2024-08-17T17:42:40.436014+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:42:40.436014+0000","last_undegraded":"2024-08-17T17:42:40.436014+0000","last_fullsized":"2024-08-17T17:42:40.436014+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":610,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435860+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:42:53.435860+0000","last_peered":"2024-08-17T17:42:53.435860+0000","last_clean":"2024-08-17T17:42:53.435860+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:42:53.435860+0000","last_undegraded":"2024-08-17T17:42:53.435860+0000","last_fullsized":"2024-08-17T17:42:53.435860+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":326,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435305+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:42:53.435305+0000","last_peered":"2024-08-17T17:42:53.435305+0000","last_clean":"2024-08-17T17:42:53.435305+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:42:53.435305+0000","last_undegraded":"2024-08-17T17:42:53.435305+0000","last_fullsized":"2024-08-17T17:42:53.435305+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":618,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428445+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:42:53.428445+0000","last_peered":"2024-08-17T17:42:53.428445+0000","last_clean":"2024-08-17T17:42:53.428445+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:42:53.428445+0000","last_undegraded":"2024-08-17T17:42:53.428445+0000","last_fullsized":"2024-08-17T17:42:53.428445+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":578,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437725+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:42:53.437725+0000","last_peered":"2024-08-17T17:42:53.437725+0000","last_clean":"2024-08-17T17:42:53.437725+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:42:53.437725+0000","last_undegraded":"2024-08-17T17:42:53.437725+0000","last_fullsized":"2024-08-17T17:42:53.437725+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":375,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441381+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:42:40.441381+0000","last_peered":"2024-08-17T17:42:40.441381+0000","last_clean":"2024-08-17T17:42:40.441381+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:42:40.441381+0000","last_undegraded":"2024-08-17T17:42:40.441381+0000","last_fullsized":"2024-08-17T17:42:40.441381+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":266,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.437319+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:42:53.437319+0000","last_peered":"2024-08-17T17:42:53.437319+0000","last_clean":"2024-08-17T17:42:53.437319+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:42:53.437319+0000","last_undegraded":"2024-08-17T17:42:53.437319+0000","last_fullsized":"2024-08-17T17:42:53.437319+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":323,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.426447+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:42:53.426447+0000","last_peered":"2024-08-17T17:42:53.426447+0000","last_clean":"2024-08-17T17:42:53.426447+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:42:53.426447+0000","last_undegraded":"2024-08-17T17:42:53.426447+0000","last_fullsized":"2024-08-17T17:42:53.426447+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":383,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.440886+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:42:40.440886+0000","last_peered":"2024-08-17T17:42:40.440886+0000","last_clean":"2024-08-17T17:42:40.440886+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:42:40.440886+0000","last_undegraded":"2024-08-17T17:42:40.440886+0000","last_fullsized":"2024-08-17T17:42:40.440886+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":246,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441372+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:42:40.441372+0000","last_peered":"2024-08-17T17:42:40.441372+0000","last_clean":"2024-08-17T17:42:40.441372+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:42:40.441372+0000","last_undegraded":"2024-08-17T17:42:40.441372+0000","last_fullsized":"2024-08-17T17:42:40.441372+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":383,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441252+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:42:40.441252+0000","last_peered":"2024-08-17T17:42:40.441252+0000","last_clean":"2024-08-17T17:42:40.441252+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:42:40.441252+0000","last_undegraded":"2024-08-17T17:42:40.441252+0000","last_fullsized":"2024-08-17T17:42:40.441252+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":246,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441917+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:42:40.441917+0000","last_peered":"2024-08-17T17:42:40.441917+0000","last_clean":"2024-08-17T17:42:40.441917+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:42:40.441917+0000","last_undegraded":"2024-08-17T17:42:40.441917+0000","last_fullsized":"2024-08-17T17:42:40.441917+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":630,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435767+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:42:53.435767+0000","last_peered":"2024-08-17T17:42:53.435767+0000","last_clean":"2024-08-17T17:42:53.435767+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:42:53.435767+0000","last_undegraded":"2024-08-17T17:42:53.435767+0000","last_fullsized":"2024-08-17T17:42:53.435767+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":604,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435477+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:42:53.435477+0000","last_peered":"2024-08-17T17:42:53.435477+0000","last_clean":"2024-08-17T17:42:53.435477+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:42:53.435477+0000","last_undegraded":"2024-08-17T17:42:53.435477+0000","last_fullsized":"2024-08-17T17:42:53.435477+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":346,"reported_epoch":391,"state":"active+clean","last_fresh":"2024-08-17T17:42:40.441100+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:42:40.441100+0000","last_peered":"2024-08-17T17:42:40.441100+0000","last_clean":"2024-08-17T17:42:40.441100+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:42:40.441100+0000","last_undegraded":"2024-08-17T17:42:40.441100+0000","last_fullsized":"2024-08-17T17:42:40.441100+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":325,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.426782+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:42:53.426782+0000","last_peered":"2024-08-17T17:42:53.426782+0000","last_clean":"2024-08-17T17:42:53.426782+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:42:53.426782+0000","last_undegraded":"2024-08-17T17:42:53.426782+0000","last_fullsized":"2024-08-17T17:42:53.426782+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3612213272,"num_objects":55119,"num_object_clones":0,"num_object_copies":165357,"num_objects_missing_on_primary":4089,"num_objects_missing":4089,"num_objects_degraded":51237,"num_objects_misplaced":22199,"num_objects_unfound":0,"num_objects_dirty":55119,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181271,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32002,"num_bytes_recovered":1974730752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13025947648,"data_stored":13025935432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104790,"ondisk_log_size":104790,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170118,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463488,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2452440,"kb_used_data":1558736,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":91292712,"statfs":{"total":95995035648,"available":93483737088,"internally_reserved":0,"allocated":1596145664,"data_stored":1593990569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492163,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840380,"kb_used_data":440884,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":92904772,"statfs":{"total":95995035648,"available":95134486528,"internally_reserved":0,"allocated":451465216,"data_stored":449332421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986754,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2376588,"kb_used_data":876228,"kb_used_omap":1,"kb_used_meta":1500350,"kb_avail":91368564,"statfs":{"total":95995035648,"available":93561409536,"internally_reserved":0,"allocated":897257472,"data_stored":895113317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1536358744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225582,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2719228,"kb_used_data":1771572,"kb_used_omap":4,"kb_used_meta":947643,"kb_avail":91025924,"statfs":{"total":95995035648,"available":93210546176,"internally_reserved":0,"allocated":1814089728,"data_stored":1811957231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":970386880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075907,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533800,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2606336,"kb_used_data":1546296,"kb_used_omap":5,"kb_used_meta":1060026,"kb_avail":91138816,"statfs":{"total":95995035648,"available":93326147584,"internally_reserved":0,"allocated":1583407104,"data_stored":1581269285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1085466864},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089777,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2333464,"kb_used_data":1324240,"kb_used_omap":5,"kb_used_meta":1009210,"kb_avail":91411688,"statfs":{"total":95995035648,"available":93605568512,"internally_reserved":0,"allocated":1356021760,"data_stored":1353866724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1033431099},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1350172672,"data_stored":1350172672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1577390080,"data_stored":1577386008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1805516800,"data_stored":1805516800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":891944960,"data_stored":891944960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":446300160,"data_stored":446300160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1588527104,"data_stored":1588527104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2026835968,"data_stored":2026831896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.388 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:03.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:03.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:03.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:03.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:03.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:03.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:03.737 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:03.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:03.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:03.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:03.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:03.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:04.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:04.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:04.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:04.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:04.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:04.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:04.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:04.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:04.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:04.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:04.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:04.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:04.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:04.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:04.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:04.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.877 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:04.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:04.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:04.998 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:04.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:04.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:05.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:05.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:05.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:05.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:05.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:05.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:05.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:05.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:05.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:05.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:05.684 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:05 smithi142 ceph-mon[106729]: pgmap v1507: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51210/164118 objects degraded (31.203%); 22083/164118 objects misplaced (13.456%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:05.684 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:05 smithi142 ceph-mon[103614]: pgmap v1507: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51210/164118 objects degraded (31.203%); 22083/164118 objects misplaced (13.456%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:05.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:05.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:05.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:05 smithi152 ceph-mon[93456]: pgmap v1507: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 51210/164118 objects degraded (31.203%); 22083/164118 objects misplaced (13.456%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:05.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:05.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:05.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:05.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:05.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:06.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:06.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:06.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:06.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:06.270 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:06.270 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:06.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:06.371 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:06.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:06.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:06.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:06.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:06.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51210/164118 objects degraded (31.203%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:06.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51210/164118 objects degraded (31.203%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:06.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:06.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:06.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51210/164118 objects degraded (31.203%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:06.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:06.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:06.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:06.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:06.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:06.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:07.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:07.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:07.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:07.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:07.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:07.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:07.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:07.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:43:07.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:07.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:07.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:07.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:07.616 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:07.633 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:07 smithi142 ceph-mon[106729]: pgmap v1508: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 140 op/s; 51158/163515 objects degraded (31.286%); 21980/163515 objects misplaced (13.442%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:43:07.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:07 smithi142 ceph-mon[103614]: pgmap v1508: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 140 op/s; 51158/163515 objects degraded (31.286%); 21980/163515 objects misplaced (13.442%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:43:07.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:07.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:07 smithi152 ceph-mon[93456]: pgmap v1508: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 140 op/s; 51158/163515 objects degraded (31.286%); 21980/163515 objects misplaced (13.442%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:43:07.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:07.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:07.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:07.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:07.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:08.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:08.124 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:08.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:08.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:08.312 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:08.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:08.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:08.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:08.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:08.493 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:08.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:08.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:08.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:08.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:08.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:08.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:08.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:08.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:08.970 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:08.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:08.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:09.058 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:09.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:09.120 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:09.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:09.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:09.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:09.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:09.451 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:43:09.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:09.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:09.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:09.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:09.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:09.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:09.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:09 smithi152 ceph-mon[93456]: pgmap v1509: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 51158/162738 objects degraded (31.436%); 21956/162738 objects misplaced (13.492%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:43:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:09 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3264753705' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:43:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:43:09.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:09.865 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:09 smithi142 ceph-mon[106729]: pgmap v1509: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 51158/162738 objects degraded (31.436%); 21956/162738 objects misplaced (13.492%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:43:09.866 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:09 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3264753705' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:43:09.866 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:43:09.866 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:09 smithi142 ceph-mon[103614]: pgmap v1509: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 135 op/s; 51158/162738 objects degraded (31.436%); 21956/162738 objects misplaced (13.492%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:43:09.866 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:09 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3264753705' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:43:09.866 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:43:09.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:09.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:09.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:10.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:10.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:10.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:10.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:10.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:10.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:10.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:10.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:10.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:10 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:10.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:43:10.508 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:10 smithi152 ceph-mon[93456]: osdmap e393: 8 total, 8 up, 6 in 2024-08-17T17:43:10.509 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:43:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:43:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:10.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:10.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:10 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:43:10.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:10 smithi142 ceph-mon[106729]: osdmap e393: 8 total, 8 up, 6 in 2024-08-17T17:43:10.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:10 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:10.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:43:10.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:10 smithi142 ceph-mon[103614]: osdmap e393: 8 total, 8 up, 6 in 2024-08-17T17:43:10.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:10.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:10.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:10.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:10.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:10.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:10.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:10.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:11.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:11.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:11.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:11.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:11.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:11.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:11.394 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:11.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:11.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:11 smithi152 ceph-mon[93456]: pgmap v1511: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 51079/161814 objects degraded (31.566%); 21768/161814 objects misplaced (13.452%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:43:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51079/161814 objects degraded (31.566%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:11.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:11.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:11.748 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:11 smithi142 ceph-mon[106729]: pgmap v1511: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 51079/161814 objects degraded (31.566%); 21768/161814 objects misplaced (13.452%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:43:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51079/161814 objects degraded (31.566%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:11 smithi142 ceph-mon[103614]: pgmap v1511: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 51079/161814 objects degraded (31.566%); 21768/161814 objects misplaced (13.452%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:43:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51079/161814 objects degraded (31.566%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:11.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:11.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:11.901 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:11.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:12.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:12.027 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:12.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:43:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:43:11] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-17T17:43:12.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:12.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:12.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:12.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:12.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:12.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:12.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:12.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:12.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:12.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:12.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:12.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.701 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.702 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:12.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:43:12.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:43:12.703 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:12.772 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:43:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:43:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:12.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:12.896 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:12.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:12.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:12.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:13.020 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:13.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:13.148 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:13.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:13.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:13.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:13.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:13.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:13.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:13.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:13.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:13.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:13.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:13.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:13 smithi152 ceph-mon[93456]: pgmap v1512: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 51079/160896 objects degraded (31.747%); 21713/160896 objects misplaced (13.495%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:43:13.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:13 smithi142 ceph-mon[106729]: pgmap v1512: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 51079/160896 objects degraded (31.747%); 21713/160896 objects misplaced (13.495%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:43:13.827 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:13 smithi142 ceph-mon[103614]: pgmap v1512: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 51079/160896 objects degraded (31.747%); 21713/160896 objects misplaced (13.495%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:43:13.832 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:13.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:13.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:13.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:13.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:14.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:14.028 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:14.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:14.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:14.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:14.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:14.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:14.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:14.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:14.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:14.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:14.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:43:14.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:14.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:14.752 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:14.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:14.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:14.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:14.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:15.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:15.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:15.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:15.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:15.198 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:43:15.198 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:15.217 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:43:15.218 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:15.235 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:43:15.236 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:15.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:15.253 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:43:15.253 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:15.270 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:43:15.270 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:15.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:15.285 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:43:15.285 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:15.301 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:43:15.301 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:15.319 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:43:15.319 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:15.354 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:15.354 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:15.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:15.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:15.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:15.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:15.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:15.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:15 smithi142 ceph-mon[103614]: pgmap v1513: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 136 op/s; 51012/160026 objects degraded (31.877%); 21582/160026 objects misplaced (13.487%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:43:15.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1418747520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:15.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:15 smithi142 ceph-mon[106729]: pgmap v1513: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 136 op/s; 51012/160026 objects degraded (31.877%); 21582/160026 objects misplaced (13.487%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:43:15.624 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1418747520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:15.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:15.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:15.768 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:15 smithi152 ceph-mon[93456]: pgmap v1513: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 136 op/s; 51012/160026 objects degraded (31.877%); 21582/160026 objects misplaced (13.487%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:43:15.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1418747520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:15.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:15.891 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:15.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:15.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:16.018 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:16.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:16.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:16.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:43:16.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:16.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:16.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:16.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.393 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:16.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:16.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:16.520 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:16.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:16.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:16.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:16.769 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 51012/160026 objects degraded (31.877%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:16.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 51012/160026 objects degraded (31.877%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:16.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 51012/160026 objects degraded (31.877%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:16.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:16.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:16.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:17.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:17.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:17.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:17.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:17.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:17.266 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:17.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:17.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:17.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:17.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:17.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:17.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:17.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:17.764 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:17 smithi142 ceph-mon[106729]: pgmap v1514: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 50964/159027 objects degraded (32.047%); 21414/159027 objects misplaced (13.466%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:43:17.765 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:17 smithi142 ceph-mon[103614]: pgmap v1514: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 50964/159027 objects degraded (32.047%); 21414/159027 objects misplaced (13.466%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:43:17.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:17.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:17 smithi152 ceph-mon[93456]: pgmap v1514: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 50964/159027 objects degraded (32.047%); 21414/159027 objects misplaced (13.466%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:43:17.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:17.890 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:17.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:17.953 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:43:17.954 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:43:17.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:18.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:18.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:18.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:18.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:18.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:18.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:18.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:18.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:18.510 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:18.609 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:18.610 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:18.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:18.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:18.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:18.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:18.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:18.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:18.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:18.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:19.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:19.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:19.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:19.150 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1514,"stamp":"2024-08-17T17:43:16.373213+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3472824293,"num_objects":53009,"num_object_clones":0,"num_object_copies":159027,"num_objects_missing_on_primary":3941,"num_objects_missing":3941,"num_objects_degraded":50964,"num_objects_misplaced":21414,"num_objects_unfound":0,"num_objects_dirty":53009,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":183685,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32443,"num_bytes_recovered":2003313294,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104933,"ondisk_log_size":104933,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13072916,"kb_used_data":7237476,"kb_used_omap":27,"kb_used_meta":5835364,"kb_avail":549397996,"statfs":{"total":575970213888,"available":562583547904,"internally_reserved":0,"allocated":7411175424,"data_stored":7398271269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5975413480},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-98041856,"num_objects":-1496,"num_object_clones":0,"num_object_copies":-4488,"num_objects_missing_on_primary":-98,"num_objects_missing":-98,"num_objects_degraded":-194,"num_objects_misplaced":-566,"num_objects_unfound":0,"num_objects_dirty":-1496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1496,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":19464192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002817"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":630,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455696+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:43:09.455696+0000","last_peered":"2024-08-17T17:43:09.455696+0000","last_clean":"2024-08-17T17:43:09.455696+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:43:09.455696+0000","last_undegraded":"2024-08-17T17:43:09.455696+0000","last_fullsized":"2024-08-17T17:43:09.455696+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":102,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428099+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:42:53.428099+0000","last_peered":"2024-08-17T17:42:53.428099+0000","last_clean":"2024-08-17T17:42:53.428099+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:42:53.428099+0000","last_undegraded":"2024-08-17T17:42:53.428099+0000","last_fullsized":"2024-08-17T17:42:53.428099+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":296,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.434861+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:42:53.434861+0000","last_peered":"2024-08-17T17:42:53.434861+0000","last_clean":"2024-08-17T17:42:53.434861+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:42:53.434861+0000","last_undegraded":"2024-08-17T17:42:53.434861+0000","last_fullsized":"2024-08-17T17:42:53.434861+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":328,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456860+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:43:09.456860+0000","last_peered":"2024-08-17T17:43:09.456860+0000","last_clean":"2024-08-17T17:43:09.456860+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:43:09.456860+0000","last_undegraded":"2024-08-17T17:43:09.456860+0000","last_fullsized":"2024-08-17T17:43:09.456860+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":505,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456601+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:43:09.456601+0000","last_peered":"2024-08-17T17:43:09.456601+0000","last_clean":"2024-08-17T17:43:09.456601+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:43:09.456601+0000","last_undegraded":"2024-08-17T17:43:09.456601+0000","last_fullsized":"2024-08-17T17:43:09.456601+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":245,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427628+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:42:53.427628+0000","last_peered":"2024-08-17T17:42:53.427628+0000","last_clean":"2024-08-17T17:42:53.427628+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:42:53.427628+0000","last_undegraded":"2024-08-17T17:42:53.427628+0000","last_fullsized":"2024-08-17T17:42:53.427628+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":598,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455666+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:43:09.455666+0000","last_peered":"2024-08-17T17:43:09.455666+0000","last_clean":"2024-08-17T17:43:09.455666+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:43:09.455666+0000","last_undegraded":"2024-08-17T17:43:09.455666+0000","last_fullsized":"2024-08-17T17:43:09.455666+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":413,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.430136+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:42:53.430136+0000","last_peered":"2024-08-17T17:42:53.430136+0000","last_clean":"2024-08-17T17:42:53.430136+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:42:53.430136+0000","last_undegraded":"2024-08-17T17:42:53.430136+0000","last_fullsized":"2024-08-17T17:42:53.430136+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"393'11401","reported_seq":18146,"reported_epoch":393,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:43:14.967356+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:43:14.967356+0000","last_peered":"2024-08-17T17:43:14.967356+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:43:14.967356+0000","last_undegraded":"2024-08-17T17:43:14.967356+0000","last_fullsized":"2024-08-17T17:43:14.967356+0000","mapping_epoch":337,"log_start":"326'4563","ondisk_log_start":"326'4563","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6838,"log_dups_size":0,"ondisk_log_size":6838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216465408,"num_objects":3303,"num_object_clones":0,"num_object_copies":9909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3303,"num_objects_unfound":0,"num_objects_dirty":3303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11401,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3320,"num_bytes_recovered":217579520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3303}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":98,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.483822+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:42:53.483822+0000","last_peered":"2024-08-17T17:42:53.483822+0000","last_clean":"2024-08-17T17:42:53.483822+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:42:53.483822+0000","last_undegraded":"2024-08-17T17:42:53.483822+0000","last_fullsized":"2024-08-17T17:42:53.483822+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435355+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:42:53.435355+0000","last_peered":"2024-08-17T17:42:53.435355+0000","last_clean":"2024-08-17T17:42:53.435355+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:42:53.435355+0000","last_undegraded":"2024-08-17T17:42:53.435355+0000","last_fullsized":"2024-08-17T17:42:53.435355+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"393'11603","reported_seq":11694,"reported_epoch":393,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:11.797629+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:43:11.797629+0000","last_peered":"2024-08-17T17:43:11.797629+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:43:11.797629+0000","last_undegraded":"2024-08-17T17:43:11.797629+0000","last_fullsized":"2024-08-17T17:43:11.797629+0000","mapping_epoch":337,"log_start":"327'4982","ondisk_log_start":"327'4982","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227475456,"num_objects":3471,"num_object_clones":0,"num_object_copies":10413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3471,"num_objects_unfound":0,"num_objects_dirty":3471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11603,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3471}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":100,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428074+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:42:53.428074+0000","last_peered":"2024-08-17T17:42:53.428074+0000","last_clean":"2024-08-17T17:42:53.428074+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:42:53.428074+0000","last_undegraded":"2024-08-17T17:42:53.428074+0000","last_fullsized":"2024-08-17T17:42:53.428074+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":364,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427555+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:42:53.427555+0000","last_peered":"2024-08-17T17:42:53.427555+0000","last_clean":"2024-08-17T17:42:53.427555+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:42:53.427555+0000","last_undegraded":"2024-08-17T17:42:53.427555+0000","last_fullsized":"2024-08-17T17:42:53.427555+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"393'11428","reported_seq":11522,"reported_epoch":393,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:14.946778+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:43:14.946778+0000","last_peered":"2024-08-17T17:43:14.946778+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:43:14.946778+0000","last_undegraded":"2024-08-17T17:43:14.946778+0000","last_fullsized":"2024-08-17T17:43:14.946778+0000","mapping_epoch":337,"log_start":"326'4553","ondisk_log_start":"326'4553","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6875,"log_dups_size":0,"ondisk_log_size":6875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3306,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11428,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3306}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":96,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.434770+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:42:53.434770+0000","last_peered":"2024-08-17T17:42:53.434770+0000","last_clean":"2024-08-17T17:42:53.434770+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:42:53.434770+0000","last_undegraded":"2024-08-17T17:42:53.434770+0000","last_fullsized":"2024-08-17T17:42:53.434770+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":374,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.430254+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:42:53.430254+0000","last_peered":"2024-08-17T17:42:53.430254+0000","last_clean":"2024-08-17T17:42:53.430254+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:42:53.430254+0000","last_undegraded":"2024-08-17T17:42:53.430254+0000","last_fullsized":"2024-08-17T17:42:53.430254+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"393'11637","reported_seq":11804,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:14.155084+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:43:14.155084+0000","last_peered":"2024-08-17T17:43:14.155084+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:43:14.155084+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"326'4613","ondisk_log_start":"326'4613","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217645056,"num_objects":3321,"num_object_clones":0,"num_object_copies":9963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11637,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-3448},{"shards":"0,3,7","objects":3321}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":542,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455857+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:43:09.455857+0000","last_peered":"2024-08-17T17:43:09.455857+0000","last_clean":"2024-08-17T17:43:09.455857+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:43:09.455857+0000","last_undegraded":"2024-08-17T17:43:09.455857+0000","last_fullsized":"2024-08-17T17:43:09.455857+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":308,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.461274+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:43:09.461274+0000","last_peered":"2024-08-17T17:43:09.461274+0000","last_clean":"2024-08-17T17:43:09.461274+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:43:09.461274+0000","last_undegraded":"2024-08-17T17:43:09.461274+0000","last_fullsized":"2024-08-17T17:43:09.461274+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"393'11278","reported_seq":11360,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.801814+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:43:11.801814+0000","last_peered":"2024-08-17T17:43:11.801814+0000","last_clean":"2024-08-17T17:43:11.801814+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:43:11.801814+0000","last_undegraded":"2024-08-17T17:43:11.801814+0000","last_fullsized":"2024-08-17T17:43:11.801814+0000","mapping_epoch":319,"log_start":"327'4674","ondisk_log_start":"327'4674","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217317376,"num_objects":3316,"num_object_clones":0,"num_object_copies":9948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11278,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":631,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456297+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:43:09.456297+0000","last_peered":"2024-08-17T17:43:09.456297+0000","last_clean":"2024-08-17T17:43:09.456297+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:43:09.456297+0000","last_undegraded":"2024-08-17T17:43:09.456297+0000","last_fullsized":"2024-08-17T17:43:09.456297+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":347,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427824+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:42:53.427824+0000","last_peered":"2024-08-17T17:42:53.427824+0000","last_clean":"2024-08-17T17:42:53.427824+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:42:53.427824+0000","last_undegraded":"2024-08-17T17:42:53.427824+0000","last_fullsized":"2024-08-17T17:42:53.427824+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"393'11511","reported_seq":11603,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:12.009364+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:43:12.009364+0000","last_peered":"2024-08-17T17:43:12.009364+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:43:12.009364+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'4935","ondisk_log_start":"327'4935","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6576,"log_dups_size":0,"ondisk_log_size":6576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224198656,"num_objects":3421,"num_object_clones":0,"num_object_copies":10263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11511,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3421},{"shards":"1,6","objects":-3945}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":348,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.430277+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:42:53.430277+0000","last_peered":"2024-08-17T17:42:53.430277+0000","last_clean":"2024-08-17T17:42:53.430277+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:42:53.430277+0000","last_undegraded":"2024-08-17T17:42:53.430277+0000","last_fullsized":"2024-08-17T17:42:53.430277+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":374,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428178+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:42:53.428178+0000","last_peered":"2024-08-17T17:42:53.428178+0000","last_clean":"2024-08-17T17:42:53.428178+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:42:53.428178+0000","last_undegraded":"2024-08-17T17:42:53.428178+0000","last_fullsized":"2024-08-17T17:42:53.428178+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"393'11661","reported_seq":11829,"reported_epoch":393,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:14.014134+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:43:14.014134+0000","last_peered":"2024-08-17T17:43:14.014134+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:43:14.014134+0000","last_undegraded":"2024-08-17T17:43:14.014134+0000","last_fullsized":"2024-08-17T17:43:14.014134+0000","mapping_epoch":337,"log_start":"328'6061","ondisk_log_start":"328'6061","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5600,"log_dups_size":0,"ondisk_log_size":5600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219480064,"num_objects":3349,"num_object_clones":0,"num_object_copies":10047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3349,"num_objects_unfound":0,"num_objects_dirty":3349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11661,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3349}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":385,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427761+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:42:53.427761+0000","last_peered":"2024-08-17T17:42:53.427761+0000","last_clean":"2024-08-17T17:42:53.427761+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:42:53.427761+0000","last_undegraded":"2024-08-17T17:42:53.427761+0000","last_fullsized":"2024-08-17T17:42:53.427761+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":347,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428213+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:42:53.428213+0000","last_peered":"2024-08-17T17:42:53.428213+0000","last_clean":"2024-08-17T17:42:53.428213+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:42:53.428213+0000","last_undegraded":"2024-08-17T17:42:53.428213+0000","last_fullsized":"2024-08-17T17:42:53.428213+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"393'11456","reported_seq":18365,"reported_epoch":393,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:14.949014+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:43:14.949014+0000","last_peered":"2024-08-17T17:43:14.949014+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:43:14.949014+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:43:14.949014+0000","mapping_epoch":337,"log_start":"326'4541","ondisk_log_start":"326'4541","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212860928,"num_objects":3248,"num_object_clones":0,"num_object_copies":9744,"num_objects_missing_on_primary":3941,"num_objects_missing":3941,"num_objects_degraded":3248,"num_objects_misplaced":3941,"num_objects_unfound":0,"num_objects_dirty":3248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11456,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3399,"num_bytes_recovered":222691328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":3235},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":601,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456414+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:43:09.456414+0000","last_peered":"2024-08-17T17:43:09.456414+0000","last_clean":"2024-08-17T17:43:09.456414+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:43:09.456414+0000","last_undegraded":"2024-08-17T17:43:09.456414+0000","last_fullsized":"2024-08-17T17:43:09.456414+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":538,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455758+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:43:09.455758+0000","last_peered":"2024-08-17T17:43:09.455758+0000","last_clean":"2024-08-17T17:43:09.455758+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:43:09.455758+0000","last_undegraded":"2024-08-17T17:43:09.455758+0000","last_fullsized":"2024-08-17T17:43:09.455758+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"393'11602","reported_seq":18567,"reported_epoch":393,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:13.984036+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:43:13.984036+0000","last_peered":"2024-08-17T17:43:13.984036+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:43:13.984036+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:43:13.984036+0000","mapping_epoch":337,"log_start":"328'6044","ondisk_log_start":"328'6044","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5558,"log_dups_size":0,"ondisk_log_size":5558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215810072,"num_objects":3294,"num_object_clones":0,"num_object_copies":9882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3294,"num_objects_misplaced":4044,"num_objects_unfound":0,"num_objects_dirty":3294,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11602,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3404,"num_bytes_recovered":223019008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3294}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":731,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:09.457027+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:43:09.457027+0000","last_peered":"2024-08-17T17:43:09.457027+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:43:09.457027+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":322,"reported_epoch":392,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:42:53.435054+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:42:53.435054+0000","last_peered":"2024-08-17T17:42:53.435054+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:42:53.435054+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:42:53.435054+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":580,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456655+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:43:09.456655+0000","last_peered":"2024-08-17T17:43:09.456655+0000","last_clean":"2024-08-17T17:43:09.456655+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:43:09.456655+0000","last_undegraded":"2024-08-17T17:43:09.456655+0000","last_fullsized":"2024-08-17T17:43:09.456655+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":357,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.431449+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:42:53.431449+0000","last_peered":"2024-08-17T17:42:53.431449+0000","last_clean":"2024-08-17T17:42:53.431449+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:42:53.431449+0000","last_undegraded":"2024-08-17T17:42:53.431449+0000","last_fullsized":"2024-08-17T17:42:53.431449+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"393'11428","reported_seq":11598,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:14.955156+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:43:14.955156+0000","last_peered":"2024-08-17T17:43:14.955156+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:43:14.955156+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'4421","ondisk_log_start":"326'4421","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217317376,"num_objects":3316,"num_object_clones":0,"num_object_copies":9948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11428,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-3331},{"shards":"0,4,7","objects":3316}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":396,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.446803+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:43:09.446803+0000","last_peered":"2024-08-17T17:43:09.446803+0000","last_clean":"2024-08-17T17:43:09.446803+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:43:09.446803+0000","last_undegraded":"2024-08-17T17:43:09.446803+0000","last_fullsized":"2024-08-17T17:43:09.446803+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":329,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427673+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:42:53.427673+0000","last_peered":"2024-08-17T17:42:53.427673+0000","last_clean":"2024-08-17T17:42:53.427673+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:42:53.427673+0000","last_undegraded":"2024-08-17T17:42:53.427673+0000","last_fullsized":"2024-08-17T17:42:53.427673+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":322,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.460956+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:43:09.460956+0000","last_peered":"2024-08-17T17:43:09.460956+0000","last_clean":"2024-08-17T17:43:09.460956+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:43:09.460956+0000","last_undegraded":"2024-08-17T17:43:09.460956+0000","last_fullsized":"2024-08-17T17:43:09.460956+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"393'11481","reported_seq":11573,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:12.083563+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:43:12.083563+0000","last_peered":"2024-08-17T17:43:12.083563+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:43:12.083563+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"327'4917","ondisk_log_start":"327'4917","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6564,"log_dups_size":0,"ondisk_log_size":6564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217251840,"num_objects":3315,"num_object_clones":0,"num_object_copies":9945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11481,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3315},{"shards":"5,6","objects":-3981}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":488,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455845+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:43:09.455845+0000","last_peered":"2024-08-17T17:43:09.455845+0000","last_clean":"2024-08-17T17:43:09.455845+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:43:09.455845+0000","last_undegraded":"2024-08-17T17:43:09.455845+0000","last_fullsized":"2024-08-17T17:43:09.455845+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":129,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.430018+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:42:53.430018+0000","last_peered":"2024-08-17T17:42:53.430018+0000","last_clean":"2024-08-17T17:42:53.430018+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:42:53.430018+0000","last_undegraded":"2024-08-17T17:42:53.430018+0000","last_fullsized":"2024-08-17T17:42:53.430018+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":673,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.448420+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:43:09.448420+0000","last_peered":"2024-08-17T17:43:09.448420+0000","last_clean":"2024-08-17T17:43:09.448420+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:43:09.448420+0000","last_undegraded":"2024-08-17T17:43:09.448420+0000","last_fullsized":"2024-08-17T17:43:09.448420+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"393'11406","reported_seq":12300,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:14.660401+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:43:14.660401+0000","last_peered":"2024-08-17T17:43:14.660401+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:43:14.660401+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'4416","ondisk_log_start":"326'4416","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212336640,"num_objects":3240,"num_object_clones":0,"num_object_copies":9720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11406,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3240},{"shards":"0,5","objects":-3414}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":464,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.447293+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:43:09.447293+0000","last_peered":"2024-08-17T17:43:09.447293+0000","last_clean":"2024-08-17T17:43:09.447293+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:43:09.447293+0000","last_undegraded":"2024-08-17T17:43:09.447293+0000","last_fullsized":"2024-08-17T17:43:09.447293+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":368,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428274+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:42:53.428274+0000","last_peered":"2024-08-17T17:42:53.428274+0000","last_clean":"2024-08-17T17:42:53.428274+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:42:53.428274+0000","last_undegraded":"2024-08-17T17:42:53.428274+0000","last_fullsized":"2024-08-17T17:42:53.428274+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":259,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455615+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:43:09.455615+0000","last_peered":"2024-08-17T17:43:09.455615+0000","last_clean":"2024-08-17T17:43:09.455615+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:43:09.455615+0000","last_undegraded":"2024-08-17T17:43:09.455615+0000","last_fullsized":"2024-08-17T17:43:09.455615+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"393'11543","reported_seq":26556,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:12.643297+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:43:12.643297+0000","last_peered":"2024-08-17T17:43:12.643297+0000","last_clean":"2024-08-17T17:43:12.643297+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:43:12.643297+0000","last_undegraded":"2024-08-17T17:43:12.643297+0000","last_fullsized":"2024-08-17T17:43:12.643297+0000","mapping_epoch":355,"log_start":"327'5230","ondisk_log_start":"327'5230","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215810048,"num_objects":3293,"num_object_clones":0,"num_object_copies":9879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11543,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":307,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.434683+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:42:53.434683+0000","last_peered":"2024-08-17T17:42:53.434683+0000","last_clean":"2024-08-17T17:42:53.434683+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:42:53.434683+0000","last_undegraded":"2024-08-17T17:42:53.434683+0000","last_fullsized":"2024-08-17T17:42:53.434683+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":128,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455530+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:43:09.455530+0000","last_peered":"2024-08-17T17:43:09.455530+0000","last_clean":"2024-08-17T17:43:09.455530+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:43:09.455530+0000","last_undegraded":"2024-08-17T17:43:09.455530+0000","last_fullsized":"2024-08-17T17:43:09.455530+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":623,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.447494+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:43:09.447494+0000","last_peered":"2024-08-17T17:43:09.447494+0000","last_clean":"2024-08-17T17:43:09.447494+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:43:09.447494+0000","last_undegraded":"2024-08-17T17:43:09.447494+0000","last_fullsized":"2024-08-17T17:43:09.447494+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"393'11217","reported_seq":25798,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:12.598114+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:43:12.598114+0000","last_peered":"2024-08-17T17:43:12.598114+0000","last_clean":"2024-08-17T17:43:12.598114+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:43:12.598114+0000","last_undegraded":"2024-08-17T17:43:12.598114+0000","last_fullsized":"2024-08-17T17:43:12.598114+0000","mapping_epoch":345,"log_start":"327'4872","ondisk_log_start":"327'4872","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6345,"log_dups_size":0,"ondisk_log_size":6345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208863232,"num_objects":3187,"num_object_clones":0,"num_object_copies":9561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11217,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":303,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435356+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:42:53.435356+0000","last_peered":"2024-08-17T17:42:53.435356+0000","last_clean":"2024-08-17T17:42:53.435356+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:42:53.435356+0000","last_undegraded":"2024-08-17T17:42:53.435356+0000","last_fullsized":"2024-08-17T17:42:53.435356+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":379,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456717+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:43:09.456717+0000","last_peered":"2024-08-17T17:43:09.456717+0000","last_clean":"2024-08-17T17:43:09.456717+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:43:09.456717+0000","last_undegraded":"2024-08-17T17:43:09.456717+0000","last_fullsized":"2024-08-17T17:43:09.456717+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":445,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.434962+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:42:53.434962+0000","last_peered":"2024-08-17T17:42:53.434962+0000","last_clean":"2024-08-17T17:42:53.434962+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:42:53.434962+0000","last_undegraded":"2024-08-17T17:42:53.434962+0000","last_fullsized":"2024-08-17T17:42:53.434962+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"393'11431","reported_seq":11620,"reported_epoch":393,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:12.630385+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:43:12.630385+0000","last_peered":"2024-08-17T17:43:12.630385+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:43:12.630385+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'5087","ondisk_log_start":"327'5087","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11431,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":3353},{"shards":"0,3","objects":-3895}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":654,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455851+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:43:09.455851+0000","last_peered":"2024-08-17T17:43:09.455851+0000","last_clean":"2024-08-17T17:43:09.455851+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:43:09.455851+0000","last_undegraded":"2024-08-17T17:43:09.455851+0000","last_fullsized":"2024-08-17T17:43:09.455851+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":622,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.449081+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:43:09.449081+0000","last_peered":"2024-08-17T17:43:09.449081+0000","last_clean":"2024-08-17T17:43:09.449081+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:43:09.449081+0000","last_undegraded":"2024-08-17T17:43:09.449081+0000","last_fullsized":"2024-08-17T17:43:09.449081+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":640,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456232+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:43:09.456232+0000","last_peered":"2024-08-17T17:43:09.456232+0000","last_clean":"2024-08-17T17:43:09.456232+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:43:09.456232+0000","last_undegraded":"2024-08-17T17:43:09.456232+0000","last_fullsized":"2024-08-17T17:43:09.456232+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"393'11324","reported_seq":26014,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:11.858454+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:43:11.858454+0000","last_peered":"2024-08-17T17:43:11.858454+0000","last_clean":"2024-08-17T17:43:11.858454+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:43:11.858454+0000","last_undegraded":"2024-08-17T17:43:11.858454+0000","last_fullsized":"2024-08-17T17:43:11.858454+0000","mapping_epoch":347,"log_start":"327'4772","ondisk_log_start":"327'4772","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212992000,"num_objects":3250,"num_object_clones":0,"num_object_copies":9750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":312,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.457573+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:43:09.457573+0000","last_peered":"2024-08-17T17:43:09.457573+0000","last_clean":"2024-08-17T17:43:09.457573+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:43:09.457573+0000","last_undegraded":"2024-08-17T17:43:09.457573+0000","last_fullsized":"2024-08-17T17:43:09.457573+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":397,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.430701+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:42:53.430701+0000","last_peered":"2024-08-17T17:42:53.430701+0000","last_clean":"2024-08-17T17:42:53.430701+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:42:53.430701+0000","last_undegraded":"2024-08-17T17:42:53.430701+0000","last_fullsized":"2024-08-17T17:42:53.430701+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":347,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427692+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:42:53.427692+0000","last_peered":"2024-08-17T17:42:53.427692+0000","last_clean":"2024-08-17T17:42:53.427692+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:42:53.427692+0000","last_undegraded":"2024-08-17T17:42:53.427692+0000","last_fullsized":"2024-08-17T17:42:53.427692+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":347,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428205+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:42:53.428205+0000","last_peered":"2024-08-17T17:42:53.428205+0000","last_clean":"2024-08-17T17:42:53.428205+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:42:53.428205+0000","last_undegraded":"2024-08-17T17:42:53.428205+0000","last_fullsized":"2024-08-17T17:42:53.428205+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":390,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427991+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:42:53.427991+0000","last_peered":"2024-08-17T17:42:53.427991+0000","last_clean":"2024-08-17T17:42:53.427991+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:42:53.427991+0000","last_undegraded":"2024-08-17T17:42:53.427991+0000","last_fullsized":"2024-08-17T17:42:53.427991+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":629,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.447295+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:43:09.447295+0000","last_peered":"2024-08-17T17:43:09.447295+0000","last_clean":"2024-08-17T17:43:09.447295+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:43:09.447295+0000","last_undegraded":"2024-08-17T17:43:09.447295+0000","last_fullsized":"2024-08-17T17:43:09.447295+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":168,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.458751+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:43:09.458751+0000","last_peered":"2024-08-17T17:43:09.458751+0000","last_clean":"2024-08-17T17:43:09.458751+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:43:09.458751+0000","last_undegraded":"2024-08-17T17:43:09.458751+0000","last_fullsized":"2024-08-17T17:43:09.458751+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":297,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435058+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:42:53.435058+0000","last_peered":"2024-08-17T17:42:53.435058+0000","last_clean":"2024-08-17T17:42:53.435058+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:42:53.435058+0000","last_undegraded":"2024-08-17T17:42:53.435058+0000","last_fullsized":"2024-08-17T17:42:53.435058+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":587,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455468+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:43:09.455468+0000","last_peered":"2024-08-17T17:43:09.455468+0000","last_clean":"2024-08-17T17:43:09.455468+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:43:09.455468+0000","last_undegraded":"2024-08-17T17:43:09.455468+0000","last_fullsized":"2024-08-17T17:43:09.455468+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":441,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.435513+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:42:53.435513+0000","last_peered":"2024-08-17T17:42:53.435513+0000","last_clean":"2024-08-17T17:42:53.435513+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:42:53.435513+0000","last_undegraded":"2024-08-17T17:42:53.435513+0000","last_fullsized":"2024-08-17T17:42:53.435513+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":581,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455365+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:43:09.455365+0000","last_peered":"2024-08-17T17:43:09.455365+0000","last_clean":"2024-08-17T17:43:09.455365+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:43:09.455365+0000","last_undegraded":"2024-08-17T17:43:09.455365+0000","last_fullsized":"2024-08-17T17:43:09.455365+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":364,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.430662+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:42:53.430662+0000","last_peered":"2024-08-17T17:42:53.430662+0000","last_clean":"2024-08-17T17:42:53.430662+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:42:53.430662+0000","last_undegraded":"2024-08-17T17:42:53.430662+0000","last_fullsized":"2024-08-17T17:42:53.430662+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":612,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456325+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:43:09.456325+0000","last_peered":"2024-08-17T17:43:09.456325+0000","last_clean":"2024-08-17T17:43:09.456325+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:43:09.456325+0000","last_undegraded":"2024-08-17T17:43:09.456325+0000","last_fullsized":"2024-08-17T17:43:09.456325+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":328,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456518+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:43:09.456518+0000","last_peered":"2024-08-17T17:43:09.456518+0000","last_clean":"2024-08-17T17:43:09.456518+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:43:09.456518+0000","last_undegraded":"2024-08-17T17:43:09.456518+0000","last_fullsized":"2024-08-17T17:43:09.456518+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":620,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.449594+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:43:09.449594+0000","last_peered":"2024-08-17T17:43:09.449594+0000","last_clean":"2024-08-17T17:43:09.449594+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:43:09.449594+0000","last_undegraded":"2024-08-17T17:43:09.449594+0000","last_fullsized":"2024-08-17T17:43:09.449594+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":580,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456436+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:43:09.456436+0000","last_peered":"2024-08-17T17:43:09.456436+0000","last_clean":"2024-08-17T17:43:09.456436+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:43:09.456436+0000","last_undegraded":"2024-08-17T17:43:09.456436+0000","last_fullsized":"2024-08-17T17:43:09.456436+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":376,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428358+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:42:53.428358+0000","last_peered":"2024-08-17T17:42:53.428358+0000","last_clean":"2024-08-17T17:42:53.428358+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:42:53.428358+0000","last_undegraded":"2024-08-17T17:42:53.428358+0000","last_fullsized":"2024-08-17T17:42:53.428358+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":268,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455778+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:43:09.455778+0000","last_peered":"2024-08-17T17:43:09.455778+0000","last_clean":"2024-08-17T17:43:09.455778+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:43:09.455778+0000","last_undegraded":"2024-08-17T17:43:09.455778+0000","last_fullsized":"2024-08-17T17:43:09.455778+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":325,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.457383+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:43:09.457383+0000","last_peered":"2024-08-17T17:43:09.457383+0000","last_clean":"2024-08-17T17:43:09.457383+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:43:09.457383+0000","last_undegraded":"2024-08-17T17:43:09.457383+0000","last_fullsized":"2024-08-17T17:43:09.457383+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":384,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428174+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:42:53.428174+0000","last_peered":"2024-08-17T17:42:53.428174+0000","last_clean":"2024-08-17T17:42:53.428174+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:42:53.428174+0000","last_undegraded":"2024-08-17T17:42:53.428174+0000","last_fullsized":"2024-08-17T17:42:53.428174+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":247,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427909+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:42:53.427909+0000","last_peered":"2024-08-17T17:42:53.427909+0000","last_clean":"2024-08-17T17:42:53.427909+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:42:53.427909+0000","last_undegraded":"2024-08-17T17:42:53.427909+0000","last_fullsized":"2024-08-17T17:42:53.427909+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":384,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427730+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:42:53.427730+0000","last_peered":"2024-08-17T17:42:53.427730+0000","last_clean":"2024-08-17T17:42:53.427730+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:42:53.427730+0000","last_undegraded":"2024-08-17T17:42:53.427730+0000","last_fullsized":"2024-08-17T17:42:53.427730+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":247,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.428048+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:42:53.428048+0000","last_peered":"2024-08-17T17:42:53.428048+0000","last_clean":"2024-08-17T17:42:53.428048+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:42:53.428048+0000","last_undegraded":"2024-08-17T17:42:53.428048+0000","last_fullsized":"2024-08-17T17:42:53.428048+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":632,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.455942+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:43:09.455942+0000","last_peered":"2024-08-17T17:43:09.455942+0000","last_clean":"2024-08-17T17:43:09.455942+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:43:09.455942+0000","last_undegraded":"2024-08-17T17:43:09.455942+0000","last_fullsized":"2024-08-17T17:43:09.455942+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":606,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.456688+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:43:09.456688+0000","last_peered":"2024-08-17T17:43:09.456688+0000","last_clean":"2024-08-17T17:43:09.456688+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:43:09.456688+0000","last_undegraded":"2024-08-17T17:43:09.456688+0000","last_fullsized":"2024-08-17T17:43:09.456688+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":347,"reported_epoch":392,"state":"active+clean","last_fresh":"2024-08-17T17:42:53.427904+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:42:53.427904+0000","last_peered":"2024-08-17T17:42:53.427904+0000","last_clean":"2024-08-17T17:42:53.427904+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:42:53.427904+0000","last_undegraded":"2024-08-17T17:42:53.427904+0000","last_fullsized":"2024-08-17T17:42:53.427904+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":327,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.457032+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:43:09.457032+0000","last_peered":"2024-08-17T17:43:09.457032+0000","last_clean":"2024-08-17T17:43:09.457032+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:43:09.457032+0000","last_undegraded":"2024-08-17T17:43:09.457032+0000","last_fullsized":"2024-08-17T17:43:09.457032+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3472228376,"num_objects":52983,"num_object_clones":0,"num_object_copies":158949,"num_objects_missing_on_primary":3941,"num_objects_missing":3941,"num_objects_degraded":50961,"num_objects_misplaced":21414,"num_objects_unfound":0,"num_objects_dirty":52983,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183407,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32402,"num_bytes_recovered":2000945152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12671856640,"data_stored":12671844424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104726,"ondisk_log_size":104726,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170122,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463491,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2398312,"kb_used_data":1500448,"kb_used_omap":1,"kb_used_meta":897854,"kb_avail":91346840,"statfs":{"total":95995035648,"available":93539164160,"internally_reserved":0,"allocated":1536458752,"data_stored":1534295864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919402840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492166,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":827916,"kb_used_data":424260,"kb_used_omap":7,"kb_used_meta":403640,"kb_avail":92917236,"statfs":{"total":95995035648,"available":95147249664,"internally_reserved":0,"allocated":434442240,"data_stored":432301652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":413327693},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986757,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2348444,"kb_used_data":843924,"kb_used_omap":1,"kb_used_meta":1504510,"kb_avail":91396708,"statfs":{"total":95995035648,"available":93590228992,"internally_reserved":0,"allocated":864178176,"data_stored":862026228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1540618584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225585,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2656972,"kb_used_data":1705156,"kb_used_omap":4,"kb_used_meta":951803,"kb_avail":91088180,"statfs":{"total":95995035648,"available":93274296320,"internally_reserved":0,"allocated":1746079744,"data_stored":1743939454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":974646720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075910,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533803,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2552784,"kb_used_data":1488584,"kb_used_omap":5,"kb_used_meta":1064186,"kb_avail":91192368,"statfs":{"total":95995035648,"available":93380984832,"internally_reserved":0,"allocated":1524310016,"data_stored":1522164404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1089726704},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089780,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2288488,"kb_used_data":1275104,"kb_used_omap":5,"kb_used_meta":1013370,"kb_avail":91456664,"statfs":{"total":95995035648,"available":93651623936,"internally_reserved":0,"allocated":1305706496,"data_stored":1303543667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1037690939},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1299578880,"data_stored":1299578880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1514409984,"data_stored":1514405912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1737490432,"data_stored":1737490432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":858456064,"data_stored":858456064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":428081152,"data_stored":428081152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1527185408,"data_stored":1527185408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1967394816,"data_stored":1967390744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:19.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:43:19.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:19.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:19.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:19.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:19.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:19.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:19.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:19.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:19.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:19.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:19 smithi142 ceph-mon[103614]: from='client.47663 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:19.704 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:19 smithi142 ceph-mon[103614]: pgmap v1515: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 50964/157791 objects degraded (32.298%); 21369/157791 objects misplaced (13.543%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:43:19.704 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:19 smithi142 ceph-mon[106729]: from='client.47663 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:19.704 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:19 smithi142 ceph-mon[106729]: pgmap v1515: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 50964/157791 objects degraded (32.298%); 21369/157791 objects misplaced (13.543%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:43:19.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:19.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:19 smithi152 ceph-mon[93456]: from='client.47663 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:19 smithi152 ceph-mon[93456]: pgmap v1515: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 50964/157791 objects degraded (32.298%); 21369/157791 objects misplaced (13.543%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T17:43:19.869 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:19.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:19.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:19.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:20.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:20.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:20.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:20.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:20.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:20.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:20.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:43:20.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:20.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:20.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:20.514 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:20.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:43:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:43:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:20.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:20.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:20.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:20.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:20.761 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:20.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:20.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:20.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:20.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:21.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:21.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.141 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:21.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:21.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:21.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:21.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:21.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:21.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:21.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:21.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:21.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:21.653 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:21.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:21.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:21.770 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:184575 12.1 e393/337 12:84393136:::benchmark_data_smithi142_261410_object66378:head v 393'11678, mlcod=393'11676)", 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:21.436600+0000", 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.00118023, 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012068179999999999, 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:21.771 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436600+0000", 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436600+0000", 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436601+0000", 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6199999999999998e-07 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436605+0000", 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2420000000000002e-06 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436616+0000", 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1165000000000001e-05 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436623+0000", 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:43:21.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436676+0000", 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2604e-05 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:21.436699+0000", 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3040999999999998e-05 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:21.774 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:21.775 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:43:21.775 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 1 2024-08-17T17:43:21.775 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:43:21.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:21 smithi142 ceph-mon[103614]: pgmap v1516: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 178 op/s; 50864/156681 objects degraded (32.463%); 21142/156681 objects misplaced (13.494%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:43:21.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50864/156681 objects degraded (32.463%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:21.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:21 smithi142 ceph-mon[106729]: pgmap v1516: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 178 op/s; 50864/156681 objects degraded (32.463%); 21142/156681 objects misplaced (13.494%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:43:21.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50864/156681 objects degraded (32.463%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:21 smithi152 ceph-mon[93456]: pgmap v1516: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 178 op/s; 50864/156681 objects degraded (32.463%); 21142/156681 objects misplaced (13.494%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:43:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50864/156681 objects degraded (32.463%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:21.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:21.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:21.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:21.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:22.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:22.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:43:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:43:21] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-17T17:43:22.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:22.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:22.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:22.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:22.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:22.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:22.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:22.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:22.610 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:43:22.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:22.629 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:22.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:22.752 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:22.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:22.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:22.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:22.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:23.000 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:23.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:23.125 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:23.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:23.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:23.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:23.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:23.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:23.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:23.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:23.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:23.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:23.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:23.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:23.759 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:23 smithi142 ceph-mon[106729]: pgmap v1517: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 50864/156021 objects degraded (32.601%); 21102/156021 objects misplaced (13.525%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:23.759 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:23 smithi142 ceph-mon[103614]: pgmap v1517: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 50864/156021 objects degraded (32.601%); 21102/156021 objects misplaced (13.525%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:23 smithi152 ceph-mon[93456]: pgmap v1517: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 50864/156021 objects degraded (32.601%); 21102/156021 objects misplaced (13.525%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:23.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:23.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:23.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:23.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:23.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:23.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:24.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:24.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:24.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:24.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:24.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:24.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:24.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:24.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:24.496 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:24.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:24.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:24.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:24.742 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:24.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:24.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:24.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:24.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:24.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:24.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:25.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:25.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:25.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:25.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:25.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:25.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:25.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:25.466 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:25.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:25.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:25.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:25.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:25.727 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:25 smithi142 ceph-mon[106729]: pgmap v1518: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 153 op/s; 50826/155364 objects degraded (32.714%); 21014/155364 objects misplaced (13.526%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:43:25.727 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:25 smithi142 ceph-mon[103614]: pgmap v1518: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 153 op/s; 50826/155364 objects degraded (32.714%); 21014/155364 objects misplaced (13.526%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:43:25.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:25 smithi152 ceph-mon[93456]: pgmap v1518: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 153 op/s; 50826/155364 objects degraded (32.714%); 21014/155364 objects misplaced (13.526%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:43:25.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:25.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:25.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:25.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:25.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:26.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:26.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:26.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:26.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:26.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:26.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:26.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:26.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:26.473 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:43:26.487 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:26.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:26.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:26.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:26.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:26.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50826/155364 objects degraded (32.714%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:26 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/85717155' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:26.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:26.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50826/155364 objects degraded (32.714%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:26 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/85717155' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50826/155364 objects degraded (32.714%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:26 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/85717155' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:26.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:26.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:26.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:26.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:27.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:27.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:27.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:27.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:27.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:27.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:27.468 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:27.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:27.587 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:27.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:27.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:27.726 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:27.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:27.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[106729]: pgmap v1519: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 153 op/s; 50778/154512 objects degraded (32.863%); 20862/154512 objects misplaced (13.502%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:27.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:27.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/85717155' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:27.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[106729]: osdmap e394: 8 total, 8 up, 6 in 2024-08-17T17:43:27.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[103614]: pgmap v1519: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 153 op/s; 50778/154512 objects degraded (32.863%); 20862/154512 objects misplaced (13.502%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:27.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:27.776 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/85717155' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:27.776 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:27 smithi142 ceph-mon[103614]: osdmap e394: 8 total, 8 up, 6 in 2024-08-17T17:43:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:27 smithi152 ceph-mon[93456]: pgmap v1519: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 153 op/s; 50778/154512 objects degraded (32.863%); 20862/154512 objects misplaced (13.502%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:27 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/85717155' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:27 smithi152 ceph-mon[93456]: osdmap e394: 8 total, 8 up, 6 in 2024-08-17T17:43:27.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:27.872 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:27.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:27.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:27.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:28.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:28.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:28.139 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:43:28.139 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:28.157 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:43:28.157 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:28.174 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:43:28.174 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:28.191 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:43:28.192 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:28.208 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:43:28.208 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:28.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:28.224 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:43:28.224 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:28.239 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:43:28.239 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:28.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:28.255 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:43:28.255 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:28.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:28.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:28.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:28.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:28.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:28.609 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:28.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:28.673 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:43:28.673 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 50778/153315 objects degraded (33.120%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2289,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":394,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":51105,"data_bytes":3348043749,"bytes_used":13131444224,"bytes_avail":562838769664,"bytes_total":575970213888,"degraded_objects":50778,"degraded_total":153315,"degraded_ratio":0.33120046962136773,"misplaced_objects":20814,"misplaced_total":153315,"misplaced_ratio":0.13575971040015655,"recovering_objects_per_sec":28,"recovering_bytes_per_sec":1854176,"recovering_keys_per_sec":0,"num_objects_recovered":283,"num_bytes_recovered":18546688,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":149},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":34,"modified":"2024-08-17T17:38:50.299533+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:43:28.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:28.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:28.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/749014584' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:28.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:28.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/749014584' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/749014584' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:28.883 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:28.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:28.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:29.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:29.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:29.109 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:29.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:29.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:29.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:29.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:29.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:29.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:29.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:29.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:29.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:29.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:43:29.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:29.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:29.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:29.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:29 smithi142 ceph-mon[103614]: pgmap v1521: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 50778/153315 objects degraded (33.120%); 20814/153315 objects misplaced (13.576%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:29.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/364756537' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:29.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:29 smithi142 ceph-mon[106729]: pgmap v1521: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 50778/153315 objects degraded (33.120%); 20814/153315 objects misplaced (13.576%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:29.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/364756537' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:29.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:29.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:29 smithi152 ceph-mon[93456]: pgmap v1521: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 50778/153315 objects degraded (33.120%); 20814/153315 objects misplaced (13.576%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:43:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/364756537' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:29.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:29.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:29.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:29.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:30.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:30.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:30.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:30.135 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:30.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:30.236 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:43:30.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:30.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:30.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:30.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:30.367 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:30.393 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:30.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:30.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:43:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:43:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:30.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:30.640 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:30.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:30.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:30.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:30.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:30.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:30.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:30.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:30.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:31.014 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:31.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:31.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:31.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:43:31.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:31.264 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:31.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:31.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:31.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:31.510 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:31 smithi152 ceph-mon[93456]: pgmap v1522: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 50667/152043 objects degraded (33.324%); 20552/152043 objects misplaced (13.517%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50667/152043 objects degraded (33.324%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:31.610 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:31.612 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:31.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:31.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:31.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:31.742 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:31 smithi142 ceph-mon[106729]: pgmap v1522: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 50667/152043 objects degraded (33.324%); 20552/152043 objects misplaced (13.517%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:31.742 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50667/152043 objects degraded (33.324%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:31.742 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:31 smithi142 ceph-mon[103614]: pgmap v1522: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 50667/152043 objects degraded (33.324%); 20552/152043 objects misplaced (13.517%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:31.742 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50667/152043 objects degraded (33.324%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:31.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:31.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:31.889 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:31.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:31.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:32.014 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:32.116 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:43:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:43:31] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:43:32.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:32.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:32.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:32.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:32.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:32.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:32.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:32.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:32.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:32.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:32.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:32.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:32.808 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:32.808 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:32.869 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:32.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:32.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:32.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:33.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:33.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:33.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:33.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:33.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:33.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:33.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:33.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:33.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:33.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:33.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:33.640 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:33.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:33.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:33.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:33 smithi142 ceph-mon[106729]: pgmap v1523: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 50667/151212 objects degraded (33.507%); 20495/151212 objects misplaced (13.554%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:33.747 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:33 smithi142 ceph-mon[103614]: pgmap v1523: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 50667/151212 objects degraded (33.507%); 20495/151212 objects misplaced (13.554%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:33.752 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:33.752 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:33.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:33 smithi152 ceph-mon[93456]: pgmap v1523: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 50667/151212 objects degraded (33.507%); 20495/151212 objects misplaced (13.554%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:33.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:33.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:33.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:33.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:33.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:34.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:34.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:34.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:34.136 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:34.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:34.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:34.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:34.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:34.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:34.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:34.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:34.610 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:34.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:34.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:34.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:34.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:34.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:34.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:34.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:34.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:35.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:35.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:35.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:35.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:43:35.245 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:35.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:35.373 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:35.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:35.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:35.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:35.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:35.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:35.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:35.640 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:35.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:35.765 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:35.786 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:35 smithi142 ceph-mon[106729]: pgmap v1524: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 50628/150453 objects degraded (33.650%); 20392/150453 objects misplaced (13.554%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:35.786 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:35 smithi142 ceph-mon[103614]: pgmap v1524: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 50628/150453 objects degraded (33.650%); 20392/150453 objects misplaced (13.554%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:35 smithi152 ceph-mon[93456]: pgmap v1524: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 50628/150453 objects degraded (33.650%); 20392/150453 objects misplaced (13.554%); 1.8 MiB/s, 29 objects/s recovering 2024-08-17T17:43:35.867 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:35.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:35.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:35.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:36.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:36.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:36.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:36.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:36.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:36.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:36.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:36.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:36.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:36.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:36.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:36.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:36.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:36.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:36.758 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50628/150453 objects degraded (33.650%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:36.758 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50628/150453 objects degraded (33.650%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:36.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:36.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50628/150453 objects degraded (33.650%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:36.873 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:36.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:36.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:36.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:37.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:37.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:37.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:37.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:37.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:37.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:37.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:37.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:37.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:37.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:37.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:37.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:37.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:37.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:37.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:37.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:37.787 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:37 smithi142 ceph-mon[106729]: pgmap v1525: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 168 op/s; 50577/149469 objects degraded (33.838%); 20219/149469 objects misplaced (13.527%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:43:37.787 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:37 smithi142 ceph-mon[103614]: pgmap v1525: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 168 op/s; 50577/149469 objects degraded (33.838%); 20219/149469 objects misplaced (13.527%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:43:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:37 smithi152 ceph-mon[93456]: pgmap v1525: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 168 op/s; 50577/149469 objects degraded (33.838%); 20219/149469 objects misplaced (13.527%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:43:37.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:37.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:37.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:37.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:38.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:38.029 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:38.043 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:43:38.044 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:43:38.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:38.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:38.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:38.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:38.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:38.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:38.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:38.501 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:43:38.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3487297769' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:38.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3487297769' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:38.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:38.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:38.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:38.778 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3487297769' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:43:38.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:38.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:38.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:39.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:39.030 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:39.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:39.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:39.241 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:43:39.241 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:39.248 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:43:39.248 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:39.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:39.268 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:43:39.268 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:39.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:39.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:39.290 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:43:39.290 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:39.311 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:43:39.312 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:39.330 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:43:39.331 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:39.351 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:43:39.351 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:39.368 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:43:39.368 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:39.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:39.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:39.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:39.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:39.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:39.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:39.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:39.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:39.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:39.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:39 smithi152 ceph-mon[93456]: from='client.47687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:39 smithi152 ceph-mon[93456]: pgmap v1526: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 179 op/s; 50577/148110 objects degraded (34.148%); 20162/148110 objects misplaced (13.613%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:43:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:39 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3487297769' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:43:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:39 smithi152 ceph-mon[93456]: osdmap e395: 8 total, 8 up, 6 in 2024-08-17T17:43:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1458898049' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[106729]: from='client.47687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[106729]: pgmap v1526: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 179 op/s; 50577/148110 objects degraded (34.148%); 20162/148110 objects misplaced (13.613%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:43:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3487297769' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:43:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[106729]: osdmap e395: 8 total, 8 up, 6 in 2024-08-17T17:43:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1458898049' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[103614]: from='client.47687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[103614]: pgmap v1526: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 179 op/s; 50577/148110 objects degraded (34.148%); 20162/148110 objects misplaced (13.613%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:43:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:43:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3487297769' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:43:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[103614]: osdmap e395: 8 total, 8 up, 6 in 2024-08-17T17:43:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1458898049' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:39.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:39.903 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:39.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:40.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:40.026 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:40.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:40.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:40.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:40.273 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:40.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:40.401 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:40.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:40.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:43:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:43:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:40.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:40.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:40.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:40.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:40.841 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1525,"stamp":"2024-08-17T17:43:36.378573+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3264026597,"num_objects":49823,"num_object_clones":0,"num_object_copies":149469,"num_objects_missing_on_primary":3731,"num_objects_missing":3731,"num_objects_degraded":50577,"num_objects_misplaced":20219,"num_objects_unfound":0,"num_objects_dirty":49823,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":186871,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33043,"num_bytes_recovered":2042634894,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104819,"ondisk_log_size":104819,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12598452,"kb_used_data":6803204,"kb_used_omap":27,"kb_used_meta":5795172,"kb_avail":549872460,"statfs":{"total":575970213888,"available":563069399040,"internally_reserved":0,"allocated":6966480896,"data_stored":6953529983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5934256872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-110166016,"num_objects":-1681,"num_object_clones":0,"num_object_copies":-5043,"num_objects_missing_on_primary":-103,"num_objects_missing":-103,"num_objects_degraded":-201,"num_objects_misplaced":-643,"num_objects_unfound":0,"num_objects_dirty":-1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1681,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":317,"num_bytes_recovered":20774912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002745"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":632,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477566+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:43:26.477566+0000","last_peered":"2024-08-17T17:43:26.477566+0000","last_clean":"2024-08-17T17:43:26.477566+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:43:26.477566+0000","last_undegraded":"2024-08-17T17:43:26.477566+0000","last_fullsized":"2024-08-17T17:43:26.477566+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":103,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468823+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:43:09.468823+0000","last_peered":"2024-08-17T17:43:09.468823+0000","last_clean":"2024-08-17T17:43:09.468823+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:43:09.468823+0000","last_undegraded":"2024-08-17T17:43:09.468823+0000","last_fullsized":"2024-08-17T17:43:09.468823+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":297,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444844+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:43:09.444844+0000","last_peered":"2024-08-17T17:43:09.444844+0000","last_clean":"2024-08-17T17:43:09.444844+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:43:09.444844+0000","last_undegraded":"2024-08-17T17:43:09.444844+0000","last_fullsized":"2024-08-17T17:43:09.444844+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":330,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477736+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:43:26.477736+0000","last_peered":"2024-08-17T17:43:26.477736+0000","last_clean":"2024-08-17T17:43:26.477736+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:43:26.477736+0000","last_undegraded":"2024-08-17T17:43:26.477736+0000","last_fullsized":"2024-08-17T17:43:26.477736+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":507,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478111+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:43:26.478111+0000","last_peered":"2024-08-17T17:43:26.478111+0000","last_clean":"2024-08-17T17:43:26.478111+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:43:26.478111+0000","last_undegraded":"2024-08-17T17:43:26.478111+0000","last_fullsized":"2024-08-17T17:43:26.478111+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":246,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.467987+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:43:09.467987+0000","last_peered":"2024-08-17T17:43:09.467987+0000","last_clean":"2024-08-17T17:43:09.467987+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:43:09.467987+0000","last_undegraded":"2024-08-17T17:43:09.467987+0000","last_fullsized":"2024-08-17T17:43:09.467987+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":600,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478304+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:43:26.478304+0000","last_peered":"2024-08-17T17:43:26.478304+0000","last_clean":"2024-08-17T17:43:26.478304+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:43:26.478304+0000","last_undegraded":"2024-08-17T17:43:26.478304+0000","last_fullsized":"2024-08-17T17:43:26.478304+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":414,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.449470+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:43:09.449470+0000","last_peered":"2024-08-17T17:43:09.449470+0000","last_clean":"2024-08-17T17:43:09.449470+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:43:09.449470+0000","last_undegraded":"2024-08-17T17:43:09.449470+0000","last_fullsized":"2024-08-17T17:43:09.449470+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"394'11615","reported_seq":18789,"reported_epoch":394,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:43:35.196248+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:43:35.196248+0000","last_peered":"2024-08-17T17:43:35.196248+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:43:35.196248+0000","last_undegraded":"2024-08-17T17:43:35.196248+0000","last_fullsized":"2024-08-17T17:43:35.196248+0000","mapping_epoch":337,"log_start":"327'4763","ondisk_log_start":"327'4763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6852,"log_dups_size":0,"ondisk_log_size":6852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202440704,"num_objects":3089,"num_object_clones":0,"num_object_copies":9267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3089,"num_objects_unfound":0,"num_objects_dirty":3089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11615,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3534,"num_bytes_recovered":231604224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3089}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":99,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.515087+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:43:09.515087+0000","last_peered":"2024-08-17T17:43:09.515087+0000","last_clean":"2024-08-17T17:43:09.515087+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:43:09.515087+0000","last_undegraded":"2024-08-17T17:43:09.515087+0000","last_fullsized":"2024-08-17T17:43:09.515087+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":280,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444708+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:43:09.444708+0000","last_peered":"2024-08-17T17:43:09.444708+0000","last_clean":"2024-08-17T17:43:09.444708+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:43:09.444708+0000","last_undegraded":"2024-08-17T17:43:09.444708+0000","last_fullsized":"2024-08-17T17:43:09.444708+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"394'11793","reported_seq":11885,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:32.170197+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:43:32.170197+0000","last_peered":"2024-08-17T17:43:32.170197+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:43:32.170197+0000","last_undegraded":"2024-08-17T17:43:32.170197+0000","last_fullsized":"2024-08-17T17:43:32.170197+0000","mapping_epoch":337,"log_start":"327'5182","ondisk_log_start":"327'5182","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215023616,"num_objects":3281,"num_object_clones":0,"num_object_copies":9843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3281,"num_objects_unfound":0,"num_objects_dirty":3281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11793,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3281}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":101,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468676+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:43:09.468676+0000","last_peered":"2024-08-17T17:43:09.468676+0000","last_clean":"2024-08-17T17:43:09.468676+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:43:09.468676+0000","last_undegraded":"2024-08-17T17:43:09.468676+0000","last_fullsized":"2024-08-17T17:43:09.468676+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":365,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.467871+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:43:09.467871+0000","last_peered":"2024-08-17T17:43:09.467871+0000","last_clean":"2024-08-17T17:43:09.467871+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:43:09.467871+0000","last_undegraded":"2024-08-17T17:43:09.467871+0000","last_fullsized":"2024-08-17T17:43:09.467871+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"394'11623","reported_seq":11718,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:35.162997+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:43:35.162997+0000","last_peered":"2024-08-17T17:43:35.162997+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:43:35.162997+0000","last_undegraded":"2024-08-17T17:43:35.162997+0000","last_fullsized":"2024-08-17T17:43:35.162997+0000","mapping_epoch":337,"log_start":"327'4753","ondisk_log_start":"327'4753","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203882496,"num_objects":3111,"num_object_clones":0,"num_object_copies":9333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3111,"num_objects_unfound":0,"num_objects_dirty":3111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11623,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3111}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":97,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444837+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:43:09.444837+0000","last_peered":"2024-08-17T17:43:09.444837+0000","last_clean":"2024-08-17T17:43:09.444837+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:43:09.444837+0000","last_undegraded":"2024-08-17T17:43:09.444837+0000","last_fullsized":"2024-08-17T17:43:09.444837+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":375,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.449492+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:43:09.449492+0000","last_peered":"2024-08-17T17:43:09.449492+0000","last_clean":"2024-08-17T17:43:09.449492+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:43:09.449492+0000","last_undegraded":"2024-08-17T17:43:09.449492+0000","last_fullsized":"2024-08-17T17:43:09.449492+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"394'11842","reported_seq":12011,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:34.973999+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:43:34.973999+0000","last_peered":"2024-08-17T17:43:34.973999+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:43:34.973999+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'4813","ondisk_log_start":"327'4813","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204210176,"num_objects":3116,"num_object_clones":0,"num_object_copies":9348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11842,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-3653},{"shards":"0,3,7","objects":3116}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":544,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477716+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:43:26.477716+0000","last_peered":"2024-08-17T17:43:26.477716+0000","last_clean":"2024-08-17T17:43:26.477716+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:43:26.477716+0000","last_undegraded":"2024-08-17T17:43:26.477716+0000","last_fullsized":"2024-08-17T17:43:26.477716+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":310,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.515146+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:43:26.515146+0000","last_peered":"2024-08-17T17:43:26.515146+0000","last_clean":"2024-08-17T17:43:26.515146+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:43:26.515146+0000","last_undegraded":"2024-08-17T17:43:26.515146+0000","last_fullsized":"2024-08-17T17:43:26.515146+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"394'11481","reported_seq":11564,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:32.178758+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:43:32.178758+0000","last_peered":"2024-08-17T17:43:32.178758+0000","last_clean":"2024-08-17T17:43:32.178758+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:43:32.178758+0000","last_undegraded":"2024-08-17T17:43:32.178758+0000","last_fullsized":"2024-08-17T17:43:32.178758+0000","mapping_epoch":319,"log_start":"327'4874","ondisk_log_start":"327'4874","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204013568,"num_objects":3113,"num_object_clones":0,"num_object_copies":9339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11481,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":633,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478155+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:43:26.478155+0000","last_peered":"2024-08-17T17:43:26.478155+0000","last_clean":"2024-08-17T17:43:26.478155+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:43:26.478155+0000","last_undegraded":"2024-08-17T17:43:26.478155+0000","last_fullsized":"2024-08-17T17:43:26.478155+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":348,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468396+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:43:09.468396+0000","last_peered":"2024-08-17T17:43:09.468396+0000","last_clean":"2024-08-17T17:43:09.468396+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:43:09.468396+0000","last_undegraded":"2024-08-17T17:43:09.468396+0000","last_fullsized":"2024-08-17T17:43:09.468396+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"394'11738","reported_seq":11831,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:32.180967+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:43:32.180967+0000","last_peered":"2024-08-17T17:43:32.180967+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:43:32.180967+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'5135","ondisk_log_start":"327'5135","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6603,"log_dups_size":0,"ondisk_log_size":6603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11738,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3194},{"shards":"1,6","objects":-4172}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":349,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.449614+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:43:09.449614+0000","last_peered":"2024-08-17T17:43:09.449614+0000","last_clean":"2024-08-17T17:43:09.449614+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:43:09.449614+0000","last_undegraded":"2024-08-17T17:43:09.449614+0000","last_fullsized":"2024-08-17T17:43:09.449614+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":375,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468481+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:43:09.468481+0000","last_peered":"2024-08-17T17:43:09.468481+0000","last_clean":"2024-08-17T17:43:09.468481+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:43:09.468481+0000","last_undegraded":"2024-08-17T17:43:09.468481+0000","last_fullsized":"2024-08-17T17:43:09.468481+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"394'11870","reported_seq":12040,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:33.926827+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:43:33.926827+0000","last_peered":"2024-08-17T17:43:33.926827+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:43:33.926827+0000","last_undegraded":"2024-08-17T17:43:33.926827+0000","last_fullsized":"2024-08-17T17:43:33.926827+0000","mapping_epoch":337,"log_start":"328'6261","ondisk_log_start":"328'6261","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205783040,"num_objects":3140,"num_object_clones":0,"num_object_copies":9420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3140,"num_objects_unfound":0,"num_objects_dirty":3140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11870,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3140}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":386,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468130+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:43:09.468130+0000","last_peered":"2024-08-17T17:43:09.468130+0000","last_clean":"2024-08-17T17:43:09.468130+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:43:09.468130+0000","last_undegraded":"2024-08-17T17:43:09.468130+0000","last_fullsized":"2024-08-17T17:43:09.468130+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":348,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468800+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:43:09.468800+0000","last_peered":"2024-08-17T17:43:09.468800+0000","last_clean":"2024-08-17T17:43:09.468800+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:43:09.468800+0000","last_undegraded":"2024-08-17T17:43:09.468800+0000","last_fullsized":"2024-08-17T17:43:09.468800+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"394'11666","reported_seq":18995,"reported_epoch":394,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:35.144782+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:43:35.144782+0000","last_peered":"2024-08-17T17:43:35.144782+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:43:35.144782+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:43:35.144782+0000","mapping_epoch":337,"log_start":"327'4841","ondisk_log_start":"327'4841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6825,"log_dups_size":0,"ondisk_log_size":6825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199098368,"num_objects":3038,"num_object_clones":0,"num_object_copies":9114,"num_objects_missing_on_primary":3731,"num_objects_missing":3731,"num_objects_degraded":3038,"num_objects_misplaced":3731,"num_objects_unfound":0,"num_objects_dirty":3038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11666,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3608,"num_bytes_recovered":236388352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":3025}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":603,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477394+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:43:26.477394+0000","last_peered":"2024-08-17T17:43:26.477394+0000","last_clean":"2024-08-17T17:43:26.477394+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:43:26.477394+0000","last_undegraded":"2024-08-17T17:43:26.477394+0000","last_fullsized":"2024-08-17T17:43:26.477394+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":540,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478380+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:43:26.478380+0000","last_peered":"2024-08-17T17:43:26.478380+0000","last_clean":"2024-08-17T17:43:26.478380+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:43:26.478380+0000","last_undegraded":"2024-08-17T17:43:26.478380+0000","last_fullsized":"2024-08-17T17:43:26.478380+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"394'11779","reported_seq":19100,"reported_epoch":394,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:33.966818+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:43:33.966818+0000","last_peered":"2024-08-17T17:43:33.966818+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:43:33.966818+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:43:33.966818+0000","mapping_epoch":337,"log_start":"328'6144","ondisk_log_start":"328'6144","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204210200,"num_objects":3117,"num_object_clones":0,"num_object_copies":9351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3117,"num_objects_misplaced":3867,"num_objects_unfound":0,"num_objects_dirty":3117,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11779,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3581,"num_bytes_recovered":234618880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3117}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":733,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:26.514796+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:43:26.514796+0000","last_peered":"2024-08-17T17:43:26.514796+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:43:26.514796+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":323,"reported_epoch":393,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:43:09.444751+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:43:09.444751+0000","last_peered":"2024-08-17T17:43:09.444751+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:43:09.444751+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:43:09.444751+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":582,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477658+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:43:26.477658+0000","last_peered":"2024-08-17T17:43:26.477658+0000","last_clean":"2024-08-17T17:43:26.477658+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:43:26.477658+0000","last_undegraded":"2024-08-17T17:43:26.477658+0000","last_fullsized":"2024-08-17T17:43:26.477658+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":358,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.450170+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:43:09.450170+0000","last_peered":"2024-08-17T17:43:09.450170+0000","last_clean":"2024-08-17T17:43:09.450170+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:43:09.450170+0000","last_undegraded":"2024-08-17T17:43:09.450170+0000","last_fullsized":"2024-08-17T17:43:09.450170+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"394'11625","reported_seq":11797,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:34.915200+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:43:34.915200+0000","last_peered":"2024-08-17T17:43:34.915200+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:43:34.915200+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"326'4621","ondisk_log_start":"326'4621","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204406784,"num_objects":3119,"num_object_clones":0,"num_object_copies":9357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11625,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-3528},{"shards":"0,4,7","objects":3119}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":398,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477053+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:43:26.477053+0000","last_peered":"2024-08-17T17:43:26.477053+0000","last_clean":"2024-08-17T17:43:26.477053+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:43:26.477053+0000","last_undegraded":"2024-08-17T17:43:26.477053+0000","last_fullsized":"2024-08-17T17:43:26.477053+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":330,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.467745+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:43:09.467745+0000","last_peered":"2024-08-17T17:43:09.467745+0000","last_clean":"2024-08-17T17:43:09.467745+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:43:09.467745+0000","last_undegraded":"2024-08-17T17:43:09.467745+0000","last_fullsized":"2024-08-17T17:43:09.467745+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":324,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.515460+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:43:26.515460+0000","last_peered":"2024-08-17T17:43:26.515460+0000","last_clean":"2024-08-17T17:43:26.515460+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:43:26.515460+0000","last_undegraded":"2024-08-17T17:43:26.515460+0000","last_fullsized":"2024-08-17T17:43:26.515460+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"394'11680","reported_seq":11773,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:32.152423+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:43:32.152423+0000","last_peered":"2024-08-17T17:43:32.152423+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:43:32.152423+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"327'5117","ondisk_log_start":"327'5117","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204210176,"num_objects":3116,"num_object_clones":0,"num_object_copies":9348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11680,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3116},{"shards":"5,6","objects":-4180}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":490,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477244+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:43:26.477244+0000","last_peered":"2024-08-17T17:43:26.477244+0000","last_clean":"2024-08-17T17:43:26.477244+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:43:26.477244+0000","last_undegraded":"2024-08-17T17:43:26.477244+0000","last_fullsized":"2024-08-17T17:43:26.477244+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.449317+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:43:09.449317+0000","last_peered":"2024-08-17T17:43:09.449317+0000","last_clean":"2024-08-17T17:43:09.449317+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:43:09.449317+0000","last_undegraded":"2024-08-17T17:43:09.449317+0000","last_fullsized":"2024-08-17T17:43:09.449317+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":675,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478618+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:43:26.478618+0000","last_peered":"2024-08-17T17:43:26.478618+0000","last_clean":"2024-08-17T17:43:26.478618+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:43:26.478618+0000","last_undegraded":"2024-08-17T17:43:26.478618+0000","last_fullsized":"2024-08-17T17:43:26.478618+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"394'11596","reported_seq":12492,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:34.931209+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:43:34.931209+0000","last_peered":"2024-08-17T17:43:34.931209+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:43:34.931209+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'4616","ondisk_log_start":"326'4616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199884800,"num_objects":3050,"num_object_clones":0,"num_object_copies":9150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11596,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":3050},{"shards":"0,5","objects":-3604}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":467,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477865+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:43:26.477865+0000","last_peered":"2024-08-17T17:43:26.477865+0000","last_clean":"2024-08-17T17:43:26.477865+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:43:26.477865+0000","last_undegraded":"2024-08-17T17:43:26.477865+0000","last_fullsized":"2024-08-17T17:43:26.477865+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":369,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.469058+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:43:09.469058+0000","last_peered":"2024-08-17T17:43:09.469058+0000","last_clean":"2024-08-17T17:43:09.469058+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:43:09.469058+0000","last_undegraded":"2024-08-17T17:43:09.469058+0000","last_fullsized":"2024-08-17T17:43:09.469058+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":261,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477489+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:43:26.477489+0000","last_peered":"2024-08-17T17:43:26.477489+0000","last_clean":"2024-08-17T17:43:26.477489+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:43:26.477489+0000","last_undegraded":"2024-08-17T17:43:26.477489+0000","last_fullsized":"2024-08-17T17:43:26.477489+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"394'11727","reported_seq":26742,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:32.651413+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:43:32.651413+0000","last_peered":"2024-08-17T17:43:32.651413+0000","last_clean":"2024-08-17T17:43:32.651413+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:43:32.651413+0000","last_undegraded":"2024-08-17T17:43:32.651413+0000","last_fullsized":"2024-08-17T17:43:32.651413+0000","mapping_epoch":355,"log_start":"327'5430","ondisk_log_start":"327'5430","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6297,"log_dups_size":0,"ondisk_log_size":6297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203751424,"num_objects":3109,"num_object_clones":0,"num_object_copies":9327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11727,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":308,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444769+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:43:09.444769+0000","last_peered":"2024-08-17T17:43:09.444769+0000","last_clean":"2024-08-17T17:43:09.444769+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:43:09.444769+0000","last_undegraded":"2024-08-17T17:43:09.444769+0000","last_fullsized":"2024-08-17T17:43:09.444769+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":130,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477412+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:43:26.477412+0000","last_peered":"2024-08-17T17:43:26.477412+0000","last_clean":"2024-08-17T17:43:26.477412+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:43:26.477412+0000","last_undegraded":"2024-08-17T17:43:26.477412+0000","last_fullsized":"2024-08-17T17:43:26.477412+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":626,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477685+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:43:26.477685+0000","last_peered":"2024-08-17T17:43:26.477685+0000","last_clean":"2024-08-17T17:43:26.477685+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:43:26.477685+0000","last_undegraded":"2024-08-17T17:43:26.477685+0000","last_fullsized":"2024-08-17T17:43:26.477685+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"394'11430","reported_seq":26013,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:32.720726+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:43:32.720726+0000","last_peered":"2024-08-17T17:43:32.720726+0000","last_clean":"2024-08-17T17:43:32.720726+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:43:32.720726+0000","last_undegraded":"2024-08-17T17:43:32.720726+0000","last_fullsized":"2024-08-17T17:43:32.720726+0000","mapping_epoch":345,"log_start":"327'5172","ondisk_log_start":"327'5172","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194904064,"num_objects":2974,"num_object_clones":0,"num_object_copies":8922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11430,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":304,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444866+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:43:09.444866+0000","last_peered":"2024-08-17T17:43:09.444866+0000","last_clean":"2024-08-17T17:43:09.444866+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:43:09.444866+0000","last_undegraded":"2024-08-17T17:43:09.444866+0000","last_fullsized":"2024-08-17T17:43:09.444866+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":381,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.515095+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:43:26.515095+0000","last_peered":"2024-08-17T17:43:26.515095+0000","last_clean":"2024-08-17T17:43:26.515095+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:43:26.515095+0000","last_undegraded":"2024-08-17T17:43:26.515095+0000","last_fullsized":"2024-08-17T17:43:26.515095+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":446,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444661+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:43:09.444661+0000","last_peered":"2024-08-17T17:43:09.444661+0000","last_clean":"2024-08-17T17:43:09.444661+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:43:09.444661+0000","last_undegraded":"2024-08-17T17:43:09.444661+0000","last_fullsized":"2024-08-17T17:43:09.444661+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"394'11613","reported_seq":11804,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:32.502532+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:43:32.502532+0000","last_peered":"2024-08-17T17:43:32.502532+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:43:32.502532+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'5287","ondisk_log_start":"327'5287","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6326,"log_dups_size":0,"ondisk_log_size":6326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207814656,"num_objects":3171,"num_object_clones":0,"num_object_copies":9513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11613,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":3171},{"shards":"0,3","objects":-4077}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":656,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478460+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:43:26.478460+0000","last_peered":"2024-08-17T17:43:26.478460+0000","last_clean":"2024-08-17T17:43:26.478460+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:43:26.478460+0000","last_undegraded":"2024-08-17T17:43:26.478460+0000","last_fullsized":"2024-08-17T17:43:26.478460+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":624,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.479369+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:43:26.479369+0000","last_peered":"2024-08-17T17:43:26.479369+0000","last_clean":"2024-08-17T17:43:26.479369+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:43:26.479369+0000","last_undegraded":"2024-08-17T17:43:26.479369+0000","last_fullsized":"2024-08-17T17:43:26.479369+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":642,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.476729+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:43:26.476729+0000","last_peered":"2024-08-17T17:43:26.476729+0000","last_clean":"2024-08-17T17:43:26.476729+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:43:26.476729+0000","last_undegraded":"2024-08-17T17:43:26.476729+0000","last_fullsized":"2024-08-17T17:43:26.476729+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"394'11515","reported_seq":26206,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:32.111324+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:43:32.111324+0000","last_peered":"2024-08-17T17:43:32.111324+0000","last_clean":"2024-08-17T17:43:32.111324+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:43:32.111324+0000","last_undegraded":"2024-08-17T17:43:32.111324+0000","last_fullsized":"2024-08-17T17:43:32.111324+0000","mapping_epoch":347,"log_start":"327'4972","ondisk_log_start":"327'4972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6543,"log_dups_size":0,"ondisk_log_size":6543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200474624,"num_objects":3059,"num_object_clones":0,"num_object_copies":9177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11515,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":314,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.515371+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:43:26.515371+0000","last_peered":"2024-08-17T17:43:26.515371+0000","last_clean":"2024-08-17T17:43:26.515371+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:43:26.515371+0000","last_undegraded":"2024-08-17T17:43:26.515371+0000","last_fullsized":"2024-08-17T17:43:26.515371+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":398,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.451288+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:43:09.451288+0000","last_peered":"2024-08-17T17:43:09.451288+0000","last_clean":"2024-08-17T17:43:09.451288+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:43:09.451288+0000","last_undegraded":"2024-08-17T17:43:09.451288+0000","last_fullsized":"2024-08-17T17:43:09.451288+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":348,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468055+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:43:09.468055+0000","last_peered":"2024-08-17T17:43:09.468055+0000","last_clean":"2024-08-17T17:43:09.468055+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:43:09.468055+0000","last_undegraded":"2024-08-17T17:43:09.468055+0000","last_fullsized":"2024-08-17T17:43:09.468055+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468988+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:43:09.468988+0000","last_peered":"2024-08-17T17:43:09.468988+0000","last_clean":"2024-08-17T17:43:09.468988+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:43:09.468988+0000","last_undegraded":"2024-08-17T17:43:09.468988+0000","last_fullsized":"2024-08-17T17:43:09.468988+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":391,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468598+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:43:09.468598+0000","last_peered":"2024-08-17T17:43:09.468598+0000","last_clean":"2024-08-17T17:43:09.468598+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:43:09.468598+0000","last_undegraded":"2024-08-17T17:43:09.468598+0000","last_fullsized":"2024-08-17T17:43:09.468598+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":631,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477444+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:43:26.477444+0000","last_peered":"2024-08-17T17:43:26.477444+0000","last_clean":"2024-08-17T17:43:26.477444+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:43:26.477444+0000","last_undegraded":"2024-08-17T17:43:26.477444+0000","last_fullsized":"2024-08-17T17:43:26.477444+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":171,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477245+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:43:26.477245+0000","last_peered":"2024-08-17T17:43:26.477245+0000","last_clean":"2024-08-17T17:43:26.477245+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:43:26.477245+0000","last_undegraded":"2024-08-17T17:43:26.477245+0000","last_fullsized":"2024-08-17T17:43:26.477245+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":298,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.444940+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:43:09.444940+0000","last_peered":"2024-08-17T17:43:09.444940+0000","last_clean":"2024-08-17T17:43:09.444940+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:43:09.444940+0000","last_undegraded":"2024-08-17T17:43:09.444940+0000","last_fullsized":"2024-08-17T17:43:09.444940+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":589,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.476961+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:43:26.476961+0000","last_peered":"2024-08-17T17:43:26.476961+0000","last_clean":"2024-08-17T17:43:26.476961+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:43:26.476961+0000","last_undegraded":"2024-08-17T17:43:26.476961+0000","last_fullsized":"2024-08-17T17:43:26.476961+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":442,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.445167+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:43:09.445167+0000","last_peered":"2024-08-17T17:43:09.445167+0000","last_clean":"2024-08-17T17:43:09.445167+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:43:09.445167+0000","last_undegraded":"2024-08-17T17:43:09.445167+0000","last_fullsized":"2024-08-17T17:43:09.445167+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":583,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.476624+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:43:26.476624+0000","last_peered":"2024-08-17T17:43:26.476624+0000","last_clean":"2024-08-17T17:43:26.476624+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:43:26.476624+0000","last_undegraded":"2024-08-17T17:43:26.476624+0000","last_fullsized":"2024-08-17T17:43:26.476624+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":365,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.451214+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:43:09.451214+0000","last_peered":"2024-08-17T17:43:09.451214+0000","last_clean":"2024-08-17T17:43:09.451214+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:43:09.451214+0000","last_undegraded":"2024-08-17T17:43:09.451214+0000","last_fullsized":"2024-08-17T17:43:09.451214+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":614,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.476812+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:43:26.476812+0000","last_peered":"2024-08-17T17:43:26.476812+0000","last_clean":"2024-08-17T17:43:26.476812+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:43:26.476812+0000","last_undegraded":"2024-08-17T17:43:26.476812+0000","last_fullsized":"2024-08-17T17:43:26.476812+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":330,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478035+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:43:26.478035+0000","last_peered":"2024-08-17T17:43:26.478035+0000","last_clean":"2024-08-17T17:43:26.478035+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:43:26.478035+0000","last_undegraded":"2024-08-17T17:43:26.478035+0000","last_fullsized":"2024-08-17T17:43:26.478035+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":622,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478356+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:43:26.478356+0000","last_peered":"2024-08-17T17:43:26.478356+0000","last_clean":"2024-08-17T17:43:26.478356+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:43:26.478356+0000","last_undegraded":"2024-08-17T17:43:26.478356+0000","last_fullsized":"2024-08-17T17:43:26.478356+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":582,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477057+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:43:26.477057+0000","last_peered":"2024-08-17T17:43:26.477057+0000","last_clean":"2024-08-17T17:43:26.477057+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:43:26.477057+0000","last_undegraded":"2024-08-17T17:43:26.477057+0000","last_fullsized":"2024-08-17T17:43:26.477057+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":377,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468235+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:43:09.468235+0000","last_peered":"2024-08-17T17:43:09.468235+0000","last_clean":"2024-08-17T17:43:09.468235+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:43:09.468235+0000","last_undegraded":"2024-08-17T17:43:09.468235+0000","last_fullsized":"2024-08-17T17:43:09.468235+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":271,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477642+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:43:26.477642+0000","last_peered":"2024-08-17T17:43:26.477642+0000","last_clean":"2024-08-17T17:43:26.477642+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:43:26.477642+0000","last_undegraded":"2024-08-17T17:43:26.477642+0000","last_fullsized":"2024-08-17T17:43:26.477642+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":327,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.515004+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:43:26.515004+0000","last_peered":"2024-08-17T17:43:26.515004+0000","last_clean":"2024-08-17T17:43:26.515004+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:43:26.515004+0000","last_undegraded":"2024-08-17T17:43:26.515004+0000","last_fullsized":"2024-08-17T17:43:26.515004+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":385,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468964+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:43:09.468964+0000","last_peered":"2024-08-17T17:43:09.468964+0000","last_clean":"2024-08-17T17:43:09.468964+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:43:09.468964+0000","last_undegraded":"2024-08-17T17:43:09.468964+0000","last_fullsized":"2024-08-17T17:43:09.468964+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":248,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468512+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:43:09.468512+0000","last_peered":"2024-08-17T17:43:09.468512+0000","last_clean":"2024-08-17T17:43:09.468512+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:43:09.468512+0000","last_undegraded":"2024-08-17T17:43:09.468512+0000","last_fullsized":"2024-08-17T17:43:09.468512+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":385,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468136+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:43:09.468136+0000","last_peered":"2024-08-17T17:43:09.468136+0000","last_clean":"2024-08-17T17:43:09.468136+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:43:09.468136+0000","last_undegraded":"2024-08-17T17:43:09.468136+0000","last_fullsized":"2024-08-17T17:43:09.468136+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":248,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.468350+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:43:09.468350+0000","last_peered":"2024-08-17T17:43:09.468350+0000","last_clean":"2024-08-17T17:43:09.468350+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:43:09.468350+0000","last_undegraded":"2024-08-17T17:43:09.468350+0000","last_fullsized":"2024-08-17T17:43:09.468350+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":634,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.477325+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:43:26.477325+0000","last_peered":"2024-08-17T17:43:26.477325+0000","last_clean":"2024-08-17T17:43:26.477325+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:43:26.477325+0000","last_undegraded":"2024-08-17T17:43:26.477325+0000","last_fullsized":"2024-08-17T17:43:26.477325+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":608,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.478189+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:43:26.478189+0000","last_peered":"2024-08-17T17:43:26.478189+0000","last_clean":"2024-08-17T17:43:26.478189+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:43:26.478189+0000","last_undegraded":"2024-08-17T17:43:26.478189+0000","last_fullsized":"2024-08-17T17:43:26.478189+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":348,"reported_epoch":393,"state":"active+clean","last_fresh":"2024-08-17T17:43:09.467762+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:43:09.467762+0000","last_peered":"2024-08-17T17:43:09.467762+0000","last_clean":"2024-08-17T17:43:09.467762+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:43:09.467762+0000","last_undegraded":"2024-08-17T17:43:09.467762+0000","last_fullsized":"2024-08-17T17:43:09.467762+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":330,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.515254+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:43:26.515254+0000","last_peered":"2024-08-17T17:43:26.515254+0000","last_clean":"2024-08-17T17:43:26.515254+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:43:26.515254+0000","last_undegraded":"2024-08-17T17:43:26.515254+0000","last_fullsized":"2024-08-17T17:43:26.515254+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3263430680,"num_objects":49797,"num_object_clones":0,"num_object_copies":149391,"num_objects_missing_on_primary":3731,"num_objects_missing":3731,"num_objects_degraded":50574,"num_objects_misplaced":20219,"num_objects_unfound":0,"num_objects_dirty":49797,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186593,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33002,"num_bytes_recovered":2040266752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12162052096,"data_stored":12162039880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104612,"ondisk_log_size":104612,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170126,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463495,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299448,"kb_used_data":1409904,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":91445704,"statfs":{"total":95995035648,"available":93640400896,"internally_reserved":0,"allocated":1443741696,"data_stored":1441571015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492170,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":788444,"kb_used_data":397268,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92956708,"statfs":{"total":95995035648,"available":95187668992,"internally_reserved":0,"allocated":406802432,"data_stored":404654051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986761,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2289708,"kb_used_data":793508,"kb_used_omap":1,"kb_used_meta":1496190,"kb_avail":91455444,"statfs":{"total":95995035648,"available":93650374656,"internally_reserved":0,"allocated":812552192,"data_stored":810392451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1532098904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225589,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2550684,"kb_used_data":1603028,"kb_used_omap":4,"kb_used_meta":947643,"kb_avail":91194468,"statfs":{"total":95995035648,"available":93383135232,"internally_reserved":0,"allocated":1641500672,"data_stored":1639352589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":970386880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075914,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533807,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2449824,"kb_used_data":1398104,"kb_used_omap":5,"kb_used_meta":1051706,"kb_avail":91295328,"statfs":{"total":95995035648,"available":93486415872,"internally_reserved":0,"allocated":1431658496,"data_stored":1429505091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1076947184},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089784,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2220344,"kb_used_data":1201392,"kb_used_omap":5,"kb_used_meta":1018938,"kb_avail":91524808,"statfs":{"total":95995035648,"available":93721403392,"internally_reserved":0,"allocated":1230225408,"data_stored":1228054786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1043392571},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1221984256,"data_stored":1221984256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1422528512,"data_stored":1422524440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1636302848,"data_stored":1636302848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":803733504,"data_stored":803733504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":401604608,"data_stored":401604608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1433731072,"data_stored":1433731072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1902907392,"data_stored":1902903320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:40.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:40.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:40.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:41.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:41.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:41.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:41.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:41.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:41.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:41.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:41.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:41.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:41.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:41.528 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:41.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:41.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:41.753 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-mon[106729]: pgmap v1528: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50456/146889 objects degraded (34.350%); 19906/146889 objects misplaced (13.552%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:43:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50456/146889 objects degraded (34.350%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:41.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-mon[103614]: pgmap v1528: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50456/146889 objects degraded (34.350%); 19906/146889 objects misplaced (13.552%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:43:41.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50456/146889 objects degraded (34.350%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:41.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:41.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:41.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:41 smithi152 ceph-mon[93456]: pgmap v1528: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50456/146889 objects degraded (34.350%); 19906/146889 objects misplaced (13.552%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:43:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50456/146889 objects degraded (34.350%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:41.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:41.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:41.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:41.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:42.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:42.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:43:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:43:41] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:43:42.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:42.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:42.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:42.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:42.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:42.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:42.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:42.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:42.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:42.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:42.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:42.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:42.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:42.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:42.780 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:42.784 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:42 smithi142 ceph-mon[103614]: pgmap v1529: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50456/146067 objects degraded (34.543%); 19852/146067 objects misplaced (13.591%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:43:42.784 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:42 smithi142 ceph-mon[106729]: pgmap v1529: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50456/146067 objects degraded (34.543%); 19852/146067 objects misplaced (13.591%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:43:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:42 smithi152 ceph-mon[93456]: pgmap v1529: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50456/146067 objects degraded (34.543%); 19852/146067 objects misplaced (13.591%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:43:42.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:42.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:42.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:43.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:43.028 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:43.103 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131382 12.b 12:d72dad81:::benchmark_data_smithi142_261410_object13185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 523.59488144299996, 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.96633504400000003, 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.104 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131382 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:43:43.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482429+0000", 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482430+0000", 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5700000000000002e-07 2024-08-17T17:43:43.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482433+0000", 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.692e-06 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482445+0000", 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1564e-05 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482450+0000", 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5159999999999996e-06 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482507+0000", 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7800000000000002e-05 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.482519+0000", 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1567e-05 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.436998+0000", 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.95447953500000005 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.437123+0000", 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012464299999999999 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:43:43.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.437248+0000", 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012493899999999999 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448394+0000", 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011145649000000001 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448408+0000", 2024-08-17T17:43:43.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4669e-05 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448665+0000", 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000256531 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448725+0000", 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0028999999999997e-05 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.111 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448764+0000", 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9252999999999998e-05 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131388 12.b 12:d3e07cb9:::benchmark_data_smithi142_261410_object13191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 523.52605592400005, 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.89768371400000002, 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.112 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131388 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:43:43.113 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551255+0000", 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551256+0000", 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5230000000000001e-06 2024-08-17T17:43:43.114 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551262+0000", 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7860000000000002e-06 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551283+0000", 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.084e-05 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.115 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551292+0000", 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4839999999999993e-06 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551356+0000", 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4068999999999999e-05 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:43:43.116 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.551365+0000", 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8510000000000005e-06 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441263+0000", 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88989788199999997 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441385+0000", 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121975 2024-08-17T17:43:43.117 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.441494+0000", 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108819 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448506+0000", 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0070124380000000002 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.118 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448539+0000", 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3089000000000001e-05 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448827+0000", 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00028744399999999998 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.119 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448896+0000", 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8782000000000007e-05 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448938+0000", 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2731999999999997e-05 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.120 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:131408 12.b 12:df15f4a1:::benchmark_data_smithi142_261410_object13211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 523.08385428300005, 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.45568646400000001, 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131408 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.121 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993456+0000", 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993458+0000", 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0509999999999999e-06 2024-08-17T17:43:43.122 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993466+0000", 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5270000000000001e-06 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993479+0000", 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3621000000000001e-05 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993488+0000", 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2369999999999992e-06 2024-08-17T17:43:43.123 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993547+0000", 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9639000000000003e-05 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:34:59.993556+0000", 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9940000000000006e-06 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.124 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444651+0000", 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.45109483099999997 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444777+0000", 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012631799999999999 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.444902+0000", 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000124721 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.125 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.448979+0000", 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040765480000000002 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449036+0000", 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7623e-05 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449058+0000", 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1386000000000002e-05 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.126 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449104+0000", 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6388999999999999e-05 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:35:00.449143+0000", 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8578999999999998e-05 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.127 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:137199 12.a 12:52e19bf6:::benchmark_data_smithi142_261410_object19002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 447.598601917, 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.056001591, 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.128 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 137199 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.129 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478710+0000", 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4600000000000003e-07 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478714+0000", 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7330000000000003e-06 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.130 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478724+0000", 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478731+0000", 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1169999999999998e-06 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478795+0000", 2024-08-17T17:43:43.131 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4641999999999995e-05 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478804+0000", 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531212+0000", 2024-08-17T17:43:43.132 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.052407855 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531320+0000", 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000107692 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531454+0000", 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134055 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.133 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534306+0000", 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028519069999999999 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534367+0000", 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1266000000000006e-05 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534598+0000", 2024-08-17T17:43:43.134 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000231373 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534651+0000", 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2369e-05 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534710+0000", 2024-08-17T17:43:43.135 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9517e-05 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146690 12.8 12:1be560a4:::benchmark_data_smithi142_261410_object28493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.626791+0000", 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 340.45051998000002, 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.026113173999999999, 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.136 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146690 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626791+0000", 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626791+0000", 2024-08-17T17:43:43.137 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626792+0000", 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6099999999999999e-07 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626795+0000", 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8340000000000001e-06 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.138 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626808+0000", 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2320999999999999e-05 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626815+0000", 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9229999999999996e-06 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626860+0000", 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5343999999999998e-05 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.139 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.626973+0000", 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011325299999999999 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.627073+0000", 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9480000000000003e-05 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.140 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633055+0000", 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059819809999999999 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633121+0000", 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6036000000000003e-05 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.141 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.650236+0000", 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017115253 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.650277+0000", 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0666e-05 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.652904+0000", 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002627122 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.142 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:146692 12.8 12:11913193:::benchmark_data_smithi142_261410_object28495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 340.448153031, 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025921260000000002, 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.143 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 146692 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.144 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629158+0000", 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1099999999999995e-07 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629162+0000", 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7390000000000001e-06 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.145 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629171+0000", 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9160000000000007e-06 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629177+0000", 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5339999999999996e-06 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629761+0000", 2024-08-17T17:43:43.146 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058357700000000005 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.629901+0000", 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014035100000000001 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.630040+0000", 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000139147 2024-08-17T17:43:43.147 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633107+0000", 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00306668 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.633129+0000", 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1738000000000001e-05 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.148 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.652958+0000", 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019828983000000001 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.652998+0000", 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0197999999999997e-05 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:38:02.655079+0000", 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002080686 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.149 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPush(12.d 376/337 [PushOp(12:b17145dc:::benchmark_data_smithi142_261410_object40071:head, version: 324'2525, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:b17145dc:::benchmark_data_smithi142_261410_object40071:head@324'2525, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.829676+0000", 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 228.24763424400001, 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.033488366999999998, 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.150 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829676+0000", 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829676+0000", 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.151 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829677+0000", 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7400000000000002e-07 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829705+0000", 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7934999999999999e-05 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829721+0000", 2024-08-17T17:43:43.152 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5410999999999999e-05 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829732+0000", 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1357e-05 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829804+0000", 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2395000000000001e-05 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.153 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.829813+0000", 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4640000000000006e-06 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.863165+0000", 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.033351931000000001 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.154 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:158269 12.e e376/337 12:7a28666e:::benchmark_data_smithi142_261410_object40072:head v 376'9843, mlcod=324'725)", 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 228.246315751, 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.037591274000000001, 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:43:43.155 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830996+0000", 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9500000000000001e-07 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.156 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.831003+0000", 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9709999999999998e-06 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861303+0000", 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030299947000000001 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861329+0000", 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6420000000000001e-05 2024-08-17T17:43:43.157 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861402+0000", 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2539999999999993e-05 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861430+0000", 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8707000000000001e-05 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.158 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.868547+0000", 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0071166140000000003 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.868586+0000", 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9180000000000001e-05 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.159 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:161158 12.8 12:1a81df31:::benchmark_data_smithi142_261410_object42961:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:19.738953+0000", 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 203.33835750700001, 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027048839000000002, 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 161158 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.160 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738953+0000", 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738953+0000", 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738954+0000", 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1289999999999999e-06 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.161 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738958+0000", 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0600000000000001e-06 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738970+0000", 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1385e-05 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.738976+0000", 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5649999999999998e-06 2024-08-17T17:43:43.162 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.744444+0000", 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0054678749999999996 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.744628+0000", 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018376199999999999 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.744801+0000", 2024-08-17T17:43:43.163 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017283500000000001 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.748151+0000", 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003350319 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.748172+0000", 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1104999999999999e-05 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.164 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.761762+0000", 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013589752 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.761832+0000", 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0297999999999997e-05 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:19.766002+0000", 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041697540000000003 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.165 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:175766 12.5 e389/332 12:a1f3cbc8:::benchmark_data_smithi142_261410_object57569:head v 389'11041, mlcod=322'100)", 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:22.553147+0000", 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 80.524163544000004, 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025919838000000001, 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.166 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553147+0000", 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553147+0000", 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553148+0000", 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1739999999999999e-06 2024-08-17T17:43:43.167 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553156+0000", 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8869999999999992e-06 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553173+0000", 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7309e-05 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.168 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553182+0000", 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6489999999999997e-06 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553261+0000", 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8843000000000004e-05 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.553286+0000", 2024-08-17T17:43:43.169 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5344e-05 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.579011+0000", 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025724436999999999 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:22.579067+0000", 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6195e-05 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.170 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176488 12.8 12:1bdea9b6:::benchmark_data_smithi142_261410_object58291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.125797+0000", 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 74.951513058000003, 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.20733109, 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.171 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176488 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125797+0000", 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125797+0000", 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.172 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125799+0000", 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.111e-06 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125804+0000", 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9200000000000003e-06 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125814+0000", 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0101e-05 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.173 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125820+0000", 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3679999999999998e-06 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125918+0000", 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7891000000000002e-05 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.126103+0000", 2024-08-17T17:43:43.174 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000185012 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.126274+0000", 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000171158 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.129440+0000", 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003165541 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.175 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.129466+0000", 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6562000000000001e-05 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137780+0000", 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0083137720000000005 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.176 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137812+0000", 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2169000000000003e-05 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333129+0000", 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19531648500000001 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:176488 12.8 e389/332 ondisk, result = 0)", 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 74.939591945999993, 2024-08-17T17:43:43.177 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.195464622, 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.178 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7100000000000001e-07 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137724+0000", 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9019999999999998e-06 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137732+0000", 2024-08-17T17:43:43.179 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7889999999999995e-06 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137737+0000", 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7679999999999999e-06 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137771+0000", 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4144999999999999e-05 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137776+0000", 2024-08-17T17:43:43.180 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3750000000000002e-06 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333183+0000", 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19540697200000001 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:176491 12.5 e389/332 12:a00ed9f7:::benchmark_data_smithi142_261410_object58294:head v 389'11091, mlcod=322'100)", 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.142636+0000", 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 74.934674576000006, 2024-08-17T17:43:43.181 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19829270600000001, 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142636+0000", 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.182 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142636+0000", 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142637+0000", 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0359999999999999e-06 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142644+0000", 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7850000000000003e-06 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.183 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333074+0000", 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.190430558 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333085+0000", 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0677e-05 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333168+0000", 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2996000000000003e-05 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.184 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333193+0000", 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4790000000000002e-05 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.340871+0000", 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0076780340000000002 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.340929+0000", 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.783e-05 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.185 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:176493 12.3 e389/337 12:c78d0f5f:::benchmark_data_smithi142_261410_object58296:head v 389'11204, mlcod=324'710)", 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.172427+0000", 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 74.904883975999994, 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.16378883399999999, 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.186 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172427+0000", 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172427+0000", 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.187 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172428+0000", 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1090000000000001e-06 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172437+0000", 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6719999999999999e-06 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333053+0000", 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.160615863 2024-08-17T17:43:43.188 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333072+0000", 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8984999999999999e-05 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333136+0000", 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3948000000000005e-05 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333166+0000", 2024-08-17T17:43:43.189 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9390000000000002e-05 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.336181+0000", 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003015551 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.336215+0000", 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4316000000000001e-05 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.190 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176660 12.8 12:1585a442:::benchmark_data_smithi142_261410_object58463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:29.396554+0000", 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 73.680756467999998, 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027716471999999999, 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176660 2024-08-17T17:43:43.191 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396554+0000", 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396554+0000", 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.192 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396556+0000", 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0880000000000002e-06 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396563+0000", 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7390000000000002e-06 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396577+0000", 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3940999999999999e-05 2024-08-17T17:43:43.193 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396585+0000", 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6179999999999995e-06 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396640+0000", 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4537999999999999e-05 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396749+0000", 2024-08-17T17:43:43.194 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109213 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.396930+0000", 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000180378 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.403119+0000", 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0061895370000000002 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.195 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.403153+0000", 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4357000000000003e-05 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.421654+0000", 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018500149 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.421697+0000", 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3238999999999997e-05 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.196 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.424271+0000", 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025736750000000001 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176662 12.8 12:192ed456:::benchmark_data_smithi142_261410_object58465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:29.397965+0000", 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 73.679345144999999, 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.028194844, 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.197 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176662 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.397965+0000", 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.198 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.397965+0000", 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.397966+0000", 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2399999999999997e-07 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.397970+0000", 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.619e-06 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.199 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.397977+0000", 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4000000000000003e-06 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.397982+0000", 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6600000000000003e-06 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.200 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.400076+0000", 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020938039999999999 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.400205+0000", 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000129531 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.400320+0000", 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115102 2024-08-17T17:43:43.201 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.403222+0000", 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029020959999999998 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.403580+0000", 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035755500000000001 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.424322+0000", 2024-08-17T17:43:43.202 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.020741946000000001 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.424360+0000", 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8188000000000001e-05 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:29.426160+0000", 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001800219 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.203 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176872 12.b 12:d3dc57ed:::benchmark_data_smithi142_261410_object58675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e390)", 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:30.728836+0000", 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 72.348474135999993, 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024938608000000001, 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.204 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176872 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728836+0000", 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.205 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728836+0000", 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728838+0000", 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.297e-06 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728843+0000", 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2739999999999998e-06 2024-08-17T17:43:43.206 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728855+0000", 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.167e-05 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728861+0000", 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7220000000000002e-06 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.207 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.728924+0000", 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2459000000000006e-05 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.729060+0000", 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013643600000000001 2024-08-17T17:43:43.208 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:43.209 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.729194+0000", 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013375299999999999 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.738858+0000", 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0096635220000000008 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.742728+0000", 2024-08-17T17:43:43.210 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038701489999999998 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.753709+0000", 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010980962 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.753746+0000", 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7455e-05 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.211 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:30.753775+0000", 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8909000000000001e-05 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:185596 12.8 12:12db3b2a:::benchmark_data_smithi142_261410_object67399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:27.627278+0000", 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 15.450032454, 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.021774160000000001, 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.212 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 185596 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.627278+0000", 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.213 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.627278+0000", 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.627279+0000", 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4099999999999999e-07 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.214 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.627281+0000", 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3089999999999998e-06 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.627287+0000", 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2299999999999996e-06 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.627292+0000", 2024-08-17T17:43:43.215 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6700000000000002e-06 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.630308+0000", 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030164609999999998 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.630506+0000", 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000197443 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.216 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.630678+0000", 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017243599999999999 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.641108+0000", 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010430160000000001 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.641127+0000", 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8964e-05 2024-08-17T17:43:43.217 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.645760+0000", 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046324340000000004 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.645800+0000", 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9894000000000001e-05 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.218 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.649052+0000", 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032527179999999999 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:185599 12.8 12:115715b6:::benchmark_data_smithi142_261410_object67402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:27.631256+0000", 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 15.44605419, 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025422542999999999, 2024-08-17T17:43:43.219 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 185599 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.631256+0000", 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.220 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.631256+0000", 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.631257+0000", 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3399999999999999e-07 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.631260+0000", 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8329999999999998e-06 2024-08-17T17:43:43.221 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.631267+0000", 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.951e-06 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.631271+0000", 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5040000000000004e-06 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.222 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.637341+0000", 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060699109999999999 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.637504+0000", 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016253400000000001 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.637659+0000", 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015584 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.223 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.641241+0000", 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035816609999999999 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.641267+0000", 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6095e-05 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.653188+0000", 2024-08-17T17:43:43.224 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011921205000000001 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.653228+0000", 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9270999999999997e-05 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:27.656679+0000", 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034512039999999998 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.225 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:187696 12.8 12:1be2f01b:::benchmark_data_smithi142_261410_object69499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:40.145180+0000", 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 2.932130452, 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019101461, 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:43:43.226 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 187696 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.145180+0000", 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.145180+0000", 2024-08-17T17:43:43.227 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.145181+0000", 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2e-07 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.145183+0000", 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7590000000000002e-06 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.228 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.145191+0000", 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8609999999999998e-06 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.145197+0000", 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6550000000000001e-06 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:43:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.147791+0000", 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025939299999999999 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.147956+0000", 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000164943 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.148113+0000", 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000157329 2024-08-17T17:43:43.230 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.150499+0000", 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023854430000000001 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.151302+0000", 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00080382999999999995 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:43:43.231 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.162008+0000", 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010705492 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.162051+0000", 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2763999999999999e-05 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.164282+0000", 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0022309349999999999 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.232 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.233 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.233 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:43:43.233 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:43:43.233 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:43:43.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:43.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:43.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:43.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:43.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:43.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:43.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:43.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:43.660 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:43:43.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:43.765 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:43.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:43.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:43.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:43.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:44.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:44.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:44.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:44.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.164 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:44.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:44.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:44.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:43:44.288 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:44.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:44.317 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:44.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:44.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:44.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:44.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:44.571 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:43:44.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:44.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:44.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:44.789 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:44.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:43:44.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:44.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:44.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:44.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:45.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:45.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:45.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:45.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:45.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:45.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:45.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:45.427 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:45.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:45.550 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:45.608 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:45 smithi142 ceph-mon[103614]: pgmap v1530: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 50404/145239 objects degraded (34.704%); 19750/145239 objects misplaced (13.598%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:45.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:45 smithi142 ceph-mon[106729]: pgmap v1530: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 50404/145239 objects degraded (34.704%); 19750/145239 objects misplaced (13.598%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:45.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:45.653 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:45.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:45.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:45.806 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:45 smithi152 ceph-mon[93456]: pgmap v1530: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 50404/145239 objects degraded (34.704%); 19750/145239 objects misplaced (13.598%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:45.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:45.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:45.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:46.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:46.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:46.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:46.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:46.244 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:46.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:46.312 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:43:46.418 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:43:46.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:46.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:46.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:46.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:46.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:46.592 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188688 12.7 12:e69ff6de:::benchmark_data_smithi142_261410_object70491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.413172+0000", 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.1779378629999999, 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.177960874, 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.593 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188688 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.413172+0000", 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.413172+0000", 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.413170+0000", 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.413176+0000", 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3060000000000003e-06 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.413187+0000", 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.06e-05 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.413192+0000", 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2159999999999997e-06 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.453239+0000", 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040046340999999999 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.459849+0000", 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066109130000000004 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.549212+0000", 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.089362168000000006 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.554451+0000", 2024-08-17T17:43:46.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00523954 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188690 12.7 12:ecb85e3a:::benchmark_data_smithi142_261410_object70493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.415898+0000", 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.175211668, 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.175286144, 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.598 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188690 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.415898+0000", 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.415898+0000", 2024-08-17T17:43:46.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.415896+0000", 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.415904+0000", 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4939999999999997e-06 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.415919+0000", 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5046e-05 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.415925+0000", 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3879999999999997e-06 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.460596+0000", 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.044671324999999998 2024-08-17T17:43:46.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.466041+0000", 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054449260000000001 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.521281+0000", 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.055239943 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.526389+0000", 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051072219999999998 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.567160+0000", 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040771729 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573850+0000", 2024-08-17T17:43:46.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066899070000000001 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.582918+0000", 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090673750000000008 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188762 12.7 12:e1de4cde:::benchmark_data_smithi142_261410_object70565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.604 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.787407+0000", 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.80370234299999999, 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80382833099999995, 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188762 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.787407+0000", 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.787407+0000", 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.787406+0000", 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.787413+0000", 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8189999999999992e-06 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.787427+0000", 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3759e-05 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.787434+0000", 2024-08-17T17:43:46.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6239999999999996e-06 2024-08-17T17:43:46.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.889894+0000", 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10246013299999999 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.894710+0000", 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048160019999999998 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.969844+0000", 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.075134309999999996 2024-08-17T17:43:46.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.977009+0000", 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071642809999999998 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188709 12.7 12:eb8b9a21:::benchmark_data_smithi142_261410_object70512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.508059+0000", 2024-08-17T17:43:46.610 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.0830501340000001, 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.083218413, 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188709 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.508059+0000", 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.508059+0000", 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.508058+0000", 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.508064+0000", 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5149999999999997e-06 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.508074+0000", 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8849999999999994e-06 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.508078+0000", 2024-08-17T17:43:46.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6530000000000002e-06 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.574711+0000", 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.066632916 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.579920+0000", 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052084949999999996 2024-08-17T17:43:46.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.632839+0000", 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.052919431000000003 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.637923+0000", 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050836960000000004 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:188891 12.f e395/332 12:fe384801:::benchmark_data_smithi142_261410_object70694:head v 395'11869, mlcod=322'102)", 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.540838+0000", 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.050271438000000002, 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.050479877999999999, 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.540838+0000", 2024-08-17T17:43:46.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.540838+0000", 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.540837+0000", 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.540846+0000", 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8799999999999997e-06 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.540866+0000", 2024-08-17T17:43:46.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0530000000000002e-05 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.540873+0000", 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6239999999999996e-06 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.542483+0000", 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016095860000000001 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.542507+0000", 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3912000000000001e-05 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188789 12.7 12:edb39f2f:::benchmark_data_smithi142_261410_object70592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.944211+0000", 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.64689813799999996, 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.64713842200000005, 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.620 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188789 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.944211+0000", 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.944211+0000", 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.944210+0000", 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.944216+0000", 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5880000000000003e-06 2024-08-17T17:43:46.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.944228+0000", 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2329e-05 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.944234+0000", 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1529999999999996e-06 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.006400+0000", 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.062165977999999997 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.011615+0000", 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052150959999999998 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.071209+0000", 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.059594723000000002 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.076577+0000", 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053677439999999998 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(12.7 395/337 [PushOp(12:e366d62d:::benchmark_data_smithi142_261410_object70667:head, version: 326'4449, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:e366d62d:::benchmark_data_smithi142_261410_object70667:head@326'4449, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.553342+0000", 2024-08-17T17:43:46.625 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.037767098999999998, 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.038060286999999998, 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.553342+0000", 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.553342+0000", 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.553342+0000", 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.553388+0000", 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6622000000000002e-05 2024-08-17T17:43:46.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.553407+0000", 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9069999999999999e-05 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.553418+0000", 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0852e-05 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188858 12.7 12:eaf57f88:::benchmark_data_smithi142_261410_object70661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.422028+0000", 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.16908142400000001, 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16940066200000001, 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188858 2024-08-17T17:43:46.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.422028+0000", 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.422028+0000", 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.422027+0000", 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.422033+0000", 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0610000000000001e-06 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.422046+0000", 2024-08-17T17:43:46.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3526999999999999e-05 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.422052+0000", 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6919999999999999e-06 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.461890+0000", 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.039837984999999999 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.468274+0000", 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063839090000000001 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.526293+0000", 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.058018818999999999 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.539885+0000", 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013592266 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.633 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188803 12.7 12:e60e55c7:::benchmark_data_smithi142_261410_object70606:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.019621+0000", 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.571488153, 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.57184843299999999, 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.634 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188803 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.019621+0000", 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.019621+0000", 2024-08-17T17:43:46.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.019620+0000", 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.019626+0000", 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.677e-06 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.019638+0000", 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1894e-05 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.019644+0000", 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.567e-06 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.124293+0000", 2024-08-17T17:43:46.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10464876100000001 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.129274+0000", 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049805680000000003 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.185225+0000", 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.055951513000000001 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.190331+0000", 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051056000000000001 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188864 12.7 12:e366d62d:::benchmark_data_smithi142_261410_object70667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.446524+0000", 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.144585344, 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14498894700000001, 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.639 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for missing object", 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188864 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.446524+0000", 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.446524+0000", 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.446523+0000", 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.446530+0000", 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:43:46.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.446543+0000", 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2855e-05 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.446549+0000", 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8020000000000003e-06 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.543257+0000", 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.096707281000000006 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.551612+0000", 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083549569999999997 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.643 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188901 12.7 12.e5d05ef7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.570593+0000", 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.020516342, 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020953817, 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188901 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.570593+0000", 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.570593+0000", 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.570591+0000", 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.570599+0000", 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9410000000000007e-06 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.570612+0000", 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3156000000000001e-05 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.570618+0000", 2024-08-17T17:43:46.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0990000000000004e-06 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188825 12.7 12:ed1921fc:::benchmark_data_smithi142_261410_object70628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.647 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.137997+0000", 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.45311227999999998, 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.45357888299999999, 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188825 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.137997+0000", 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.137997+0000", 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.137996+0000", 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.138002+0000", 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4530000000000001e-06 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.138011+0000", 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7450000000000007e-06 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.138017+0000", 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.02e-06 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.208584+0000", 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.070566702999999995 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.214430+0000", 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058456050000000002 2024-08-17T17:43:46.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.270338+0000", 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.055908188999999997 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.275443+0000", 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051055370000000003 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.652 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:188902 12.f e395/332)", 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.573787+0000", 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.017322892999999999, 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017827056000000001, 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573787+0000", 2024-08-17T17:43:46.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.654 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50404/145239 objects degraded (34.704%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:46.655 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50404/145239 objects degraded (34.704%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573787+0000", 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573786+0000", 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573794+0000", 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6519999999999995e-06 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573808+0000", 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3383999999999999e-05 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.573814+0000", 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9120000000000003e-06 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188747 12.7 12:e15658c3:::benchmark_data_smithi142_261410_object70550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.743928+0000", 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.84718095100000002, 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84772076200000002, 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.657 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188747 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.743928+0000", 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.743928+0000", 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.743927+0000", 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.743933+0000", 2024-08-17T17:43:46.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5310000000000001e-06 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.743944+0000", 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1240000000000001e-05 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.743949+0000", 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.774541+0000", 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030592459999999998 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.779341+0000", 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047996410000000003 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.867170+0000", 2024-08-17T17:43:46.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.087829091999999997 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.872329+0000", 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051592230000000001 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46715.0:188690 12.7 e395/337)", 2024-08-17T17:43:46.662 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.586112+0000", 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0049975000000000002, 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055887230000000003, 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586112+0000", 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586112+0000", 2024-08-17T17:43:46.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586110+0000", 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586123+0000", 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2157999999999999e-05 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586141+0000", 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8125999999999999e-05 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586149+0000", 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6130000000000004e-06 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.665 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188750 12.7 12:e6efd0a0:::benchmark_data_smithi142_261410_object70553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.751656+0000", 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.83945324300000002, 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84007690800000001, 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188750 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.751656+0000", 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.751656+0000", 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.751655+0000", 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.751664+0000", 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4839999999999993e-06 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.751677+0000", 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2679e-05 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.751682+0000", 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5079999999999996e-06 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.779923+0000", 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028240208999999999 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.785667+0000", 2024-08-17T17:43:46.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005743972 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.840332+0000", 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.054665049 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.845475+0000", 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051429889999999997 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.670 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188783 12.7 12:e4e1909f:::benchmark_data_smithi142_261410_object70586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:45.905773+0000", 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.68533641599999995, 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.68600072300000003, 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:43:46.671 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188783 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.905773+0000", 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.905773+0000", 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.905771+0000", 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.905777+0000", 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7590000000000003e-06 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.905787+0000", 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0210000000000001e-05 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:45.905793+0000", 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1830000000000002e-06 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.000735+0000", 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.094942443000000001 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.005520+0000", 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004784506 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.101279+0000", 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.095759642000000006 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:43:46.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.106544+0000", 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052651420000000004 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46715.0:188690 12.7 e395/337)", 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.586837+0000", 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0042720329999999997, 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049734669999999996, 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:43:46.676 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586837+0000", 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586837+0000", 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586837+0000", 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586842+0000", 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3979999999999998e-06 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586861+0000", 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9032e-05 2024-08-17T17:43:46.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586868+0000", 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1180000000000003e-06 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46715.0:188690 12.7 e395/337)", 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:46.586890+0000", 2024-08-17T17:43:46.679 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00421982, 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049441750000000003, 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586890+0000", 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:43:46.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586890+0000", 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586889+0000", 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586893+0000", 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5260000000000002e-06 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586900+0000", 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.165e-06 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:46.586904+0000", 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8909999999999997e-06 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:43:46.682 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:43:46.683 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:43:46.683 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 19 2024-08-17T17:43:46.683 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:43:46.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:46.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:46.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:46.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50404/145239 objects degraded (34.704%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:46.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:46.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:46.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:46.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:47.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:47.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:47.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:47.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:47.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:47.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:47.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:47.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:47.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:47.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:47.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:47.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:47 smithi142 ceph-mon[106729]: pgmap v1531: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 50349/144267 objects degraded (34.900%); 19573/144267 objects misplaced (13.567%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:43:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:47 smithi142 ceph-mon[106729]: from='client.37842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:47 smithi142 ceph-mon[103614]: pgmap v1531: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 50349/144267 objects degraded (34.900%); 19573/144267 objects misplaced (13.567%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:43:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:47 smithi142 ceph-mon[103614]: from='client.37842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:47.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:47.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:43:47.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:47.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:47.730 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:47 smithi152 ceph-mon[93456]: pgmap v1531: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 173 op/s; 50349/144267 objects degraded (34.900%); 19573/144267 objects misplaced (13.567%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:43:47.730 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:47 smithi152 ceph-mon[93456]: from='client.37842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:47.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:47.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:47.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:47.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:47.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:47.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:48.053 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:43:48.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:48.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:48.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:48.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:48.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:48.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:48.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:48.454 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1531,"stamp":"2024-08-17T17:43:46.381330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3150387173,"num_objects":48089,"num_object_clones":0,"num_object_copies":144267,"num_objects_missing_on_primary":3625,"num_objects_missing":3625,"num_objects_degraded":50349,"num_objects_misplaced":19573,"num_objects_unfound":0,"num_objects_dirty":48089,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":188605,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33388,"num_bytes_recovered":2065244814,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104853,"ondisk_log_size":104853,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12380948,"kb_used_data":6560740,"kb_used_omap":27,"kb_used_meta":5820132,"kb_avail":550089964,"statfs":{"total":575970213888,"available":563292123136,"internally_reserved":0,"allocated":6718197760,"data_stored":6705200089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5959815912},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,0,1,0,2,2,2,6,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-113639424,"num_objects":-1734,"num_object_clones":0,"num_object_copies":-5202,"num_objects_missing_on_primary":-106,"num_objects_missing":-106,"num_objects_degraded":-228,"num_objects_misplaced":-646,"num_objects_unfound":0,"num_objects_dirty":-1734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1734,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":345,"num_bytes_recovered":22609920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002756"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":634,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510312+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:43:38.510312+0000","last_peered":"2024-08-17T17:43:38.510312+0000","last_clean":"2024-08-17T17:43:38.510312+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:43:38.510312+0000","last_undegraded":"2024-08-17T17:43:38.510312+0000","last_fullsized":"2024-08-17T17:43:38.510312+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":104,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512647+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:43:26.512647+0000","last_peered":"2024-08-17T17:43:26.512647+0000","last_clean":"2024-08-17T17:43:26.512647+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:43:26.512647+0000","last_undegraded":"2024-08-17T17:43:26.512647+0000","last_fullsized":"2024-08-17T17:43:26.512647+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":298,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.617412+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:43:26.617412+0000","last_peered":"2024-08-17T17:43:26.617412+0000","last_clean":"2024-08-17T17:43:26.617412+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:43:26.617412+0000","last_undegraded":"2024-08-17T17:43:26.617412+0000","last_fullsized":"2024-08-17T17:43:26.617412+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":332,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.511183+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:43:38.511183+0000","last_peered":"2024-08-17T17:43:38.511183+0000","last_clean":"2024-08-17T17:43:38.511183+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:43:38.511183+0000","last_undegraded":"2024-08-17T17:43:38.511183+0000","last_fullsized":"2024-08-17T17:43:38.511183+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":509,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.513361+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:43:38.513361+0000","last_peered":"2024-08-17T17:43:38.513361+0000","last_clean":"2024-08-17T17:43:38.513361+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:43:38.513361+0000","last_undegraded":"2024-08-17T17:43:38.513361+0000","last_fullsized":"2024-08-17T17:43:38.513361+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":247,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512292+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:43:26.512292+0000","last_peered":"2024-08-17T17:43:26.512292+0000","last_clean":"2024-08-17T17:43:26.512292+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:43:26.512292+0000","last_undegraded":"2024-08-17T17:43:26.512292+0000","last_fullsized":"2024-08-17T17:43:26.512292+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":602,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510301+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:43:38.510301+0000","last_peered":"2024-08-17T17:43:38.510301+0000","last_clean":"2024-08-17T17:43:38.510301+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:43:38.510301+0000","last_undegraded":"2024-08-17T17:43:38.510301+0000","last_fullsized":"2024-08-17T17:43:38.510301+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":415,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.558594+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:43:26.558594+0000","last_peered":"2024-08-17T17:43:26.558594+0000","last_clean":"2024-08-17T17:43:26.558594+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:43:26.558594+0000","last_undegraded":"2024-08-17T17:43:26.558594+0000","last_fullsized":"2024-08-17T17:43:26.558594+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"395'11731","reported_seq":19138,"reported_epoch":395,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:43:45.241115+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:43:45.241115+0000","last_peered":"2024-08-17T17:43:45.241115+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:43:45.241115+0000","last_undegraded":"2024-08-17T17:43:45.241115+0000","last_fullsized":"2024-08-17T17:43:45.241115+0000","mapping_epoch":337,"log_start":"327'4863","ondisk_log_start":"327'4863","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6868,"log_dups_size":0,"ondisk_log_size":6868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194838528,"num_objects":2973,"num_object_clones":0,"num_object_copies":8919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2973,"num_objects_unfound":0,"num_objects_dirty":2973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11731,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3650,"num_bytes_recovered":239206400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2973}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":100,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.554744+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:43:26.554744+0000","last_peered":"2024-08-17T17:43:26.554744+0000","last_clean":"2024-08-17T17:43:26.554744+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:43:26.554744+0000","last_undegraded":"2024-08-17T17:43:26.554744+0000","last_fullsized":"2024-08-17T17:43:26.554744+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":281,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.618181+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:43:26.618181+0000","last_peered":"2024-08-17T17:43:26.618181+0000","last_clean":"2024-08-17T17:43:26.618181+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:43:26.618181+0000","last_undegraded":"2024-08-17T17:43:26.618181+0000","last_fullsized":"2024-08-17T17:43:26.618181+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"395'11904","reported_seq":11997,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:42.140338+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:43:42.140338+0000","last_peered":"2024-08-17T17:43:42.140338+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:43:42.140338+0000","last_undegraded":"2024-08-17T17:43:42.140338+0000","last_fullsized":"2024-08-17T17:43:42.140338+0000","mapping_epoch":337,"log_start":"327'5382","ondisk_log_start":"327'5382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207749120,"num_objects":3170,"num_object_clones":0,"num_object_copies":9510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3170,"num_objects_unfound":0,"num_objects_dirty":3170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11904,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3170}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":102,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512820+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:43:26.512820+0000","last_peered":"2024-08-17T17:43:26.512820+0000","last_clean":"2024-08-17T17:43:26.512820+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:43:26.512820+0000","last_undegraded":"2024-08-17T17:43:26.512820+0000","last_fullsized":"2024-08-17T17:43:26.512820+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":366,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512261+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:43:26.512261+0000","last_peered":"2024-08-17T17:43:26.512261+0000","last_clean":"2024-08-17T17:43:26.512261+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:43:26.512261+0000","last_undegraded":"2024-08-17T17:43:26.512261+0000","last_fullsized":"2024-08-17T17:43:26.512261+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"395'11723","reported_seq":11819,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:45.237424+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:43:45.237424+0000","last_peered":"2024-08-17T17:43:45.237424+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:43:45.237424+0000","last_undegraded":"2024-08-17T17:43:45.237424+0000","last_fullsized":"2024-08-17T17:43:45.237424+0000","mapping_epoch":337,"log_start":"327'4853","ondisk_log_start":"327'4853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197328896,"num_objects":3011,"num_object_clones":0,"num_object_copies":9033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3011,"num_objects_unfound":0,"num_objects_dirty":3011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11723,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3011}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":98,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.617340+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:43:26.617340+0000","last_peered":"2024-08-17T17:43:26.617340+0000","last_clean":"2024-08-17T17:43:26.617340+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:43:26.617340+0000","last_undegraded":"2024-08-17T17:43:26.617340+0000","last_fullsized":"2024-08-17T17:43:26.617340+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":376,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.557681+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:43:26.557681+0000","last_peered":"2024-08-17T17:43:26.557681+0000","last_clean":"2024-08-17T17:43:26.557681+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:43:26.557681+0000","last_undegraded":"2024-08-17T17:43:26.557681+0000","last_fullsized":"2024-08-17T17:43:26.557681+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"395'11945","reported_seq":12116,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:44.913216+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:43:44.913216+0000","last_peered":"2024-08-17T17:43:44.913216+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:43:44.913216+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'4913","ondisk_log_start":"327'4913","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197459968,"num_objects":3013,"num_object_clones":0,"num_object_copies":9039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11945,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-3756},{"shards":"0,3,7","objects":3013}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":546,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510474+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:43:38.510474+0000","last_peered":"2024-08-17T17:43:38.510474+0000","last_clean":"2024-08-17T17:43:38.510474+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:43:38.510474+0000","last_undegraded":"2024-08-17T17:43:38.510474+0000","last_fullsized":"2024-08-17T17:43:38.510474+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":312,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.518613+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:43:38.518613+0000","last_peered":"2024-08-17T17:43:38.518613+0000","last_clean":"2024-08-17T17:43:38.518613+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:43:38.518613+0000","last_undegraded":"2024-08-17T17:43:38.518613+0000","last_fullsized":"2024-08-17T17:43:38.518613+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"395'11599","reported_seq":11683,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:42.075232+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:43:42.075232+0000","last_peered":"2024-08-17T17:43:42.075232+0000","last_clean":"2024-08-17T17:43:42.075232+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:43:42.075232+0000","last_undegraded":"2024-08-17T17:43:42.075232+0000","last_fullsized":"2024-08-17T17:43:42.075232+0000","mapping_epoch":319,"log_start":"327'5074","ondisk_log_start":"327'5074","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196280320,"num_objects":2995,"num_object_clones":0,"num_object_copies":8985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11599,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":635,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510318+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:43:38.510318+0000","last_peered":"2024-08-17T17:43:38.510318+0000","last_clean":"2024-08-17T17:43:38.510318+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:43:38.510318+0000","last_undegraded":"2024-08-17T17:43:38.510318+0000","last_fullsized":"2024-08-17T17:43:38.510318+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":349,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.513615+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:43:26.513615+0000","last_peered":"2024-08-17T17:43:26.513615+0000","last_clean":"2024-08-17T17:43:26.513615+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:43:26.513615+0000","last_undegraded":"2024-08-17T17:43:26.513615+0000","last_fullsized":"2024-08-17T17:43:26.513615+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"395'11836","reported_seq":11930,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:42.119797+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:43:42.119797+0000","last_peered":"2024-08-17T17:43:42.119797+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:43:42.119797+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'5235","ondisk_log_start":"327'5235","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202899456,"num_objects":3096,"num_object_clones":0,"num_object_copies":9288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11836,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3096},{"shards":"1,6","objects":-4270}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":350,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.557814+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:43:26.557814+0000","last_peered":"2024-08-17T17:43:26.557814+0000","last_clean":"2024-08-17T17:43:26.557814+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:43:26.557814+0000","last_undegraded":"2024-08-17T17:43:26.557814+0000","last_fullsized":"2024-08-17T17:43:26.557814+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":376,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.513013+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:43:26.513013+0000","last_peered":"2024-08-17T17:43:26.513013+0000","last_clean":"2024-08-17T17:43:26.513013+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:43:26.513013+0000","last_undegraded":"2024-08-17T17:43:26.513013+0000","last_fullsized":"2024-08-17T17:43:26.513013+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"395'11965","reported_seq":12137,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:43.735715+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:43:43.735715+0000","last_peered":"2024-08-17T17:43:43.735715+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:43:43.735715+0000","last_undegraded":"2024-08-17T17:43:43.735715+0000","last_fullsized":"2024-08-17T17:43:43.735715+0000","mapping_epoch":337,"log_start":"328'6361","ondisk_log_start":"328'6361","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199557120,"num_objects":3045,"num_object_clones":0,"num_object_copies":9135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3045,"num_objects_unfound":0,"num_objects_dirty":3045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11965,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3045}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":387,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512395+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:43:26.512395+0000","last_peered":"2024-08-17T17:43:26.512395+0000","last_clean":"2024-08-17T17:43:26.512395+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:43:26.512395+0000","last_undegraded":"2024-08-17T17:43:26.512395+0000","last_fullsized":"2024-08-17T17:43:26.512395+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":349,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512881+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:43:26.512881+0000","last_peered":"2024-08-17T17:43:26.512881+0000","last_clean":"2024-08-17T17:43:26.512881+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:43:26.512881+0000","last_undegraded":"2024-08-17T17:43:26.512881+0000","last_fullsized":"2024-08-17T17:43:26.512881+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"395'11776","reported_seq":19323,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:45.210693+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:43:45.210693+0000","last_peered":"2024-08-17T17:43:45.210693+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:43:45.210693+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:43:45.210693+0000","mapping_epoch":337,"log_start":"327'4941","ondisk_log_start":"327'4941","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191889408,"num_objects":2928,"num_object_clones":0,"num_object_copies":8784,"num_objects_missing_on_primary":3625,"num_objects_missing":3625,"num_objects_degraded":2928,"num_objects_misplaced":3625,"num_objects_unfound":0,"num_objects_dirty":2928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11776,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3719,"num_bytes_recovered":243662848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":8},{"shards":"1,2,7","objects":2919},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":605,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.511249+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:43:38.511249+0000","last_peered":"2024-08-17T17:43:38.511249+0000","last_clean":"2024-08-17T17:43:38.511249+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:43:38.511249+0000","last_undegraded":"2024-08-17T17:43:38.511249+0000","last_fullsized":"2024-08-17T17:43:38.511249+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":542,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510384+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:43:38.510384+0000","last_peered":"2024-08-17T17:43:38.510384+0000","last_clean":"2024-08-17T17:43:38.510384+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:43:38.510384+0000","last_undegraded":"2024-08-17T17:43:38.510384+0000","last_fullsized":"2024-08-17T17:43:38.510384+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"395'11897","reported_seq":19456,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:43.901073+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:43:43.901073+0000","last_peered":"2024-08-17T17:43:43.901073+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:43:43.901073+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:43:43.901073+0000","mapping_epoch":337,"log_start":"328'6244","ondisk_log_start":"328'6244","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5653,"log_dups_size":0,"ondisk_log_size":5653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196476952,"num_objects":2999,"num_object_clones":0,"num_object_copies":8997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2999,"num_objects_misplaced":3749,"num_objects_unfound":0,"num_objects_dirty":2999,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11897,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3699,"num_bytes_recovered":242352128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2999}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":736,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:38.514950+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:43:38.514950+0000","last_peered":"2024-08-17T17:43:38.514950+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:43:38.514950+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":324,"reported_epoch":394,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:43:26.617746+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:43:26.617746+0000","last_peered":"2024-08-17T17:43:26.617746+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:43:26.617746+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:43:26.617746+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":584,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.511084+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:43:38.511084+0000","last_peered":"2024-08-17T17:43:38.511084+0000","last_clean":"2024-08-17T17:43:38.511084+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:43:38.511084+0000","last_undegraded":"2024-08-17T17:43:38.511084+0000","last_fullsized":"2024-08-17T17:43:38.511084+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":359,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.558526+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:43:26.558526+0000","last_peered":"2024-08-17T17:43:26.558526+0000","last_clean":"2024-08-17T17:43:26.558526+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:43:26.558526+0000","last_undegraded":"2024-08-17T17:43:26.558526+0000","last_fullsized":"2024-08-17T17:43:26.558526+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"395'11722","reported_seq":11896,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:44.869741+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:43:44.869741+0000","last_peered":"2024-08-17T17:43:44.869741+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:43:44.869741+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'4721","ondisk_log_start":"327'4721","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198049792,"num_objects":3022,"num_object_clones":0,"num_object_copies":9066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11722,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-3625},{"shards":"0,4,7","objects":3022}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":401,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.506861+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:43:38.506861+0000","last_peered":"2024-08-17T17:43:38.506861+0000","last_clean":"2024-08-17T17:43:38.506861+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:43:38.506861+0000","last_undegraded":"2024-08-17T17:43:38.506861+0000","last_fullsized":"2024-08-17T17:43:38.506861+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":331,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512206+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:43:26.512206+0000","last_peered":"2024-08-17T17:43:26.512206+0000","last_clean":"2024-08-17T17:43:26.512206+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:43:26.512206+0000","last_undegraded":"2024-08-17T17:43:26.512206+0000","last_fullsized":"2024-08-17T17:43:26.512206+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":326,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.523202+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:43:38.523202+0000","last_peered":"2024-08-17T17:43:38.523202+0000","last_clean":"2024-08-17T17:43:38.523202+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:43:38.523202+0000","last_undegraded":"2024-08-17T17:43:38.523202+0000","last_fullsized":"2024-08-17T17:43:38.523202+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"395'11808","reported_seq":11902,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:41.992124+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:43:41.992124+0000","last_peered":"2024-08-17T17:43:41.992124+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:43:41.992124+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"327'5217","ondisk_log_start":"327'5217","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195821568,"num_objects":2988,"num_object_clones":0,"num_object_copies":8964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11808,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2988},{"shards":"5,6","objects":-4308}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":493,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510437+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:43:38.510437+0000","last_peered":"2024-08-17T17:43:38.510437+0000","last_clean":"2024-08-17T17:43:38.510437+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:43:38.510437+0000","last_undegraded":"2024-08-17T17:43:38.510437+0000","last_fullsized":"2024-08-17T17:43:38.510437+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.557557+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:43:26.557557+0000","last_peered":"2024-08-17T17:43:26.557557+0000","last_clean":"2024-08-17T17:43:26.557557+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:43:26.557557+0000","last_undegraded":"2024-08-17T17:43:26.557557+0000","last_fullsized":"2024-08-17T17:43:26.557557+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":677,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.508688+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:43:38.508688+0000","last_peered":"2024-08-17T17:43:38.508688+0000","last_clean":"2024-08-17T17:43:38.508688+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:43:38.508688+0000","last_undegraded":"2024-08-17T17:43:38.508688+0000","last_fullsized":"2024-08-17T17:43:38.508688+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"395'11690","reported_seq":12588,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:44.358745+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:43:44.358745+0000","last_peered":"2024-08-17T17:43:44.358745+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:43:44.358745+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"326'4616","ondisk_log_start":"326'4616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724416,"num_objects":2956,"num_object_clones":0,"num_object_copies":8868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11690,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2956},{"shards":"0,5","objects":-3698}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":469,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.507048+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:43:38.507048+0000","last_peered":"2024-08-17T17:43:38.507048+0000","last_clean":"2024-08-17T17:43:38.507048+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:43:38.507048+0000","last_undegraded":"2024-08-17T17:43:38.507048+0000","last_fullsized":"2024-08-17T17:43:38.507048+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":370,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.513099+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:43:26.513099+0000","last_peered":"2024-08-17T17:43:26.513099+0000","last_clean":"2024-08-17T17:43:26.513099+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:43:26.513099+0000","last_undegraded":"2024-08-17T17:43:26.513099+0000","last_fullsized":"2024-08-17T17:43:26.513099+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":263,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510232+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:43:38.510232+0000","last_peered":"2024-08-17T17:43:38.510232+0000","last_clean":"2024-08-17T17:43:38.510232+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:43:38.510232+0000","last_undegraded":"2024-08-17T17:43:38.510232+0000","last_fullsized":"2024-08-17T17:43:38.510232+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"395'11843","reported_seq":26860,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:42.793674+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:43:42.793674+0000","last_peered":"2024-08-17T17:43:42.793674+0000","last_clean":"2024-08-17T17:43:42.793674+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:43:42.793674+0000","last_undegraded":"2024-08-17T17:43:42.793674+0000","last_fullsized":"2024-08-17T17:43:42.793674+0000","mapping_epoch":355,"log_start":"327'5530","ondisk_log_start":"327'5530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196149248,"num_objects":2993,"num_object_clones":0,"num_object_copies":8979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11843,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":309,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.617332+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:43:26.617332+0000","last_peered":"2024-08-17T17:43:26.617332+0000","last_clean":"2024-08-17T17:43:26.617332+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:43:26.617332+0000","last_undegraded":"2024-08-17T17:43:26.617332+0000","last_fullsized":"2024-08-17T17:43:26.617332+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":132,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510143+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:43:38.510143+0000","last_peered":"2024-08-17T17:43:38.510143+0000","last_clean":"2024-08-17T17:43:38.510143+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:43:38.510143+0000","last_undegraded":"2024-08-17T17:43:38.510143+0000","last_fullsized":"2024-08-17T17:43:38.510143+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":628,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.507343+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:43:38.507343+0000","last_peered":"2024-08-17T17:43:38.507343+0000","last_clean":"2024-08-17T17:43:38.507343+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:43:38.507343+0000","last_undegraded":"2024-08-17T17:43:38.507343+0000","last_fullsized":"2024-08-17T17:43:38.507343+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"395'11543","reported_seq":26128,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:42.615006+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:43:42.615006+0000","last_peered":"2024-08-17T17:43:42.615006+0000","last_clean":"2024-08-17T17:43:42.615006+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:43:42.615006+0000","last_undegraded":"2024-08-17T17:43:42.615006+0000","last_fullsized":"2024-08-17T17:43:42.615006+0000","mapping_epoch":345,"log_start":"327'5272","ondisk_log_start":"327'5272","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6271,"log_dups_size":0,"ondisk_log_size":6271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187498496,"num_objects":2861,"num_object_clones":0,"num_object_copies":8583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11543,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":305,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.617465+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:43:26.617465+0000","last_peered":"2024-08-17T17:43:26.617465+0000","last_clean":"2024-08-17T17:43:26.617465+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:43:26.617465+0000","last_undegraded":"2024-08-17T17:43:26.617465+0000","last_fullsized":"2024-08-17T17:43:26.617465+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":383,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.515233+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:43:38.515233+0000","last_peered":"2024-08-17T17:43:38.515233+0000","last_clean":"2024-08-17T17:43:38.515233+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:43:38.515233+0000","last_undegraded":"2024-08-17T17:43:38.515233+0000","last_fullsized":"2024-08-17T17:43:38.515233+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":447,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.617876+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:43:26.617876+0000","last_peered":"2024-08-17T17:43:26.617876+0000","last_clean":"2024-08-17T17:43:26.617876+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:43:26.617876+0000","last_undegraded":"2024-08-17T17:43:26.617876+0000","last_fullsized":"2024-08-17T17:43:26.617876+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"395'11721","reported_seq":11914,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:42.809200+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:43:42.809200+0000","last_peered":"2024-08-17T17:43:42.809200+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:43:42.809200+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'5387","ondisk_log_start":"327'5387","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6334,"log_dups_size":0,"ondisk_log_size":6334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200736768,"num_objects":3063,"num_object_clones":0,"num_object_copies":9189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11721,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":3063},{"shards":"0,3","objects":-4185}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":658,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510472+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:43:38.510472+0000","last_peered":"2024-08-17T17:43:38.510472+0000","last_clean":"2024-08-17T17:43:38.510472+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:43:38.510472+0000","last_undegraded":"2024-08-17T17:43:38.510472+0000","last_fullsized":"2024-08-17T17:43:38.510472+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":626,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.508351+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:43:38.508351+0000","last_peered":"2024-08-17T17:43:38.508351+0000","last_clean":"2024-08-17T17:43:38.508351+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:43:38.508351+0000","last_undegraded":"2024-08-17T17:43:38.508351+0000","last_fullsized":"2024-08-17T17:43:38.508351+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":644,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510032+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:43:38.510032+0000","last_peered":"2024-08-17T17:43:38.510032+0000","last_clean":"2024-08-17T17:43:38.510032+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:43:38.510032+0000","last_undegraded":"2024-08-17T17:43:38.510032+0000","last_fullsized":"2024-08-17T17:43:38.510032+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"395'11624","reported_seq":26316,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:41.872118+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:43:41.872118+0000","last_peered":"2024-08-17T17:43:41.872118+0000","last_clean":"2024-08-17T17:43:41.872118+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:43:41.872118+0000","last_undegraded":"2024-08-17T17:43:41.872118+0000","last_fullsized":"2024-08-17T17:43:41.872118+0000","mapping_epoch":347,"log_start":"327'5072","ondisk_log_start":"327'5072","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193331200,"num_objects":2950,"num_object_clones":0,"num_object_copies":8850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11624,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":316,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.518480+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:43:38.518480+0000","last_peered":"2024-08-17T17:43:38.518480+0000","last_clean":"2024-08-17T17:43:38.518480+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:43:38.518480+0000","last_undegraded":"2024-08-17T17:43:38.518480+0000","last_fullsized":"2024-08-17T17:43:38.518480+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":399,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.558382+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:43:26.558382+0000","last_peered":"2024-08-17T17:43:26.558382+0000","last_clean":"2024-08-17T17:43:26.558382+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:43:26.558382+0000","last_undegraded":"2024-08-17T17:43:26.558382+0000","last_fullsized":"2024-08-17T17:43:26.558382+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":349,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512323+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:43:26.512323+0000","last_peered":"2024-08-17T17:43:26.512323+0000","last_clean":"2024-08-17T17:43:26.512323+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:43:26.512323+0000","last_undegraded":"2024-08-17T17:43:26.512323+0000","last_fullsized":"2024-08-17T17:43:26.512323+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":349,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512945+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:43:26.512945+0000","last_peered":"2024-08-17T17:43:26.512945+0000","last_clean":"2024-08-17T17:43:26.512945+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:43:26.512945+0000","last_undegraded":"2024-08-17T17:43:26.512945+0000","last_fullsized":"2024-08-17T17:43:26.512945+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":392,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.513734+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:43:26.513734+0000","last_peered":"2024-08-17T17:43:26.513734+0000","last_clean":"2024-08-17T17:43:26.513734+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:43:26.513734+0000","last_undegraded":"2024-08-17T17:43:26.513734+0000","last_fullsized":"2024-08-17T17:43:26.513734+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":633,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.507144+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:43:38.507144+0000","last_peered":"2024-08-17T17:43:38.507144+0000","last_clean":"2024-08-17T17:43:38.507144+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:43:38.507144+0000","last_undegraded":"2024-08-17T17:43:38.507144+0000","last_fullsized":"2024-08-17T17:43:38.507144+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":173,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.511056+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:43:38.511056+0000","last_peered":"2024-08-17T17:43:38.511056+0000","last_clean":"2024-08-17T17:43:38.511056+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:43:38.511056+0000","last_undegraded":"2024-08-17T17:43:38.511056+0000","last_fullsized":"2024-08-17T17:43:38.511056+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":299,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.617611+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:43:26.617611+0000","last_peered":"2024-08-17T17:43:26.617611+0000","last_clean":"2024-08-17T17:43:26.617611+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:43:26.617611+0000","last_undegraded":"2024-08-17T17:43:26.617611+0000","last_fullsized":"2024-08-17T17:43:26.617611+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":591,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510066+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:43:38.510066+0000","last_peered":"2024-08-17T17:43:38.510066+0000","last_clean":"2024-08-17T17:43:38.510066+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:43:38.510066+0000","last_undegraded":"2024-08-17T17:43:38.510066+0000","last_fullsized":"2024-08-17T17:43:38.510066+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":443,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.618568+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:43:26.618568+0000","last_peered":"2024-08-17T17:43:26.618568+0000","last_clean":"2024-08-17T17:43:26.618568+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:43:26.618568+0000","last_undegraded":"2024-08-17T17:43:26.618568+0000","last_fullsized":"2024-08-17T17:43:26.618568+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":585,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.509928+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:43:38.509928+0000","last_peered":"2024-08-17T17:43:38.509928+0000","last_clean":"2024-08-17T17:43:38.509928+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:43:38.509928+0000","last_undegraded":"2024-08-17T17:43:38.509928+0000","last_fullsized":"2024-08-17T17:43:38.509928+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":366,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.558198+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:43:26.558198+0000","last_peered":"2024-08-17T17:43:26.558198+0000","last_clean":"2024-08-17T17:43:26.558198+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:43:26.558198+0000","last_undegraded":"2024-08-17T17:43:26.558198+0000","last_fullsized":"2024-08-17T17:43:26.558198+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":616,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.511167+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:43:38.511167+0000","last_peered":"2024-08-17T17:43:38.511167+0000","last_clean":"2024-08-17T17:43:38.511167+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:43:38.511167+0000","last_undegraded":"2024-08-17T17:43:38.511167+0000","last_fullsized":"2024-08-17T17:43:38.511167+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":333,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.513411+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:43:38.513411+0000","last_peered":"2024-08-17T17:43:38.513411+0000","last_clean":"2024-08-17T17:43:38.513411+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:43:38.513411+0000","last_undegraded":"2024-08-17T17:43:38.513411+0000","last_fullsized":"2024-08-17T17:43:38.513411+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":624,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.507205+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:43:38.507205+0000","last_peered":"2024-08-17T17:43:38.507205+0000","last_clean":"2024-08-17T17:43:38.507205+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:43:38.507205+0000","last_undegraded":"2024-08-17T17:43:38.507205+0000","last_fullsized":"2024-08-17T17:43:38.507205+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":584,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510940+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:43:38.510940+0000","last_peered":"2024-08-17T17:43:38.510940+0000","last_clean":"2024-08-17T17:43:38.510940+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:43:38.510940+0000","last_undegraded":"2024-08-17T17:43:38.510940+0000","last_fullsized":"2024-08-17T17:43:38.510940+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":378,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512545+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:43:26.512545+0000","last_peered":"2024-08-17T17:43:26.512545+0000","last_clean":"2024-08-17T17:43:26.512545+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:43:26.512545+0000","last_undegraded":"2024-08-17T17:43:26.512545+0000","last_fullsized":"2024-08-17T17:43:26.512545+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":273,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510398+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:43:38.510398+0000","last_peered":"2024-08-17T17:43:38.510398+0000","last_clean":"2024-08-17T17:43:38.510398+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:43:38.510398+0000","last_undegraded":"2024-08-17T17:43:38.510398+0000","last_fullsized":"2024-08-17T17:43:38.510398+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":329,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.515420+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:43:38.515420+0000","last_peered":"2024-08-17T17:43:38.515420+0000","last_clean":"2024-08-17T17:43:38.515420+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:43:38.515420+0000","last_undegraded":"2024-08-17T17:43:38.515420+0000","last_fullsized":"2024-08-17T17:43:38.515420+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":386,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512882+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:43:26.512882+0000","last_peered":"2024-08-17T17:43:26.512882+0000","last_clean":"2024-08-17T17:43:26.512882+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:43:26.512882+0000","last_undegraded":"2024-08-17T17:43:26.512882+0000","last_fullsized":"2024-08-17T17:43:26.512882+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":249,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.513674+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:43:26.513674+0000","last_peered":"2024-08-17T17:43:26.513674+0000","last_clean":"2024-08-17T17:43:26.513674+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:43:26.513674+0000","last_undegraded":"2024-08-17T17:43:26.513674+0000","last_fullsized":"2024-08-17T17:43:26.513674+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":386,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512488+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:43:26.512488+0000","last_peered":"2024-08-17T17:43:26.512488+0000","last_clean":"2024-08-17T17:43:26.512488+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:43:26.512488+0000","last_undegraded":"2024-08-17T17:43:26.512488+0000","last_fullsized":"2024-08-17T17:43:26.512488+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":249,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512874+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:43:26.512874+0000","last_peered":"2024-08-17T17:43:26.512874+0000","last_clean":"2024-08-17T17:43:26.512874+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:43:26.512874+0000","last_undegraded":"2024-08-17T17:43:26.512874+0000","last_fullsized":"2024-08-17T17:43:26.512874+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":636,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.510482+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:43:38.510482+0000","last_peered":"2024-08-17T17:43:38.510482+0000","last_clean":"2024-08-17T17:43:38.510482+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:43:38.510482+0000","last_undegraded":"2024-08-17T17:43:38.510482+0000","last_fullsized":"2024-08-17T17:43:38.510482+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":610,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.513470+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:43:38.513470+0000","last_peered":"2024-08-17T17:43:38.513470+0000","last_clean":"2024-08-17T17:43:38.513470+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:43:38.513470+0000","last_undegraded":"2024-08-17T17:43:38.513470+0000","last_fullsized":"2024-08-17T17:43:38.513470+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":349,"reported_epoch":394,"state":"active+clean","last_fresh":"2024-08-17T17:43:26.512231+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:43:26.512231+0000","last_peered":"2024-08-17T17:43:26.512231+0000","last_clean":"2024-08-17T17:43:26.512231+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:43:26.512231+0000","last_undegraded":"2024-08-17T17:43:26.512231+0000","last_fullsized":"2024-08-17T17:43:26.512231+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":332,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.515537+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:43:38.515537+0000","last_peered":"2024-08-17T17:43:38.515537+0000","last_clean":"2024-08-17T17:43:38.515537+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:43:38.515537+0000","last_undegraded":"2024-08-17T17:43:38.515537+0000","last_fullsized":"2024-08-17T17:43:38.515537+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3149791256,"num_objects":48063,"num_object_clones":0,"num_object_copies":144189,"num_objects_missing_on_primary":3625,"num_objects_missing":3625,"num_objects_degraded":50346,"num_objects_misplaced":19573,"num_objects_unfound":0,"num_objects_dirty":48063,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188327,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33347,"num_bytes_recovered":2062876672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11884310528,"data_stored":11884298312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104646,"ondisk_log_size":104646,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170128,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463497,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2251848,"kb_used_data":1358144,"kb_used_omap":1,"kb_used_meta":893694,"kb_avail":91493304,"statfs":{"total":95995035648,"available":93689143296,"internally_reserved":0,"allocated":1390739456,"data_stored":1388560982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915143000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492172,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774700,"kb_used_data":383524,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":92970452,"statfs":{"total":95995035648,"available":95201742848,"internally_reserved":0,"allocated":392728576,"data_stored":390572402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986763,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270780,"kb_used_data":766260,"kb_used_omap":1,"kb_used_meta":1504510,"kb_avail":91474372,"statfs":{"total":95995035648,"available":93669756928,"internally_reserved":0,"allocated":784650240,"data_stored":782482706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1540618584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,2,2,2,6,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225591,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2498412,"kb_used_data":1546596,"kb_used_omap":4,"kb_used_meta":951803,"kb_avail":91246740,"statfs":{"total":95995035648,"available":93436661760,"internally_reserved":0,"allocated":1583714304,"data_stored":1581558428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":974646720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075916,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533809,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2401136,"kb_used_data":1345256,"kb_used_omap":5,"kb_used_meta":1055866,"kb_avail":91344016,"statfs":{"total":95995035648,"available":93536272384,"internally_reserved":0,"allocated":1377542144,"data_stored":1375380946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1081207024},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089786,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2184072,"kb_used_data":1160960,"kb_used_omap":5,"kb_used_meta":1023098,"kb_avail":91561080,"statfs":{"total":95995035648,"available":93758545920,"internally_reserved":0,"allocated":1188823040,"data_stored":1186644625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1047652411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1182007296,"data_stored":1182007296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1371738112,"data_stored":1371734040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1578369024,"data_stored":1578369024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":776470528,"data_stored":776470528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":387448832,"data_stored":387448832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1381761024,"data_stored":1381761024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1867255808,"data_stored":1867251736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:48.459 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:43:48.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:48.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:48.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:48.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:48.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:48.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:48.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:48.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:48.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:48.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:48.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:49.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:49.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:49.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:49.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:49.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:49.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:49.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:49.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:49.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:49.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:49.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.592 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:49.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:49.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:49.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:49 smithi152 ceph-mon[93456]: pgmap v1532: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 50349/142839 objects degraded (35.249%); 19510/142839 objects misplaced (13.659%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:43:49.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:49.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:49.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:49.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:49 smithi142 ceph-mon[106729]: pgmap v1532: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 50349/142839 objects degraded (35.249%); 19510/142839 objects misplaced (13.659%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:43:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:49 smithi142 ceph-mon[103614]: pgmap v1532: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 50349/142839 objects degraded (35.249%); 19510/142839 objects misplaced (13.659%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:43:49.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:49.968 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:49.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:49.978 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:43:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:43:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:43:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:43:49.978 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:43:50.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:50.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:50.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:50.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:50.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:50.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:50.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:50.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:50.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:50.480 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:43:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:43:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:50.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:50.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:50.687 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:50.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:50.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:50.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:43:50.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:50.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:50.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:50.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:50.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:51.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:51.108 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:43:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:51.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:51.179 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:51.180 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:51.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:51.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:51.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:51.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:51.181 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:51.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:51.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:51.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:51.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:51.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:51.452 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:43:51.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:51.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:51 smithi152 ceph-mon[93456]: pgmap v1533: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 184 op/s; 50246/141537 objects degraded (35.500%); 19238/141537 objects misplaced (13.592%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:43:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50246/141537 objects degraded (35.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1381310266' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:51.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3079282701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:51.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:51.645 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[106729]: pgmap v1533: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 184 op/s; 50246/141537 objects degraded (35.500%); 19238/141537 objects misplaced (13.592%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50246/141537 objects degraded (35.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1381310266' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3079282701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[103614]: pgmap v1533: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 184 op/s; 50246/141537 objects degraded (35.500%); 19238/141537 objects misplaced (13.592%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50246/141537 objects degraded (35.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:51.706 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1381310266' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:43:51.707 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3079282701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:51.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:51.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:51.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:51.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:51.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:52.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:52.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:43:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:43:51] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:43:52.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:52.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:52.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:52.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:52.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:52.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:52.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:52.510 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:52.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:52 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:43:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1381310266' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:52 smithi142 ceph-mon[106729]: osdmap e396: 8 total, 8 up, 6 in 2024-08-17T17:43:52.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:52 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:43:52.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1381310266' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:52.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:52 smithi142 ceph-mon[103614]: osdmap e396: 8 total, 8 up, 6 in 2024-08-17T17:43:52.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:52.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:52.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:52.790 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:52.801 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:52 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:43:52.801 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1381310266' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:43:52.801 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:52 smithi152 ceph-mon[93456]: osdmap e396: 8 total, 8 up, 6 in 2024-08-17T17:43:52.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:52.914 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:52.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:53.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:53.039 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:53.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:53.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:53.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:53.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:53.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:53.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:53.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:53.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:53.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:53.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:53.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:53.761 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:53 smithi142 ceph-mon[106729]: pgmap v1535: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 50246/140601 objects degraded (35.737%); 19175/140601 objects misplaced (13.638%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:53.761 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:53 smithi142 ceph-mon[103614]: pgmap v1535: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 50246/140601 objects degraded (35.737%); 19175/140601 objects misplaced (13.638%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:53.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:53.785 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:53.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:53 smithi152 ceph-mon[93456]: pgmap v1535: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 50246/140601 objects degraded (35.737%); 19175/140601 objects misplaced (13.638%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:43:53.841 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:53.841 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:53.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:53.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:53.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:53.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:54.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:54.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:54.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:54.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:54.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:54.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:54.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:54.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:54.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:54.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:54.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:54.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:54.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:43:54.633 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:43:54.634 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 50184/139803 objects degraded (35.896%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2315,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":396,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":46601,"data_bytes":3052869605,"bytes_used":12484419584,"bytes_avail":563485794304,"bytes_total":575970213888,"degraded_objects":50184,"degraded_total":139803,"degraded_ratio":0.35896225402888349,"misplaced_objects":19073,"misplaced_total":139803,"misplaced_ratio":0.13642768753174109,"recovering_objects_per_sec":34,"recovering_bytes_per_sec":2247214,"recovering_keys_per_sec":0,"num_objects_recovered":343,"num_bytes_recovered":22478848,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":181},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":35,"modified":"2024-08-17T17:43:38.379260+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:43:54.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:54.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:54.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:43:54.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:54.807 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:54.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:54.926 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:43:54.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:54.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:55.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:55.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:55.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:55.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:55.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:55.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:43:55.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:55.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:55.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:55.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:55.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:55.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:55.456 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:43:55.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:55.508 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:55.522 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:55.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:55.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:55.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:55.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:55.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:55 smithi142 ceph-mon[106729]: pgmap v1536: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 181 op/s; 50184/139803 objects degraded (35.896%); 19073/139803 objects misplaced (13.643%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:55.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3150649831' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:55.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:55 smithi142 ceph-mon[103614]: pgmap v1536: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 181 op/s; 50184/139803 objects degraded (35.896%); 19073/139803 objects misplaced (13.643%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:55.682 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3150649831' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:55.690 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:55.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:55.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:55.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:55 smithi152 ceph-mon[93456]: pgmap v1536: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 181 op/s; 50184/139803 objects degraded (35.896%); 19073/139803 objects misplaced (13.643%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3150649831' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:43:55.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:55.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:55.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:55.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:56.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:56.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:56.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:56.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:56.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:56.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:56.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:56.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:56.444 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:56.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:56.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50184/139803 objects degraded (35.896%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:56.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:56.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:56.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:56.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:56.820 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50184/139803 objects degraded (35.896%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50184/139803 objects degraded (35.896%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:43:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:43:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:56.881 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:43:56.881 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:43:56.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:56.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:56.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:57.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:57.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:57.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:57.196 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:57.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:57.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:57.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:57.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:57.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:57.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:57.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:57.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:57.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:57.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:43:57.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:43:57.698 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:57.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:57.812 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:57 smithi142 ceph-mon[106729]: pgmap v1537: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 50133/138603 objects degraded (36.170%); 18887/138603 objects misplaced (13.627%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:57.813 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:57 smithi142 ceph-mon[103614]: pgmap v1537: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 50133/138603 objects degraded (36.170%); 18887/138603 objects misplaced (13.627%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:57.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:57 smithi152 ceph-mon[93456]: pgmap v1537: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 50133/138603 objects degraded (36.170%); 18887/138603 objects misplaced (13.627%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:57.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:57.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:57.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:57.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:43:58.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:58.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:58.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:58.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:58.195 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:43:58.296 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:43:58.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:58.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:58.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:58.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:58.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:58.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:43:58.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:43:58.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:58.699 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:58.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:58.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:58.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:58.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:58.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:59.050 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:59.075 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:59.081 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:43:59.081 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:43:59.099 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:43:59.099 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:43:59.116 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:43:59.116 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:43:59.132 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:43:59.132 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:43:59.147 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:43:59.148 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:43:59.164 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:43:59.164 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:43:59.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:43:59.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:43:59.180 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:43:59.196 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:43:59.196 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:43:59.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:43:59.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:59.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:59.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:43:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:59.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:43:59.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:43:59.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:43:59.674 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:43:59.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:59.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:43:59.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:43:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.823 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:43:59.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:43:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:59 smithi152 ceph-mon[93456]: pgmap v1538: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 184 op/s; 50133/137310 objects degraded (36.511%); 18839/137310 objects misplaced (13.720%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:59.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:43:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3740164938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:59 smithi142 ceph-mon[106729]: pgmap v1538: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 184 op/s; 50133/137310 objects degraded (36.511%); 18839/137310 objects misplaced (13.720%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:43:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3740164938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:59 smithi142 ceph-mon[103614]: pgmap v1538: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 184 op/s; 50133/137310 objects degraded (36.511%); 18839/137310 objects misplaced (13.720%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:43:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:43:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3740164938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:43:59.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:43:59.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:43:59.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:43:59.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:00.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:00.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:00.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:00.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:00.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:00.239 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:44:00.239 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:44:00.239 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:44:00.240 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:44:00.240 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:44:00.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:00.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:00.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:00.451 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:00.553 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:44:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:44:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:00.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:00.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:00.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:00.697 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:00.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:00.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:00.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:00.949 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:00.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:01.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:01.076 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:01.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:01.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:01.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:01.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:01.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:01.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:01.488 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:44:01.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:44:01.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:01.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[106729]: pgmap v1539: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 50018/136065 objects degraded (36.760%); 18554/136065 objects misplaced (13.636%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4244617763' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 50018/136065 objects degraded (36.760%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[103614]: pgmap v1539: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 50018/136065 objects degraded (36.760%); 18554/136065 objects misplaced (13.636%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4244617763' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 50018/136065 objects degraded (36.760%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:01.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:01.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:01.815 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:01.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:01 smithi152 ceph-mon[93456]: pgmap v1539: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 50018/136065 objects degraded (36.760%); 18554/136065 objects misplaced (13.636%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4244617763' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 50018/136065 objects degraded (36.760%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:01 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:01.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:01.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:01.966 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:01.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:01.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:44:02.030 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:44:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:44:01] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-08-17T17:44:02.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:02.092 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:02.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:02.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:02.317 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:02.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:02.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:02.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:02.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:02 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:02.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:02.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:02 smithi152 ceph-mon[93456]: osdmap e397: 8 total, 8 up, 6 in 2024-08-17T17:44:02.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:02.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:02.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:02.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:02.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:02 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:02.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:02.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:02 smithi142 ceph-mon[106729]: osdmap e397: 8 total, 8 up, 6 in 2024-08-17T17:44:02.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:02 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:02.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:02.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:02 smithi142 ceph-mon[103614]: osdmap e397: 8 total, 8 up, 6 in 2024-08-17T17:44:02.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:02.830 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:02.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:02.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:02.960 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:44:02.961 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:44:02.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:02.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:02.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:03.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:03.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:03.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:03.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:03.228 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:03.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:03.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:03.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:03.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:03.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:03.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:03.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:03.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:03.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:03.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:03.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:03 smithi152 ceph-mon[93456]: pgmap v1541: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50018/135447 objects degraded (36.928%); 18512/135447 objects misplaced (13.667%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:129851 12.f 12:fa60de86:::benchmark_data_smithi142_261410_object11654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 579.12151377299995, 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34876531300000002, 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 129851 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647393+0000", 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647391+0000", 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647400+0000", 2024-08-17T17:44:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1290000000000002e-06 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647417+0000", 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7139e-05 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647425+0000", 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3310000000000004e-06 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647482+0000", 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6628000000000001e-05 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647642+0000", 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000160402 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.647812+0000", 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016993499999999999 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.650735+0000", 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002922371 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.651441+0000", 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070664600000000001 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.992978+0000", 2024-08-17T17:44:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34153630699999998 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.993043+0000", 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4863999999999994e-05 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:34:24.996158+0000", 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031156859999999999 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131638 12.f 12:fb3cd103:::benchmark_data_smithi142_261410_object13441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 540.50676559800002, 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35214242000000001, 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131638 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262141+0000", 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262140+0000", 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:44:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262145+0000", 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8250000000000004e-06 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262153+0000", 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2870000000000004e-06 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262157+0000", 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6629999999999999e-06 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262194+0000", 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7626000000000002e-05 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262367+0000", 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000172366 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.262545+0000", 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017861400000000001 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.265416+0000", 2024-08-17T17:44:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028709809999999999 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.266086+0000", 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066993100000000002 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.611065+0000", 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34497888500000001 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.611104+0000", 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8411999999999997e-05 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:03.614283+0000", 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003179856 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131763 12.f 12:f549739a:::benchmark_data_smithi142_261410_object13566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 539.21663110999998, 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37794038699999999, 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131763 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552276+0000", 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552275+0000", 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:44:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552279+0000", 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3080000000000001e-06 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552290+0000", 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0900000000000001e-05 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552295+0000", 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8879999999999996e-06 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552328+0000", 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3683999999999998e-05 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552433+0000", 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104164 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.552533+0000", 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100392 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.555327+0000", 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002794019 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.555992+0000", 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066522 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.927119+0000", 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.371126601 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.927153+0000", 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3800000000000002e-05 2024-08-17T17:44:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:04.930216+0000", 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030633309999999999 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:131820 12.9 12:9a926c4c:::benchmark_data_smithi142_261410_object13623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 538.11769422700002, 2024-08-17T17:44:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81736743099999998, 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 131820 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651212+0000", 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651211+0000", 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651218+0000", 2024-08-17T17:44:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6810000000000001e-06 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651229+0000", 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1258e-05 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651234+0000", 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5379999999999996e-06 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651287+0000", 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3192000000000002e-05 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:05.651298+0000", 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.15e-05 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461578+0000", 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81027978499999997 2024-08-17T17:44:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461702+0000", 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012391299999999999 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.461829+0000", 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012717300000000001 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.465382+0000", 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035532430000000002 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.466095+0000", 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00071217299999999997 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468482+0000", 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023874 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468535+0000", 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3319000000000002e-05 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:06.468580+0000", 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4569e-05 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:134422 12.f 12:fa29169b:::benchmark_data_smithi142_261410_object16225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-08-17T17:44:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 503.31864404100003, 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.45255063099999998, 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 134422 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450263+0000", 2024-08-17T17:44:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450261+0000", 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450270+0000", 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5539999999999998e-06 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450286+0000", 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6568000000000001e-05 2024-08-17T17:44:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450292+0000", 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1550000000000004e-06 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450333+0000", 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0472000000000001e-05 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450457+0000", 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012431599999999999 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.450565+0000", 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108013 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.453491+0000", 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029262239999999998 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.453504+0000", 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2989999999999999e-05 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.898771+0000", 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.44526617200000002 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.898830+0000", 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9506999999999998e-05 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:40.902813+0000", 2024-08-17T17:44:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039831399999999996 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:134490 12.f 12:f6a35853:::benchmark_data_smithi142_261410_object16293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 502.42498165400002, 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36548067000000001, 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 134490 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343925+0000", 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343924+0000", 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343929+0000", 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3769999999999998e-06 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343937+0000", 2024-08-17T17:44:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1969999999999996e-06 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343940+0000", 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.641e-06 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.343972+0000", 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1508999999999999e-05 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.344083+0000", 2024-08-17T17:44:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011066200000000001 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.344183+0000", 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9959999999999998e-05 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.347096+0000", 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029132110000000002 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.347261+0000", 2024-08-17T17:44:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016498399999999999 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.706519+0000", 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35925865099999998 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.706556+0000", 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6850999999999997e-05 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:41.709406+0000", 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028494760000000001 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:135448 12.f 12:fe02232f:::benchmark_data_smithi142_261410_object17251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 489.33555761999997, 2024-08-17T17:44:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.46790546399999999, 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 135448 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433349+0000", 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433347+0000", 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433357+0000", 2024-08-17T17:44:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0322999999999999e-05 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433376+0000", 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8244999999999999e-05 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433382+0000", 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433433+0000", 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0903999999999999e-05 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433554+0000", 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012107099999999999 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.433661+0000", 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010699 2024-08-17T17:44:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.436511+0000", 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028494309999999999 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.436665+0000", 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015454999999999999 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.897905+0000", 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.46124029100000002 2024-08-17T17:44:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.897963+0000", 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7336000000000001e-05 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:54.901255+0000", 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00329168 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:135735 12.f 12:ff30985f:::benchmark_data_smithi142_261410_object17538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 485.33614303600001, 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35154203099999998, 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 135735 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432764+0000", 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432762+0000", 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432767+0000", 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.262e-06 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432776+0000", 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7870000000000007e-06 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.432779+0000", 2024-08-17T17:44:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8859999999999997e-06 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436659+0000", 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003879263 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436779+0000", 2024-08-17T17:44:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119862 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.436904+0000", 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012594099999999999 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.439927+0000", 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030223350000000001 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.440109+0000", 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018190799999999999 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.781461+0000", 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34135219100000003 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.781503+0000", 2024-08-17T17:44:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1789000000000003e-05 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:35:58.784306+0000", 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028029550000000002 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:139285 12.f 12:fc2a058b:::benchmark_data_smithi142_261410_object21088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:36:42.471011+0000", 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 441.29789562500002, 2024-08-17T17:44:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37445746299999999, 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 139285 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471011+0000", 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471011+0000", 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471010+0000", 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471016+0000", 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3559999999999998e-06 2024-08-17T17:44:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471028+0000", 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2196e-05 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471033+0000", 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7040000000000002e-06 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471098+0000", 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4480000000000006e-05 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471275+0000", 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017774300000000001 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.471459+0000", 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018332600000000001 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.474531+0000", 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003072342 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.474709+0000", 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017838500000000001 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.842685+0000", 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36797549899999998 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.842733+0000", 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8000999999999997e-05 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:36:42.845468+0000", 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027356500000000001 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:142271 12.f 12:fb14d2d7:::benchmark_data_smithi142_261410_object24074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:15.842039+0000", 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 407.92686785299998, 2024-08-17T17:44:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.58777459499999996, 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 142271 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842039+0000", 2024-08-17T17:44:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842039+0000", 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842037+0000", 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842046+0000", 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0520000000000007e-06 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842062+0000", 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5586999999999998e-05 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842068+0000", 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.241e-06 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842120+0000", 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2330000000000002e-05 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842310+0000", 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018918799999999999 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.842435+0000", 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012570499999999999 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.845440+0000", 2024-08-17T17:44:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030047049999999999 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:15.846107+0000", 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066691400000000001 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.426983+0000", 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.58087590600000005 2024-08-17T17:44:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.427023+0000", 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0154999999999998e-05 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.429813+0000", 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027903659999999999 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:142273 12.f 12:ffe89de1:::benchmark_data_smithi142_261410_object24076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:16.012532+0000", 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 407.75637426999998, 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.42049373800000001, 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 142273 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012532+0000", 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012532+0000", 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012531+0000", 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012537+0000", 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9869999999999996e-06 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012566+0000", 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8473999999999999e-05 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012571+0000", 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9200000000000001e-06 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012628+0000", 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6849999999999999e-05 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.012810+0000", 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018125500000000001 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.013010+0000", 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020028099999999999 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.016348+0000", 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033381299999999999 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.016420+0000", 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1633999999999999e-05 2024-08-17T17:44:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.429875+0000", 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41345523499999998 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.429915+0000", 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9950000000000002e-05 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:16.433026+0000", 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003111309 2024-08-17T17:44:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:144433 12.f 12:f489411b:::benchmark_data_smithi142_261410_object26236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e363)", 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:39.903296+0000", 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 383.865610893, 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.357136756, 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144433 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903296+0000", 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903296+0000", 2024-08-17T17:44:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903294+0000", 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903320+0000", 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5673e-05 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903337+0000", 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7578999999999999e-05 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903361+0000", 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4402000000000001e-05 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903436+0000", 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4136999999999998e-05 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903559+0000", 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123647 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.903669+0000", 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109633 2024-08-17T17:44:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.906501+0000", 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00283169 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:39.906652+0000", 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000151108 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:40.257021+0000", 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.350369667 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:40.257060+0000", 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9096e-05 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:40.260433+0000", 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033720550000000001 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:144895 12.f 12:f9c382c1:::benchmark_data_smithi142_261410_object26698:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:37:45.567171+0000", 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 378.20173561799999, 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.37522107199999999, 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 144895 2024-08-17T17:44:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567171+0000", 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567171+0000", 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567170+0000", 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567177+0000", 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2169999999999997e-06 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567192+0000", 2024-08-17T17:44:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5469e-05 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567199+0000", 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7020000000000004e-06 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567238+0000", 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8900999999999999e-05 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567356+0000", 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011850599999999999 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.567461+0000", 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010467999999999999 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.570434+0000", 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029725390000000002 2024-08-17T17:44:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.570526+0000", 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2232999999999996e-05 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.938453+0000", 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36792676899999999 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.938517+0000", 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4128000000000007e-05 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:37:45.942392+0000", 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003875351 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:146628 12.f 12:fe6860be:::benchmark_data_smithi142_261410_object28431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:02.033397+0000", 2024-08-17T17:44:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.73550971100002, 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36051549100000002, 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 146628 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033397+0000", 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033397+0000", 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033394+0000", 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999971 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033407+0000", 2024-08-17T17:44:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2415999999999999e-05 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033428+0000", 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1021000000000001e-05 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033437+0000", 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0990000000000005e-06 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033492+0000", 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5220000000000003e-05 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033612+0000", 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011965500000000001 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.033715+0000", 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103764 2024-08-17T17:44:03.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.036595+0000", 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028794129999999999 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.036700+0000", 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105485 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.390972+0000", 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35427146399999998 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.391013+0000", 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1621e-05 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.393912+0000", 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028990109999999999 2024-08-17T17:44:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:146630 12.f 12:fe52152e:::benchmark_data_smithi142_261410_object28433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:38:02.046096+0000", 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.72281058700003, 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35065413000000001, 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.870 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 146630 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046096+0000", 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046096+0000", 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046094+0000", 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046105+0000", 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1138000000000001e-05 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046141+0000", 2024-08-17T17:44:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5884999999999999e-05 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046166+0000", 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4876e-05 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046218+0000", 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2132000000000002e-05 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046344+0000", 2024-08-17T17:44:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000126239 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.046452+0000", 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108155 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.049292+0000", 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028396860000000001 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.049329+0000", 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6860000000000003e-05 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.393971+0000", 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34464169700000002 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.394029+0000", 2024-08-17T17:44:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7855000000000001e-05 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:38:02.396750+0000", 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027216889999999998 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:155660 12.f 12:fdb35f6e:::benchmark_data_smithi142_261410_object37463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:39:31.626313+0000", 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 272.14259327299999, 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.42314358899999999, 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155660 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626313+0000", 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626313+0000", 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626312+0000", 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626320+0000", 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2519999999999999e-06 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626337+0000", 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6727000000000001e-05 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626343+0000", 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5590000000000002e-06 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626387+0000", 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3513999999999997e-05 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626506+0000", 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118824 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.626618+0000", 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111768 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.629419+0000", 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028014849999999998 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:31.629565+0000", 2024-08-17T17:44:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014616999999999999 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:32.046449+0000", 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.41688390600000003 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:32.046510+0000", 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1415999999999996e-05 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:39:32.049457+0000", 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029464840000000001 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:165832 12.f 12:f9f5aeac:::benchmark_data_smithi142_261410_object47635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:03.723175+0000", 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 180.04573117699999, 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35249307400000002, 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165832 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723175+0000", 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723175+0000", 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723174+0000", 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723180+0000", 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2230000000000004e-06 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723193+0000", 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2278e-05 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723198+0000", 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2079999999999999e-06 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723235+0000", 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6912999999999997e-05 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723354+0000", 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011915499999999999 2024-08-17T17:44:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.723459+0000", 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104681 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.726254+0000", 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027948930000000001 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:03.726419+0000", 2024-08-17T17:44:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016569499999999999 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.072089+0000", 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34567017999999999 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.072152+0000", 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2698000000000002e-05 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:04.075669+0000", 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035164739999999999 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:167333 12.f 12:f036c062:::benchmark_data_smithi142_261410_object49136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e383)", 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:41:18.525405+0000", 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 165.243502132, 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.374992718, 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 167333 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525405+0000", 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525405+0000", 2024-08-17T17:44:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525403+0000", 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525412+0000", 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2499999999999995e-06 2024-08-17T17:44:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525434+0000", 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1280999999999998e-05 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525442+0000", 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1289999999999996e-06 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525483+0000", 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1233999999999999e-05 2024-08-17T17:44:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525648+0000", 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000165076 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.525761+0000", 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011281 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.528578+0000", 2024-08-17T17:44:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028169430000000001 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.528795+0000", 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021709499999999999 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.897555+0000", 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36876039900000002 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.897596+0000", 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0664000000000002e-05 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:41:18.900397+0000", 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028013920000000002 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:184350 12.f 12:f7edee66:::benchmark_data_smithi142_261410_object66153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2024-08-17T17:44:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:19.776423+0000", 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 43.992484046000001, 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36715397999999999, 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184350 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776423+0000", 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776423+0000", 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776421+0000", 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776428+0000", 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3799999999999999e-06 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776443+0000", 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5048000000000001e-05 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776449+0000", 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2840000000000003e-06 2024-08-17T17:44:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776492+0000", 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3612000000000002e-05 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776662+0000", 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000169462 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.776841+0000", 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017893499999999999 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.779745+0000", 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029036750000000001 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:19.779910+0000", 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000165231 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:20.140933+0000", 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36102285299999998 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:20.140973+0000", 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9956999999999997e-05 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:20.143577+0000", 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026039919999999999 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:188243 12.f 12:f4678fc6:::benchmark_data_smithi142_261410_object70046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:43:43.005544+0000", 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 20.763362183000002, 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24737192699999999, 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:44:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 188243 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005544+0000", 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005544+0000", 2024-08-17T17:44:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005542+0000", 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005553+0000", 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0336000000000001e-05 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:44:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005570+0000", 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7626999999999999e-05 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005577+0000", 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1380000000000001e-06 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005624+0000", 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6258999999999999e-05 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005755+0000", 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013175099999999999 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.005913+0000", 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000157088 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.009637+0000", 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037243609999999998 2024-08-17T17:44:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.009848+0000", 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021156199999999999 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.249272+0000", 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.239424041 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.249334+0000", 2024-08-17T17:44:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1223999999999995e-05 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:43:43.252916+0000", 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035827150000000002 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:44:03.909 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:44:03.910 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:03 smithi142 ceph-mon[106729]: pgmap v1541: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50018/135447 objects degraded (36.928%); 18512/135447 objects misplaced (13.667%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:03.910 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:03 smithi142 ceph-mon[103614]: pgmap v1541: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 171 op/s; 50018/135447 objects degraded (36.928%); 18512/135447 objects misplaced (13.667%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:03.918 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1541,"stamp":"2024-08-17T17:44:02.386228+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2957711333,"num_objects":45149,"num_object_clones":0,"num_object_copies":135447,"num_objects_missing_on_primary":3458,"num_objects_missing":3458,"num_objects_degraded":50018,"num_objects_misplaced":18512,"num_objects_unfound":0,"num_objects_dirty":45149,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":191545,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33895,"num_bytes_recovered":2098471566,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104793,"ondisk_log_size":104793,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12027592,"kb_used_data":6174100,"kb_used_omap":27,"kb_used_meta":5853412,"kb_avail":550443320,"statfs":{"total":575970213888,"available":563653959680,"internally_reserved":0,"allocated":6322278400,"data_stored":6309226178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5993894632},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-112590848,"num_objects":-1718,"num_object_clones":0,"num_object_copies":-5154,"num_objects_missing_on_primary":-112,"num_objects_missing":-112,"num_objects_degraded":-228,"num_objects_misplaced":-663,"num_objects_unfound":0,"num_objects_dirty":-1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1718,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":349,"num_bytes_recovered":22872064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003150"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":636,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461031+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:43:51.461031+0000","last_peered":"2024-08-17T17:43:51.461031+0000","last_clean":"2024-08-17T17:43:51.461031+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:43:51.461031+0000","last_undegraded":"2024-08-17T17:43:51.461031+0000","last_fullsized":"2024-08-17T17:43:51.461031+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":105,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521417+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:43:38.521417+0000","last_peered":"2024-08-17T17:43:38.521417+0000","last_clean":"2024-08-17T17:43:38.521417+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:43:38.521417+0000","last_undegraded":"2024-08-17T17:43:38.521417+0000","last_fullsized":"2024-08-17T17:43:38.521417+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":299,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.524140+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:43:38.524140+0000","last_peered":"2024-08-17T17:43:38.524140+0000","last_clean":"2024-08-17T17:43:38.524140+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:43:38.524140+0000","last_undegraded":"2024-08-17T17:43:38.524140+0000","last_fullsized":"2024-08-17T17:43:38.524140+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":334,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460561+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:43:51.460561+0000","last_peered":"2024-08-17T17:43:51.460561+0000","last_clean":"2024-08-17T17:43:51.460561+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:43:51.460561+0000","last_undegraded":"2024-08-17T17:43:51.460561+0000","last_fullsized":"2024-08-17T17:43:51.460561+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":511,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460484+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:43:51.460484+0000","last_peered":"2024-08-17T17:43:51.460484+0000","last_clean":"2024-08-17T17:43:51.460484+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:43:51.460484+0000","last_undegraded":"2024-08-17T17:43:51.460484+0000","last_fullsized":"2024-08-17T17:43:51.460484+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":248,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521313+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:43:38.521313+0000","last_peered":"2024-08-17T17:43:38.521313+0000","last_clean":"2024-08-17T17:43:38.521313+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:43:38.521313+0000","last_undegraded":"2024-08-17T17:43:38.521313+0000","last_fullsized":"2024-08-17T17:43:38.521313+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":604,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460233+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:43:51.460233+0000","last_peered":"2024-08-17T17:43:51.460233+0000","last_clean":"2024-08-17T17:43:51.460233+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:43:51.460233+0000","last_undegraded":"2024-08-17T17:43:51.460233+0000","last_fullsized":"2024-08-17T17:43:51.460233+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":417,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467446+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:43:51.467446+0000","last_peered":"2024-08-17T17:43:51.467446+0000","last_clean":"2024-08-17T17:43:51.467446+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:43:51.467446+0000","last_undegraded":"2024-08-17T17:43:51.467446+0000","last_fullsized":"2024-08-17T17:43:51.467446+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"396'11906","reported_seq":19666,"reported_epoch":396,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:44:00.348720+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:44:00.348720+0000","last_peered":"2024-08-17T17:44:00.348720+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:44:00.348720+0000","last_undegraded":"2024-08-17T17:44:00.348720+0000","last_fullsized":"2024-08-17T17:44:00.348720+0000","mapping_epoch":337,"log_start":"327'5063","ondisk_log_start":"327'5063","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183369728,"num_objects":2798,"num_object_clones":0,"num_object_copies":8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2797,"num_objects_unfound":0,"num_objects_dirty":2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11906,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3826,"num_bytes_recovered":250740736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2797},{"shards":"4,6,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":101,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.570720+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:43:38.570720+0000","last_peered":"2024-08-17T17:43:38.570720+0000","last_clean":"2024-08-17T17:43:38.570720+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:43:38.570720+0000","last_undegraded":"2024-08-17T17:43:38.570720+0000","last_fullsized":"2024-08-17T17:43:38.570720+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":282,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.523273+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:43:38.523273+0000","last_peered":"2024-08-17T17:43:38.523273+0000","last_clean":"2024-08-17T17:43:38.523273+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:43:38.523273+0000","last_undegraded":"2024-08-17T17:43:38.523273+0000","last_fullsized":"2024-08-17T17:43:38.523273+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"397'12120","reported_seq":12215,"reported_epoch":397,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:02.084899+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:44:02.084899+0000","last_peered":"2024-08-17T17:44:02.084899+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:44:02.084899+0000","last_undegraded":"2024-08-17T17:44:02.084899+0000","last_fullsized":"2024-08-17T17:44:02.084899+0000","mapping_epoch":337,"log_start":"327'5582","ondisk_log_start":"327'5582","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193593344,"num_objects":2954,"num_object_clones":0,"num_object_copies":8862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2954,"num_objects_unfound":0,"num_objects_dirty":2954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12120,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2954}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":103,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.523514+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:43:38.523514+0000","last_peered":"2024-08-17T17:43:38.523514+0000","last_clean":"2024-08-17T17:43:38.523514+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:43:38.523514+0000","last_undegraded":"2024-08-17T17:43:38.523514+0000","last_fullsized":"2024-08-17T17:43:38.523514+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":367,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521220+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:43:38.521220+0000","last_peered":"2024-08-17T17:43:38.521220+0000","last_clean":"2024-08-17T17:43:38.521220+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:43:38.521220+0000","last_undegraded":"2024-08-17T17:43:38.521220+0000","last_fullsized":"2024-08-17T17:43:38.521220+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"396'11908","reported_seq":12005,"reported_epoch":396,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:00.324299+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:44:00.324299+0000","last_peered":"2024-08-17T17:44:00.324299+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:44:00.324299+0000","last_undegraded":"2024-08-17T17:44:00.324299+0000","last_fullsized":"2024-08-17T17:44:00.324299+0000","mapping_epoch":337,"log_start":"327'5053","ondisk_log_start":"327'5053","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6855,"log_dups_size":0,"ondisk_log_size":6855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185204736,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2826,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11908,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2826}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":99,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.524076+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:43:38.524076+0000","last_peered":"2024-08-17T17:43:38.524076+0000","last_clean":"2024-08-17T17:43:38.524076+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:43:38.524076+0000","last_undegraded":"2024-08-17T17:43:38.524076+0000","last_fullsized":"2024-08-17T17:43:38.524076+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":378,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467399+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:43:51.467399+0000","last_peered":"2024-08-17T17:43:51.467399+0000","last_clean":"2024-08-17T17:43:51.467399+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:43:51.467399+0000","last_undegraded":"2024-08-17T17:43:51.467399+0000","last_fullsized":"2024-08-17T17:43:51.467399+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"396'12106","reported_seq":12279,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:59.601606+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:43:59.601606+0000","last_peered":"2024-08-17T17:43:59.601606+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:43:59.601606+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'5113","ondisk_log_start":"327'5113","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186908672,"num_objects":2852,"num_object_clones":0,"num_object_copies":8556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12106,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-3917},{"shards":"0,3,7","objects":2852}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":548,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461207+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:43:51.461207+0000","last_peered":"2024-08-17T17:43:51.461207+0000","last_clean":"2024-08-17T17:43:51.461207+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:43:51.461207+0000","last_undegraded":"2024-08-17T17:43:51.461207+0000","last_fullsized":"2024-08-17T17:43:51.461207+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":314,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.470633+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:43:51.470633+0000","last_peered":"2024-08-17T17:43:51.470633+0000","last_clean":"2024-08-17T17:43:51.470633+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:43:51.470633+0000","last_undegraded":"2024-08-17T17:43:51.470633+0000","last_fullsized":"2024-08-17T17:43:51.470633+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"397'11802","reported_seq":11888,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.938198+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:44:01.938198+0000","last_peered":"2024-08-17T17:44:01.938198+0000","last_clean":"2024-08-17T17:44:01.938198+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:44:01.938198+0000","last_undegraded":"2024-08-17T17:44:01.938198+0000","last_fullsized":"2024-08-17T17:44:01.938198+0000","mapping_epoch":319,"log_start":"327'5274","ondisk_log_start":"327'5274","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182976512,"num_objects":2792,"num_object_clones":0,"num_object_copies":8376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11802,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":637,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461017+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:43:51.461017+0000","last_peered":"2024-08-17T17:43:51.461017+0000","last_clean":"2024-08-17T17:43:51.461017+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:43:51.461017+0000","last_undegraded":"2024-08-17T17:43:51.461017+0000","last_fullsized":"2024-08-17T17:43:51.461017+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":350,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521280+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:43:38.521280+0000","last_peered":"2024-08-17T17:43:38.521280+0000","last_clean":"2024-08-17T17:43:38.521280+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:43:38.521280+0000","last_undegraded":"2024-08-17T17:43:38.521280+0000","last_fullsized":"2024-08-17T17:43:38.521280+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"397'12055","reported_seq":12151,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:02.166991+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:44:02.166991+0000","last_peered":"2024-08-17T17:44:02.166991+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:44:02.166991+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'5435","ondisk_log_start":"327'5435","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188547072,"num_objects":2877,"num_object_clones":0,"num_object_copies":8631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12055,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2877},{"shards":"1,6","objects":-4489}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":352,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467568+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:43:51.467568+0000","last_peered":"2024-08-17T17:43:51.467568+0000","last_clean":"2024-08-17T17:43:51.467568+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:43:51.467568+0000","last_undegraded":"2024-08-17T17:43:51.467568+0000","last_fullsized":"2024-08-17T17:43:51.467568+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":377,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.522165+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:43:38.522165+0000","last_peered":"2024-08-17T17:43:38.522165+0000","last_clean":"2024-08-17T17:43:38.522165+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:43:38.522165+0000","last_undegraded":"2024-08-17T17:43:38.522165+0000","last_fullsized":"2024-08-17T17:43:38.522165+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"396'12113","reported_seq":12287,"reported_epoch":396,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:43:58.951640+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:43:58.951640+0000","last_peered":"2024-08-17T17:43:58.951640+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:43:58.951640+0000","last_undegraded":"2024-08-17T17:43:58.951640+0000","last_fullsized":"2024-08-17T17:43:58.951640+0000","mapping_epoch":337,"log_start":"328'6461","ondisk_log_start":"328'6461","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5652,"log_dups_size":0,"ondisk_log_size":5652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189857792,"num_objects":2897,"num_object_clones":0,"num_object_copies":8691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2897,"num_objects_unfound":0,"num_objects_dirty":2897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12113,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":2897}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":388,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521400+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:43:38.521400+0000","last_peered":"2024-08-17T17:43:38.521400+0000","last_clean":"2024-08-17T17:43:38.521400+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:43:38.521400+0000","last_undegraded":"2024-08-17T17:43:38.521400+0000","last_fullsized":"2024-08-17T17:43:38.521400+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":350,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.523632+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:43:38.523632+0000","last_peered":"2024-08-17T17:43:38.523632+0000","last_clean":"2024-08-17T17:43:38.523632+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:43:38.523632+0000","last_undegraded":"2024-08-17T17:43:38.523632+0000","last_fullsized":"2024-08-17T17:43:38.523632+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"396'11938","reported_seq":19815,"reported_epoch":396,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:00.284608+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:44:00.284608+0000","last_peered":"2024-08-17T17:44:00.284608+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:44:00.284608+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:44:00.284608+0000","mapping_epoch":337,"log_start":"327'5041","ondisk_log_start":"327'5041","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181272576,"num_objects":2766,"num_object_clones":0,"num_object_copies":8298,"num_objects_missing_on_primary":3458,"num_objects_missing":3458,"num_objects_degraded":2766,"num_objects_misplaced":3458,"num_objects_unfound":0,"num_objects_dirty":2766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11938,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3881,"num_bytes_recovered":254279680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":2752},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":607,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.462480+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:43:51.462480+0000","last_peered":"2024-08-17T17:43:51.462480+0000","last_clean":"2024-08-17T17:43:51.462480+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:43:51.462480+0000","last_undegraded":"2024-08-17T17:43:51.462480+0000","last_fullsized":"2024-08-17T17:43:51.462480+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":544,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460330+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:43:51.460330+0000","last_peered":"2024-08-17T17:43:51.460330+0000","last_clean":"2024-08-17T17:43:51.460330+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:43:51.460330+0000","last_undegraded":"2024-08-17T17:43:51.460330+0000","last_fullsized":"2024-08-17T17:43:51.460330+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"396'12066","reported_seq":19965,"reported_epoch":396,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:43:58.841307+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:43:58.841307+0000","last_peered":"2024-08-17T17:43:58.841307+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:43:58.841307+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:43:58.841307+0000","mapping_epoch":337,"log_start":"328'6444","ondisk_log_start":"328'6444","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5622,"log_dups_size":0,"ondisk_log_size":5622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185401368,"num_objects":2830,"num_object_clones":0,"num_object_copies":8490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2830,"num_objects_misplaced":3580,"num_objects_unfound":0,"num_objects_dirty":2830,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12066,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3868,"num_bytes_recovered":253427712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2830}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":738,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:51.469006+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:43:51.469006+0000","last_peered":"2024-08-17T17:43:51.469006+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:43:51.469006+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":325,"reported_epoch":395,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:43:38.523846+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:43:38.523846+0000","last_peered":"2024-08-17T17:43:38.523846+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:43:38.523846+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:43:38.523846+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":587,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460514+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:43:51.460514+0000","last_peered":"2024-08-17T17:43:51.460514+0000","last_clean":"2024-08-17T17:43:51.460514+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:43:51.460514+0000","last_undegraded":"2024-08-17T17:43:51.460514+0000","last_fullsized":"2024-08-17T17:43:51.460514+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":361,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.468013+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:43:51.468013+0000","last_peered":"2024-08-17T17:43:51.468013+0000","last_clean":"2024-08-17T17:43:51.468013+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:43:51.468013+0000","last_undegraded":"2024-08-17T17:43:51.468013+0000","last_fullsized":"2024-08-17T17:43:51.468013+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"396'11900","reported_seq":12076,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:59.571580+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:43:59.571580+0000","last_peered":"2024-08-17T17:43:59.571580+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:43:59.571580+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'4921","ondisk_log_start":"327'4921","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11900,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-3803},{"shards":"0,4,7","objects":2844}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":403,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461704+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:43:51.461704+0000","last_peered":"2024-08-17T17:43:51.461704+0000","last_clean":"2024-08-17T17:43:51.461704+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:43:51.461704+0000","last_undegraded":"2024-08-17T17:43:51.461704+0000","last_fullsized":"2024-08-17T17:43:51.461704+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":332,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521147+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:43:38.521147+0000","last_peered":"2024-08-17T17:43:38.521147+0000","last_clean":"2024-08-17T17:43:38.521147+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:43:38.521147+0000","last_undegraded":"2024-08-17T17:43:38.521147+0000","last_fullsized":"2024-08-17T17:43:38.521147+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":328,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.471513+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:43:51.471513+0000","last_peered":"2024-08-17T17:43:51.471513+0000","last_clean":"2024-08-17T17:43:51.471513+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:43:51.471513+0000","last_undegraded":"2024-08-17T17:43:51.471513+0000","last_fullsized":"2024-08-17T17:43:51.471513+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"397'12051","reported_seq":12146,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:02.233160+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:44:02.233160+0000","last_peered":"2024-08-17T17:44:02.233160+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:44:02.233160+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"327'5517","ondisk_log_start":"327'5517","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179896320,"num_objects":2745,"num_object_clones":0,"num_object_copies":8235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12051,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2745},{"shards":"5,6","objects":-4551}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":495,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.462033+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:43:51.462033+0000","last_peered":"2024-08-17T17:43:51.462033+0000","last_clean":"2024-08-17T17:43:51.462033+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:43:51.462033+0000","last_undegraded":"2024-08-17T17:43:51.462033+0000","last_fullsized":"2024-08-17T17:43:51.462033+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467342+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:43:51.467342+0000","last_peered":"2024-08-17T17:43:51.467342+0000","last_clean":"2024-08-17T17:43:51.467342+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:43:51.467342+0000","last_undegraded":"2024-08-17T17:43:51.467342+0000","last_fullsized":"2024-08-17T17:43:51.467342+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":679,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460376+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:43:51.460376+0000","last_peered":"2024-08-17T17:43:51.460376+0000","last_clean":"2024-08-17T17:43:51.460376+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:43:51.460376+0000","last_undegraded":"2024-08-17T17:43:51.460376+0000","last_fullsized":"2024-08-17T17:43:51.460376+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"396'11864","reported_seq":12764,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:59.711623+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:43:59.711623+0000","last_peered":"2024-08-17T17:43:59.711623+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:43:59.711623+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"327'4816","ondisk_log_start":"327'4816","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182321152,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11864,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2782},{"shards":"0,5","objects":-3872}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":471,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.458591+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:43:51.458591+0000","last_peered":"2024-08-17T17:43:51.458591+0000","last_clean":"2024-08-17T17:43:51.458591+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:43:51.458591+0000","last_undegraded":"2024-08-17T17:43:51.458591+0000","last_fullsized":"2024-08-17T17:43:51.458591+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":371,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521790+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:43:38.521790+0000","last_peered":"2024-08-17T17:43:38.521790+0000","last_clean":"2024-08-17T17:43:38.521790+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:43:38.521790+0000","last_undegraded":"2024-08-17T17:43:38.521790+0000","last_fullsized":"2024-08-17T17:43:38.521790+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":266,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460931+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:43:51.460931+0000","last_peered":"2024-08-17T17:43:51.460931+0000","last_clean":"2024-08-17T17:43:51.460931+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:43:51.460931+0000","last_undegraded":"2024-08-17T17:43:51.460931+0000","last_fullsized":"2024-08-17T17:43:51.460931+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"396'12005","reported_seq":27024,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:57.706966+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:43:57.706966+0000","last_peered":"2024-08-17T17:43:57.706966+0000","last_clean":"2024-08-17T17:43:57.706966+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:43:57.706966+0000","last_undegraded":"2024-08-17T17:43:57.706966+0000","last_fullsized":"2024-08-17T17:43:57.706966+0000","mapping_epoch":355,"log_start":"327'5730","ondisk_log_start":"327'5730","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185532416,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12005,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":310,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.524000+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:43:38.524000+0000","last_peered":"2024-08-17T17:43:38.524000+0000","last_clean":"2024-08-17T17:43:38.524000+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:43:38.524000+0000","last_undegraded":"2024-08-17T17:43:38.524000+0000","last_fullsized":"2024-08-17T17:43:38.524000+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460841+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:43:51.460841+0000","last_peered":"2024-08-17T17:43:51.460841+0000","last_clean":"2024-08-17T17:43:51.460841+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:43:51.460841+0000","last_undegraded":"2024-08-17T17:43:51.460841+0000","last_fullsized":"2024-08-17T17:43:51.460841+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":630,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460756+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:43:51.460756+0000","last_peered":"2024-08-17T17:43:51.460756+0000","last_clean":"2024-08-17T17:43:51.460756+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:43:51.460756+0000","last_undegraded":"2024-08-17T17:43:51.460756+0000","last_fullsized":"2024-08-17T17:43:51.460756+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"396'11695","reported_seq":26282,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:57.810068+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:43:57.810068+0000","last_peered":"2024-08-17T17:43:57.810068+0000","last_clean":"2024-08-17T17:43:57.810068+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:43:57.810068+0000","last_undegraded":"2024-08-17T17:43:57.810068+0000","last_fullsized":"2024-08-17T17:43:57.810068+0000","mapping_epoch":345,"log_start":"327'5372","ondisk_log_start":"327'5372","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6323,"log_dups_size":0,"ondisk_log_size":6323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177537024,"num_objects":2709,"num_object_clones":0,"num_object_copies":8127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11695,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":306,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.524159+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:43:38.524159+0000","last_peered":"2024-08-17T17:43:38.524159+0000","last_clean":"2024-08-17T17:43:38.524159+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:43:38.524159+0000","last_undegraded":"2024-08-17T17:43:38.524159+0000","last_fullsized":"2024-08-17T17:43:38.524159+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":385,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.469314+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:43:51.469314+0000","last_peered":"2024-08-17T17:43:51.469314+0000","last_clean":"2024-08-17T17:43:51.469314+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:43:51.469314+0000","last_undegraded":"2024-08-17T17:43:51.469314+0000","last_fullsized":"2024-08-17T17:43:51.469314+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":448,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.523753+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:43:38.523753+0000","last_peered":"2024-08-17T17:43:38.523753+0000","last_clean":"2024-08-17T17:43:38.523753+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:43:38.523753+0000","last_undegraded":"2024-08-17T17:43:38.523753+0000","last_fullsized":"2024-08-17T17:43:38.523753+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"396'11891","reported_seq":12086,"reported_epoch":396,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:43:57.680599+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:43:57.680599+0000","last_peered":"2024-08-17T17:43:57.680599+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:43:57.680599+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'5587","ondisk_log_start":"327'5587","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6304,"log_dups_size":0,"ondisk_log_size":6304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189595648,"num_objects":2893,"num_object_clones":0,"num_object_copies":8679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11891,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":2893},{"shards":"0,3","objects":-4355}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":660,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460427+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:43:51.460427+0000","last_peered":"2024-08-17T17:43:51.460427+0000","last_clean":"2024-08-17T17:43:51.460427+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:43:51.460427+0000","last_undegraded":"2024-08-17T17:43:51.460427+0000","last_fullsized":"2024-08-17T17:43:51.460427+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":628,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460598+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:43:51.460598+0000","last_peered":"2024-08-17T17:43:51.460598+0000","last_clean":"2024-08-17T17:43:51.460598+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:43:51.460598+0000","last_undegraded":"2024-08-17T17:43:51.460598+0000","last_fullsized":"2024-08-17T17:43:51.460598+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":646,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.462343+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:43:51.462343+0000","last_peered":"2024-08-17T17:43:51.462343+0000","last_clean":"2024-08-17T17:43:51.462343+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:43:51.462343+0000","last_undegraded":"2024-08-17T17:43:51.462343+0000","last_fullsized":"2024-08-17T17:43:51.462343+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"397'11847","reported_seq":26541,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:02.079660+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:44:02.079660+0000","last_peered":"2024-08-17T17:44:02.079660+0000","last_clean":"2024-08-17T17:44:02.079660+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:44:02.079660+0000","last_undegraded":"2024-08-17T17:44:02.079660+0000","last_fullsized":"2024-08-17T17:44:02.079660+0000","mapping_epoch":347,"log_start":"327'5272","ondisk_log_start":"327'5272","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6575,"log_dups_size":0,"ondisk_log_size":6575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178716672,"num_objects":2727,"num_object_clones":0,"num_object_copies":8181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11847,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":318,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.471094+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:43:51.471094+0000","last_peered":"2024-08-17T17:43:51.471094+0000","last_clean":"2024-08-17T17:43:51.471094+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:43:51.471094+0000","last_undegraded":"2024-08-17T17:43:51.471094+0000","last_fullsized":"2024-08-17T17:43:51.471094+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":401,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.468184+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:43:51.468184+0000","last_peered":"2024-08-17T17:43:51.468184+0000","last_clean":"2024-08-17T17:43:51.468184+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:43:51.468184+0000","last_undegraded":"2024-08-17T17:43:51.468184+0000","last_fullsized":"2024-08-17T17:43:51.468184+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":350,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521307+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:43:38.521307+0000","last_peered":"2024-08-17T17:43:38.521307+0000","last_clean":"2024-08-17T17:43:38.521307+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:43:38.521307+0000","last_undegraded":"2024-08-17T17:43:38.521307+0000","last_fullsized":"2024-08-17T17:43:38.521307+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":350,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521656+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:43:38.521656+0000","last_peered":"2024-08-17T17:43:38.521656+0000","last_clean":"2024-08-17T17:43:38.521656+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:43:38.521656+0000","last_undegraded":"2024-08-17T17:43:38.521656+0000","last_fullsized":"2024-08-17T17:43:38.521656+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":393,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521987+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:43:38.521987+0000","last_peered":"2024-08-17T17:43:38.521987+0000","last_clean":"2024-08-17T17:43:38.521987+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:43:38.521987+0000","last_undegraded":"2024-08-17T17:43:38.521987+0000","last_fullsized":"2024-08-17T17:43:38.521987+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":635,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.458711+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:43:51.458711+0000","last_peered":"2024-08-17T17:43:51.458711+0000","last_clean":"2024-08-17T17:43:51.458711+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:43:51.458711+0000","last_undegraded":"2024-08-17T17:43:51.458711+0000","last_fullsized":"2024-08-17T17:43:51.458711+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":175,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461422+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:43:51.461422+0000","last_peered":"2024-08-17T17:43:51.461422+0000","last_clean":"2024-08-17T17:43:51.461422+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:43:51.461422+0000","last_undegraded":"2024-08-17T17:43:51.461422+0000","last_fullsized":"2024-08-17T17:43:51.461422+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":300,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.524343+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:43:38.524343+0000","last_peered":"2024-08-17T17:43:38.524343+0000","last_clean":"2024-08-17T17:43:38.524343+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:43:38.524343+0000","last_undegraded":"2024-08-17T17:43:38.524343+0000","last_fullsized":"2024-08-17T17:43:38.524343+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":593,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461552+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:43:51.461552+0000","last_peered":"2024-08-17T17:43:51.461552+0000","last_clean":"2024-08-17T17:43:51.461552+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:43:51.461552+0000","last_undegraded":"2024-08-17T17:43:51.461552+0000","last_fullsized":"2024-08-17T17:43:51.461552+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":444,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.525488+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:43:38.525488+0000","last_peered":"2024-08-17T17:43:38.525488+0000","last_clean":"2024-08-17T17:43:38.525488+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:43:38.525488+0000","last_undegraded":"2024-08-17T17:43:38.525488+0000","last_fullsized":"2024-08-17T17:43:38.525488+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":587,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.462257+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:43:51.462257+0000","last_peered":"2024-08-17T17:43:51.462257+0000","last_clean":"2024-08-17T17:43:51.462257+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:43:51.462257+0000","last_undegraded":"2024-08-17T17:43:51.462257+0000","last_fullsized":"2024-08-17T17:43:51.462257+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":368,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.468120+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:43:51.468120+0000","last_peered":"2024-08-17T17:43:51.468120+0000","last_clean":"2024-08-17T17:43:51.468120+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:43:51.468120+0000","last_undegraded":"2024-08-17T17:43:51.468120+0000","last_fullsized":"2024-08-17T17:43:51.468120+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":618,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.462408+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:43:51.462408+0000","last_peered":"2024-08-17T17:43:51.462408+0000","last_clean":"2024-08-17T17:43:51.462408+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:43:51.462408+0000","last_undegraded":"2024-08-17T17:43:51.462408+0000","last_fullsized":"2024-08-17T17:43:51.462408+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460404+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:43:51.460404+0000","last_peered":"2024-08-17T17:43:51.460404+0000","last_clean":"2024-08-17T17:43:51.460404+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:43:51.460404+0000","last_undegraded":"2024-08-17T17:43:51.460404+0000","last_fullsized":"2024-08-17T17:43:51.460404+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":626,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461309+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:43:51.461309+0000","last_peered":"2024-08-17T17:43:51.461309+0000","last_clean":"2024-08-17T17:43:51.461309+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:43:51.461309+0000","last_undegraded":"2024-08-17T17:43:51.461309+0000","last_fullsized":"2024-08-17T17:43:51.461309+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":586,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461268+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:43:51.461268+0000","last_peered":"2024-08-17T17:43:51.461268+0000","last_clean":"2024-08-17T17:43:51.461268+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:43:51.461268+0000","last_undegraded":"2024-08-17T17:43:51.461268+0000","last_fullsized":"2024-08-17T17:43:51.461268+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":379,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521431+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:43:38.521431+0000","last_peered":"2024-08-17T17:43:38.521431+0000","last_clean":"2024-08-17T17:43:38.521431+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:43:38.521431+0000","last_undegraded":"2024-08-17T17:43:38.521431+0000","last_fullsized":"2024-08-17T17:43:38.521431+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":275,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.461118+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:43:51.461118+0000","last_peered":"2024-08-17T17:43:51.461118+0000","last_clean":"2024-08-17T17:43:51.461118+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:43:51.461118+0000","last_undegraded":"2024-08-17T17:43:51.461118+0000","last_fullsized":"2024-08-17T17:43:51.461118+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":331,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.469644+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:43:51.469644+0000","last_peered":"2024-08-17T17:43:51.469644+0000","last_clean":"2024-08-17T17:43:51.469644+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:43:51.469644+0000","last_undegraded":"2024-08-17T17:43:51.469644+0000","last_fullsized":"2024-08-17T17:43:51.469644+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":387,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521591+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:43:38.521591+0000","last_peered":"2024-08-17T17:43:38.521591+0000","last_clean":"2024-08-17T17:43:38.521591+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:43:38.521591+0000","last_undegraded":"2024-08-17T17:43:38.521591+0000","last_fullsized":"2024-08-17T17:43:38.521591+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":250,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521906+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:43:38.521906+0000","last_peered":"2024-08-17T17:43:38.521906+0000","last_clean":"2024-08-17T17:43:38.521906+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:43:38.521906+0000","last_undegraded":"2024-08-17T17:43:38.521906+0000","last_fullsized":"2024-08-17T17:43:38.521906+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":387,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521271+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:43:38.521271+0000","last_peered":"2024-08-17T17:43:38.521271+0000","last_clean":"2024-08-17T17:43:38.521271+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:43:38.521271+0000","last_undegraded":"2024-08-17T17:43:38.521271+0000","last_fullsized":"2024-08-17T17:43:38.521271+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":250,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.522096+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:43:38.522096+0000","last_peered":"2024-08-17T17:43:38.522096+0000","last_clean":"2024-08-17T17:43:38.522096+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:43:38.522096+0000","last_undegraded":"2024-08-17T17:43:38.522096+0000","last_fullsized":"2024-08-17T17:43:38.522096+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":638,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.462120+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:43:51.462120+0000","last_peered":"2024-08-17T17:43:51.462120+0000","last_clean":"2024-08-17T17:43:51.462120+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:43:51.462120+0000","last_undegraded":"2024-08-17T17:43:51.462120+0000","last_fullsized":"2024-08-17T17:43:51.462120+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":612,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460566+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:43:51.460566+0000","last_peered":"2024-08-17T17:43:51.460566+0000","last_clean":"2024-08-17T17:43:51.460566+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:43:51.460566+0000","last_undegraded":"2024-08-17T17:43:51.460566+0000","last_fullsized":"2024-08-17T17:43:51.460566+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":350,"reported_epoch":395,"state":"active+clean","last_fresh":"2024-08-17T17:43:38.521158+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:43:38.521158+0000","last_peered":"2024-08-17T17:43:38.521158+0000","last_clean":"2024-08-17T17:43:38.521158+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:43:38.521158+0000","last_undegraded":"2024-08-17T17:43:38.521158+0000","last_fullsized":"2024-08-17T17:43:38.521158+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":334,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.469610+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:43:51.469610+0000","last_peered":"2024-08-17T17:43:51.469610+0000","last_clean":"2024-08-17T17:43:51.469610+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:43:51.469610+0000","last_undegraded":"2024-08-17T17:43:51.469610+0000","last_fullsized":"2024-08-17T17:43:51.469610+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2957115416,"num_objects":45123,"num_object_clones":0,"num_object_copies":135369,"num_objects_missing_on_primary":3458,"num_objects_missing":3458,"num_objects_degraded":50015,"num_objects_misplaced":18512,"num_objects_unfound":0,"num_objects_dirty":45123,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191267,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33854,"num_bytes_recovered":2096103424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11435716608,"data_stored":11435704392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104586,"ondisk_log_size":104586,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170131,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463501,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2161772,"kb_used_data":1259744,"kb_used_omap":1,"kb_used_meta":902014,"kb_avail":91583380,"statfs":{"total":95995035648,"available":93781381120,"internally_reserved":0,"allocated":1289977856,"data_stored":1287783796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923662680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492175,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":755836,"kb_used_data":360500,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":92989316,"statfs":{"total":95995035648,"available":95221059584,"internally_reserved":0,"allocated":369152000,"data_stored":366988033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986766,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2230540,"kb_used_data":721860,"kb_used_omap":1,"kb_used_meta":1508670,"kb_avail":91514612,"statfs":{"total":95995035648,"available":93710962688,"internally_reserved":0,"allocated":739184640,"data_stored":737009313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1544878424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225594,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2419260,"kb_used_data":1459124,"kb_used_omap":4,"kb_used_meta":960123,"kb_avail":91325892,"statfs":{"total":95995035648,"available":93517713408,"internally_reserved":0,"allocated":1494142976,"data_stored":1491979307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":983166400},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075919,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533812,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2337984,"kb_used_data":1273784,"kb_used_omap":5,"kb_used_meta":1064186,"kb_avail":91407168,"statfs":{"total":95995035648,"available":93600940032,"internally_reserved":0,"allocated":1304354816,"data_stored":1302185825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1089726704},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089789,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2122200,"kb_used_data":1099088,"kb_used_omap":5,"kb_used_meta":1023098,"kb_avail":91622952,"statfs":{"total":95995035648,"available":93821902848,"internally_reserved":0,"allocated":1125466112,"data_stored":1123279904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1047652411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1117913088,"data_stored":1117913088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1298272256,"data_stored":1298268184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1488781312,"data_stored":1488781312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":732364800,"data_stored":732364800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":363855872,"data_stored":363855872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1280901120,"data_stored":1280901120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1814368256,"data_stored":1814364184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:44:03.933 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:03.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:04.035 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:04.058 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:04.160 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:04.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:04.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:04.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:04.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:44:04.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:04.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:04.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:04.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:04 smithi152 ceph-mon[93456]: from='client.47741 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:04.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:04.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:04 smithi142 ceph-mon[106729]: from='client.47741 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:04 smithi142 ceph-mon[103614]: from='client.47741 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:04.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:44:04.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:04.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:04.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:05.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:05.093 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:05.189 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:05.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:05.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:05.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:05.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:05.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:05.477 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:05.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:05 smithi152 ceph-mon[93456]: pgmap v1542: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 49966/134697 objects degraded (37.095%); 18400/134697 objects misplaced (13.660%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:05.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:05.595 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:44:05.596 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:05.596 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:05 smithi142 ceph-mon[103614]: pgmap v1542: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 49966/134697 objects degraded (37.095%); 18400/134697 objects misplaced (13.660%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:05.596 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:05 smithi142 ceph-mon[106729]: pgmap v1542: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 49966/134697 objects degraded (37.095%); 18400/134697 objects misplaced (13.660%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:05.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:05.614 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:44:05.614 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:05.630 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:05.632 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:44:05.633 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:05.651 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:44:05.651 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:05.667 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:44:05.667 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:05.682 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:44:05.683 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:05.697 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:44:05.698 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:05.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:05.713 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:44:05.713 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:05.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:05.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:05.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:05.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:05.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:05.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:06.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.098 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:06.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:06.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:06.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:06.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:06.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:06.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:06.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:06.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:06.603 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:06.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:06.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:06.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/601041041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:06.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49966/134697 objects degraded (37.095%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:06.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:06 smithi142 ceph-mon[106729]: pgmap v1543: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 49912/133659 objects degraded (37.343%); 18225/133659 objects misplaced (13.635%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:06.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/601041041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:06.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49966/134697 objects degraded (37.095%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:06.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:06 smithi142 ceph-mon[103614]: pgmap v1543: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 49912/133659 objects degraded (37.343%); 18225/133659 objects misplaced (13.635%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/601041041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49966/134697 objects degraded (37.095%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:06 smithi152 ceph-mon[93456]: pgmap v1543: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 49912/133659 objects degraded (37.343%); 18225/133659 objects misplaced (13.635%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:06.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:06.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:06.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:44:06.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:06.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:06.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:07.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:07.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:07.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:07.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:07.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:07.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:07.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:07.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:07.484 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:07.583 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:07.583 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:07.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:07.613 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:07.677 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:07.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:07.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:07.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:07.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:07.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:07.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:07.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:08.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:08.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:08.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:08.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:08.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:08.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:08.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:08.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:08.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:08.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:08.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:08.609 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:08.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:44:08.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:08.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:08.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:44:08.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:08.864 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:08.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:08.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:08.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:09.085 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:09.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:09.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:09.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:09.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:09.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:09.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:09.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:09.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:09.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:09.530 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:09 smithi152 ceph-mon[93456]: pgmap v1544: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 49912/132039 objects degraded (37.801%); 18136/132039 objects misplaced (13.735%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:09.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:09.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:09.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:09.752 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:09.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:09 smithi142 ceph-mon[106729]: pgmap v1544: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 49912/132039 objects degraded (37.801%); 18136/132039 objects misplaced (13.735%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:09 smithi142 ceph-mon[103614]: pgmap v1544: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 175 op/s; 49912/132039 objects degraded (37.801%); 18136/132039 objects misplaced (13.735%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:09.879 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:09.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:09.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:10.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:10.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:10.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:10.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:10.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:10.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:10.322 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:44:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:44:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:10.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:10.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:10.448 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:44:10.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:10.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:10.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:10.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:10.670 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3260563226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:10.671 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3260563226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:10.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:10.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:10.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3260563226' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:10.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:10.877 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:44:10.878 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:44:10.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:10.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:10.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:10.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:11.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:11.038 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:11.038 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:11.111 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:11.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:11.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:11.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:11.246 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.270 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:11.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:11.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:11.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:11.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:11.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:11.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:11.625 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:11.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:11 smithi152 ceph-mon[93456]: pgmap v1545: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 49846/131130 objects degraded (38.013%); 18009/131130 objects misplaced (13.734%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:44:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:11 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3260563226' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:11 smithi152 ceph-mon[93456]: osdmap e398: 8 total, 8 up, 6 in 2024-08-17T17:44:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49846/131130 objects degraded (38.013%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:11.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:44:11.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[106729]: pgmap v1545: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 49846/131130 objects degraded (38.013%); 18009/131130 objects misplaced (13.734%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3260563226' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[106729]: osdmap e398: 8 total, 8 up, 6 in 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49846/131130 objects degraded (38.013%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[103614]: pgmap v1545: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 49846/131130 objects degraded (38.013%); 18009/131130 objects misplaced (13.734%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3260563226' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[103614]: osdmap e398: 8 total, 8 up, 6 in 2024-08-17T17:44:11.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49846/131130 objects degraded (38.013%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:11.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:11.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:11.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:11.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:11.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:12.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:12.041 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:12.085 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:44:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:44:11] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:44:12.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:12.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:12.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:12.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:12.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:12.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:12.498 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:12.525 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:12.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:12.653 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:12.755 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:12.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:12 smithi152 ceph-mon[93456]: from='client.47759 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[106729]: from='client.47759 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[103614]: from='client.47759 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:12.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:12.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:12.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:12.985 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1546,"stamp":"2024-08-17T17:44:10.448414+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2850625509,"num_objects":43515,"num_object_clones":0,"num_object_copies":130545,"num_objects_missing_on_primary":3347,"num_objects_missing":3347,"num_objects_degraded":49789,"num_objects_misplaced":17812,"num_objects_unfound":0,"num_objects_dirty":43515,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":193179,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34250,"num_bytes_recovered":2121736846,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104927,"ondisk_log_size":104927,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11770580,"kb_used_data":5950308,"kb_used_omap":27,"kb_used_meta":5820132,"kb_avail":550700332,"statfs":{"total":575970213888,"available":563917139968,"internally_reserved":0,"allocated":6093115392,"data_stored":6080024205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":5959815912},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,1,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-120586240,"num_objects":-1840,"num_object_clones":0,"num_object_copies":-5520,"num_objects_missing_on_primary":-111,"num_objects_missing":-111,"num_objects_degraded":-229,"num_objects_misplaced":-742,"num_objects_unfound":0,"num_objects_dirty":-1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1840,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":355,"num_bytes_recovered":23265280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.963151"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":638,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516767+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:44:01.516767+0000","last_peered":"2024-08-17T17:44:01.516767+0000","last_clean":"2024-08-17T17:44:01.516767+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:44:01.516767+0000","last_undegraded":"2024-08-17T17:44:01.516767+0000","last_fullsized":"2024-08-17T17:44:01.516767+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":106,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459884+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:43:51.459884+0000","last_peered":"2024-08-17T17:43:51.459884+0000","last_clean":"2024-08-17T17:43:51.459884+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:43:51.459884+0000","last_undegraded":"2024-08-17T17:43:51.459884+0000","last_fullsized":"2024-08-17T17:43:51.459884+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":300,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.476946+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:43:51.476946+0000","last_peered":"2024-08-17T17:43:51.476946+0000","last_clean":"2024-08-17T17:43:51.476946+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:43:51.476946+0000","last_undegraded":"2024-08-17T17:43:51.476946+0000","last_fullsized":"2024-08-17T17:43:51.476946+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":336,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516569+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:44:01.516569+0000","last_peered":"2024-08-17T17:44:01.516569+0000","last_clean":"2024-08-17T17:44:01.516569+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:44:01.516569+0000","last_undegraded":"2024-08-17T17:44:01.516569+0000","last_fullsized":"2024-08-17T17:44:01.516569+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":513,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.517183+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:44:01.517183+0000","last_peered":"2024-08-17T17:44:01.517183+0000","last_clean":"2024-08-17T17:44:01.517183+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:44:01.517183+0000","last_undegraded":"2024-08-17T17:44:01.517183+0000","last_fullsized":"2024-08-17T17:44:01.517183+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":249,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459533+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:43:51.459533+0000","last_peered":"2024-08-17T17:43:51.459533+0000","last_clean":"2024-08-17T17:43:51.459533+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:43:51.459533+0000","last_undegraded":"2024-08-17T17:43:51.459533+0000","last_fullsized":"2024-08-17T17:43:51.459533+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":607,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.517175+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:44:01.517175+0000","last_peered":"2024-08-17T17:44:01.517175+0000","last_clean":"2024-08-17T17:44:01.517175+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:44:01.517175+0000","last_undegraded":"2024-08-17T17:44:01.517175+0000","last_fullsized":"2024-08-17T17:44:01.517175+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":417,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467446+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:43:51.467446+0000","last_peered":"2024-08-17T17:43:51.467446+0000","last_clean":"2024-08-17T17:43:51.467446+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:43:51.467446+0000","last_undegraded":"2024-08-17T17:43:51.467446+0000","last_fullsized":"2024-08-17T17:43:51.467446+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"397'12033","reported_seq":20046,"reported_epoch":397,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:44:10.372264+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:44:10.372264+0000","last_peered":"2024-08-17T17:44:10.372264+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:44:10.372264+0000","last_undegraded":"2024-08-17T17:44:10.372264+0000","last_fullsized":"2024-08-17T17:44:10.372264+0000","mapping_epoch":337,"log_start":"327'5163","ondisk_log_start":"327'5163","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175046656,"num_objects":2671,"num_object_clones":0,"num_object_copies":8013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2671,"num_objects_unfound":0,"num_objects_dirty":2671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12033,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3952,"num_bytes_recovered":258998272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2671}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":102,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.523314+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:43:51.523314+0000","last_peered":"2024-08-17T17:43:51.523314+0000","last_clean":"2024-08-17T17:43:51.523314+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:43:51.523314+0000","last_undegraded":"2024-08-17T17:43:51.523314+0000","last_fullsized":"2024-08-17T17:43:51.523314+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":283,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.478206+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:43:51.478206+0000","last_peered":"2024-08-17T17:43:51.478206+0000","last_clean":"2024-08-17T17:43:51.478206+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:43:51.478206+0000","last_undegraded":"2024-08-17T17:43:51.478206+0000","last_fullsized":"2024-08-17T17:43:51.478206+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"397'12209","reported_seq":12304,"reported_epoch":397,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:07.126298+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:44:07.126298+0000","last_peered":"2024-08-17T17:44:07.126298+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:44:07.126298+0000","last_undegraded":"2024-08-17T17:44:07.126298+0000","last_fullsized":"2024-08-17T17:44:07.126298+0000","mapping_epoch":337,"log_start":"327'5682","ondisk_log_start":"327'5682","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187760640,"num_objects":2865,"num_object_clones":0,"num_object_copies":8595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2865,"num_objects_unfound":0,"num_objects_dirty":2865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12209,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2865}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":104,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460417+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:43:51.460417+0000","last_peered":"2024-08-17T17:43:51.460417+0000","last_clean":"2024-08-17T17:43:51.460417+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:43:51.460417+0000","last_undegraded":"2024-08-17T17:43:51.460417+0000","last_fullsized":"2024-08-17T17:43:51.460417+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":368,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459515+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:43:51.459515+0000","last_peered":"2024-08-17T17:43:51.459515+0000","last_clean":"2024-08-17T17:43:51.459515+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:43:51.459515+0000","last_undegraded":"2024-08-17T17:43:51.459515+0000","last_fullsized":"2024-08-17T17:43:51.459515+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"397'12043","reported_seq":12141,"reported_epoch":397,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:10.247518+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:44:10.247518+0000","last_peered":"2024-08-17T17:44:10.247518+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:44:10.247518+0000","last_undegraded":"2024-08-17T17:44:10.247518+0000","last_fullsized":"2024-08-17T17:44:10.247518+0000","mapping_epoch":337,"log_start":"327'5153","ondisk_log_start":"327'5153","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176357376,"num_objects":2691,"num_object_clones":0,"num_object_copies":8073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2691,"num_objects_unfound":0,"num_objects_dirty":2691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12043,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2691}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":100,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.476881+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:43:51.476881+0000","last_peered":"2024-08-17T17:43:51.476881+0000","last_clean":"2024-08-17T17:43:51.476881+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:43:51.476881+0000","last_undegraded":"2024-08-17T17:43:51.476881+0000","last_fullsized":"2024-08-17T17:43:51.476881+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":378,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467399+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:43:51.467399+0000","last_peered":"2024-08-17T17:43:51.467399+0000","last_clean":"2024-08-17T17:43:51.467399+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:43:51.467399+0000","last_undegraded":"2024-08-17T17:43:51.467399+0000","last_fullsized":"2024-08-17T17:43:51.467399+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"397'12231","reported_seq":12407,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:09.952180+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:44:09.952180+0000","last_peered":"2024-08-17T17:44:09.952180+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:44:09.952180+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'5213","ondisk_log_start":"327'5213","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178716672,"num_objects":2727,"num_object_clones":0,"num_object_copies":8181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12231,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-4042},{"shards":"0,3,7","objects":2727}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":550,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516936+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:44:01.516936+0000","last_peered":"2024-08-17T17:44:01.516936+0000","last_clean":"2024-08-17T17:44:01.516936+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:44:01.516936+0000","last_undegraded":"2024-08-17T17:44:01.516936+0000","last_fullsized":"2024-08-17T17:44:01.516936+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":316,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.521671+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:44:01.521671+0000","last_peered":"2024-08-17T17:44:01.521671+0000","last_clean":"2024-08-17T17:44:01.521671+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:44:01.521671+0000","last_undegraded":"2024-08-17T17:44:01.521671+0000","last_fullsized":"2024-08-17T17:44:01.521671+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"397'11856","reported_seq":11942,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:06.979151+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:44:06.979151+0000","last_peered":"2024-08-17T17:44:06.979151+0000","last_clean":"2024-08-17T17:44:06.979151+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:44:06.979151+0000","last_undegraded":"2024-08-17T17:44:06.979151+0000","last_fullsized":"2024-08-17T17:44:06.979151+0000","mapping_epoch":319,"log_start":"327'5274","ondisk_log_start":"327'5274","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179437568,"num_objects":2738,"num_object_clones":0,"num_object_copies":8214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11856,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":640,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516972+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:44:01.516972+0000","last_peered":"2024-08-17T17:44:01.516972+0000","last_clean":"2024-08-17T17:44:01.516972+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:44:01.516972+0000","last_undegraded":"2024-08-17T17:44:01.516972+0000","last_fullsized":"2024-08-17T17:44:01.516972+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":351,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459978+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:43:51.459978+0000","last_peered":"2024-08-17T17:43:51.459978+0000","last_clean":"2024-08-17T17:43:51.459978+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:43:51.459978+0000","last_undegraded":"2024-08-17T17:43:51.459978+0000","last_fullsized":"2024-08-17T17:43:51.459978+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"397'12132","reported_seq":12228,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:07.010573+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:44:07.010573+0000","last_peered":"2024-08-17T17:44:07.010573+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:44:07.010573+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'5535","ondisk_log_start":"327'5535","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183500800,"num_objects":2800,"num_object_clones":0,"num_object_copies":8400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12132,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2800},{"shards":"1,6","objects":-4566}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":352,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467568+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:43:51.467568+0000","last_peered":"2024-08-17T17:43:51.467568+0000","last_clean":"2024-08-17T17:43:51.467568+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:43:51.467568+0000","last_undegraded":"2024-08-17T17:43:51.467568+0000","last_fullsized":"2024-08-17T17:43:51.467568+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":378,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460553+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:43:51.460553+0000","last_peered":"2024-08-17T17:43:51.460553+0000","last_clean":"2024-08-17T17:43:51.460553+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:43:51.460553+0000","last_undegraded":"2024-08-17T17:43:51.460553+0000","last_fullsized":"2024-08-17T17:43:51.460553+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"397'12234","reported_seq":12410,"reported_epoch":397,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:08.743005+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:44:08.743005+0000","last_peered":"2024-08-17T17:44:08.743005+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:44:08.743005+0000","last_undegraded":"2024-08-17T17:44:08.743005+0000","last_fullsized":"2024-08-17T17:44:08.743005+0000","mapping_epoch":337,"log_start":"328'6661","ondisk_log_start":"328'6661","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5573,"log_dups_size":0,"ondisk_log_size":5573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181927936,"num_objects":2776,"num_object_clones":0,"num_object_copies":8328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2776,"num_objects_unfound":0,"num_objects_dirty":2776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12234,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":2776}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":389,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459607+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:43:51.459607+0000","last_peered":"2024-08-17T17:43:51.459607+0000","last_clean":"2024-08-17T17:43:51.459607+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:43:51.459607+0000","last_undegraded":"2024-08-17T17:43:51.459607+0000","last_fullsized":"2024-08-17T17:43:51.459607+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":351,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460603+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:43:51.460603+0000","last_peered":"2024-08-17T17:43:51.460603+0000","last_clean":"2024-08-17T17:43:51.460603+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:43:51.460603+0000","last_undegraded":"2024-08-17T17:43:51.460603+0000","last_fullsized":"2024-08-17T17:43:51.460603+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"397'12049","reported_seq":20149,"reported_epoch":397,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:10.341857+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:44:10.341857+0000","last_peered":"2024-08-17T17:44:10.341857+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:44:10.341857+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:44:10.341857+0000","mapping_epoch":337,"log_start":"327'5141","ondisk_log_start":"327'5141","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6908,"log_dups_size":0,"ondisk_log_size":6908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173998080,"num_objects":2655,"num_object_clones":0,"num_object_copies":7965,"num_objects_missing_on_primary":3347,"num_objects_missing":3347,"num_objects_degraded":2655,"num_objects_misplaced":3347,"num_objects_unfound":0,"num_objects_dirty":2655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12049,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3992,"num_bytes_recovered":261554176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":2641},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":609,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516416+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:44:01.516416+0000","last_peered":"2024-08-17T17:44:01.516416+0000","last_clean":"2024-08-17T17:44:01.516416+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:44:01.516416+0000","last_undegraded":"2024-08-17T17:44:01.516416+0000","last_fullsized":"2024-08-17T17:44:01.516416+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":546,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.517170+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:44:01.517170+0000","last_peered":"2024-08-17T17:44:01.517170+0000","last_clean":"2024-08-17T17:44:01.517170+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:44:01.517170+0000","last_undegraded":"2024-08-17T17:44:01.517170+0000","last_fullsized":"2024-08-17T17:44:01.517170+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"397'12184","reported_seq":20321,"reported_epoch":397,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:09.007218+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:44:09.007218+0000","last_peered":"2024-08-17T17:44:09.007218+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:44:09.007218+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:44:09.007218+0000","mapping_epoch":337,"log_start":"328'6544","ondisk_log_start":"328'6544","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177668120,"num_objects":2712,"num_object_clones":0,"num_object_copies":8136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2712,"num_objects_misplaced":3462,"num_objects_unfound":0,"num_objects_dirty":2712,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12184,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3986,"num_bytes_recovered":261160960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2712}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":740,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:01.521148+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:44:01.521148+0000","last_peered":"2024-08-17T17:44:01.521148+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:44:01.521148+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":326,"reported_epoch":396,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:43:51.476633+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:43:51.476633+0000","last_peered":"2024-08-17T17:43:51.476633+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:43:51.476633+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:43:51.476633+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":589,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516518+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:44:01.516518+0000","last_peered":"2024-08-17T17:44:01.516518+0000","last_clean":"2024-08-17T17:44:01.516518+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:44:01.516518+0000","last_undegraded":"2024-08-17T17:44:01.516518+0000","last_fullsized":"2024-08-17T17:44:01.516518+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":361,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.468013+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:43:51.468013+0000","last_peered":"2024-08-17T17:43:51.468013+0000","last_clean":"2024-08-17T17:43:51.468013+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:43:51.468013+0000","last_undegraded":"2024-08-17T17:43:51.468013+0000","last_fullsized":"2024-08-17T17:43:51.468013+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"397'12012","reported_seq":12190,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:09.909257+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:44:09.909257+0000","last_peered":"2024-08-17T17:44:09.909257+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:44:09.909257+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'5021","ondisk_log_start":"327'5021","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179044352,"num_objects":2732,"num_object_clones":0,"num_object_copies":8196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12012,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-3915},{"shards":"0,4,7","objects":2732}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":405,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.486224+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:44:01.486224+0000","last_peered":"2024-08-17T17:44:01.486224+0000","last_clean":"2024-08-17T17:44:01.486224+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:44:01.486224+0000","last_undegraded":"2024-08-17T17:44:01.486224+0000","last_fullsized":"2024-08-17T17:44:01.486224+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":333,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459439+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:43:51.459439+0000","last_peered":"2024-08-17T17:43:51.459439+0000","last_clean":"2024-08-17T17:43:51.459439+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:43:51.459439+0000","last_undegraded":"2024-08-17T17:43:51.459439+0000","last_fullsized":"2024-08-17T17:43:51.459439+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":331,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.522011+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:44:01.522011+0000","last_peered":"2024-08-17T17:44:01.522011+0000","last_clean":"2024-08-17T17:44:01.522011+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:44:01.522011+0000","last_undegraded":"2024-08-17T17:44:01.522011+0000","last_fullsized":"2024-08-17T17:44:01.522011+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"397'12112","reported_seq":12208,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:07.106469+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:44:07.106469+0000","last_peered":"2024-08-17T17:44:07.106469+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:44:07.106469+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"327'5517","ondisk_log_start":"327'5517","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175898624,"num_objects":2684,"num_object_clones":0,"num_object_copies":8052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12112,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2684},{"shards":"5,6","objects":-4612}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":497,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516963+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:44:01.516963+0000","last_peered":"2024-08-17T17:44:01.516963+0000","last_clean":"2024-08-17T17:44:01.516963+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:44:01.516963+0000","last_undegraded":"2024-08-17T17:44:01.516963+0000","last_fullsized":"2024-08-17T17:44:01.516963+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.467342+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:43:51.467342+0000","last_peered":"2024-08-17T17:43:51.467342+0000","last_clean":"2024-08-17T17:43:51.467342+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:43:51.467342+0000","last_undegraded":"2024-08-17T17:43:51.467342+0000","last_fullsized":"2024-08-17T17:43:51.467342+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":681,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.486984+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:44:01.486984+0000","last_peered":"2024-08-17T17:44:01.486984+0000","last_clean":"2024-08-17T17:44:01.486984+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:44:01.486984+0000","last_undegraded":"2024-08-17T17:44:01.486984+0000","last_fullsized":"2024-08-17T17:44:01.486984+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"397'11971","reported_seq":12873,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:09.980459+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:44:09.980459+0000","last_peered":"2024-08-17T17:44:09.980459+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:44:09.980459+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"327'4916","ondisk_log_start":"327'4916","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175308800,"num_objects":2675,"num_object_clones":0,"num_object_copies":8025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11971,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2675},{"shards":"0,5","objects":-3979}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":473,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.486497+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:44:01.486497+0000","last_peered":"2024-08-17T17:44:01.486497+0000","last_clean":"2024-08-17T17:44:01.486497+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:44:01.486497+0000","last_undegraded":"2024-08-17T17:44:01.486497+0000","last_fullsized":"2024-08-17T17:44:01.486497+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":372,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460270+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:43:51.460270+0000","last_peered":"2024-08-17T17:43:51.460270+0000","last_clean":"2024-08-17T17:43:51.460270+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:43:51.460270+0000","last_undegraded":"2024-08-17T17:43:51.460270+0000","last_fullsized":"2024-08-17T17:43:51.460270+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":268,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516667+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:44:01.516667+0000","last_peered":"2024-08-17T17:44:01.516667+0000","last_clean":"2024-08-17T17:44:01.516667+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:44:01.516667+0000","last_undegraded":"2024-08-17T17:44:01.516667+0000","last_fullsized":"2024-08-17T17:44:01.516667+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"397'12113","reported_seq":27134,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:07.561293+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:44:07.561293+0000","last_peered":"2024-08-17T17:44:07.561293+0000","last_clean":"2024-08-17T17:44:07.561293+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:44:07.561293+0000","last_undegraded":"2024-08-17T17:44:07.561293+0000","last_fullsized":"2024-08-17T17:44:07.561293+0000","mapping_epoch":355,"log_start":"327'5830","ondisk_log_start":"327'5830","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178454528,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12113,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.476800+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:43:51.476800+0000","last_peered":"2024-08-17T17:43:51.476800+0000","last_clean":"2024-08-17T17:43:51.476800+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:43:51.476800+0000","last_undegraded":"2024-08-17T17:43:51.476800+0000","last_fullsized":"2024-08-17T17:43:51.476800+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":137,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516617+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:44:01.516617+0000","last_peered":"2024-08-17T17:44:01.516617+0000","last_clean":"2024-08-17T17:44:01.516617+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:44:01.516617+0000","last_undegraded":"2024-08-17T17:44:01.516617+0000","last_fullsized":"2024-08-17T17:44:01.516617+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":632,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.486673+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:44:01.486673+0000","last_peered":"2024-08-17T17:44:01.486673+0000","last_clean":"2024-08-17T17:44:01.486673+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:44:01.486673+0000","last_undegraded":"2024-08-17T17:44:01.486673+0000","last_fullsized":"2024-08-17T17:44:01.486673+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"397'11812","reported_seq":26401,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:07.700714+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:44:07.700714+0000","last_peered":"2024-08-17T17:44:07.700714+0000","last_clean":"2024-08-17T17:44:07.700714+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:44:07.700714+0000","last_undegraded":"2024-08-17T17:44:07.700714+0000","last_fullsized":"2024-08-17T17:44:07.700714+0000","mapping_epoch":345,"log_start":"327'5472","ondisk_log_start":"327'5472","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6340,"log_dups_size":0,"ondisk_log_size":6340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169869312,"num_objects":2592,"num_object_clones":0,"num_object_copies":7776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11812,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":307,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.477367+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:43:51.477367+0000","last_peered":"2024-08-17T17:43:51.477367+0000","last_clean":"2024-08-17T17:43:51.477367+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:43:51.477367+0000","last_undegraded":"2024-08-17T17:43:51.477367+0000","last_fullsized":"2024-08-17T17:43:51.477367+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":388,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.521462+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:44:01.521462+0000","last_peered":"2024-08-17T17:44:01.521462+0000","last_clean":"2024-08-17T17:44:01.521462+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:44:01.521462+0000","last_undegraded":"2024-08-17T17:44:01.521462+0000","last_fullsized":"2024-08-17T17:44:01.521462+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":449,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.476538+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:43:51.476538+0000","last_peered":"2024-08-17T17:43:51.476538+0000","last_clean":"2024-08-17T17:43:51.476538+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:43:51.476538+0000","last_undegraded":"2024-08-17T17:43:51.476538+0000","last_fullsized":"2024-08-17T17:43:51.476538+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"397'12002","reported_seq":12199,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:07.670505+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:44:07.670505+0000","last_peered":"2024-08-17T17:44:07.670505+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:44:07.670505+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"327'5687","ondisk_log_start":"327'5687","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6315,"log_dups_size":0,"ondisk_log_size":6315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182321152,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12002,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":2782},{"shards":"0,3","objects":-4466}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":662,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.517226+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:44:01.517226+0000","last_peered":"2024-08-17T17:44:01.517226+0000","last_clean":"2024-08-17T17:44:01.517226+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:44:01.517226+0000","last_undegraded":"2024-08-17T17:44:01.517226+0000","last_fullsized":"2024-08-17T17:44:01.517226+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":630,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.487269+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:44:01.487269+0000","last_peered":"2024-08-17T17:44:01.487269+0000","last_clean":"2024-08-17T17:44:01.487269+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:44:01.487269+0000","last_undegraded":"2024-08-17T17:44:01.487269+0000","last_fullsized":"2024-08-17T17:44:01.487269+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":648,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516222+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:44:01.516222+0000","last_peered":"2024-08-17T17:44:01.516222+0000","last_clean":"2024-08-17T17:44:01.516222+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:44:01.516222+0000","last_undegraded":"2024-08-17T17:44:01.516222+0000","last_fullsized":"2024-08-17T17:44:01.516222+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"397'11908","reported_seq":26602,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:06.938967+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:44:06.938967+0000","last_peered":"2024-08-17T17:44:06.938967+0000","last_clean":"2024-08-17T17:44:06.938967+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:44:06.938967+0000","last_undegraded":"2024-08-17T17:44:06.938967+0000","last_fullsized":"2024-08-17T17:44:06.938967+0000","mapping_epoch":347,"log_start":"327'5372","ondisk_log_start":"327'5372","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174718976,"num_objects":2666,"num_object_clones":0,"num_object_copies":7998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11908,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":321,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.521678+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:44:01.521678+0000","last_peered":"2024-08-17T17:44:01.521678+0000","last_clean":"2024-08-17T17:44:01.521678+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:44:01.521678+0000","last_undegraded":"2024-08-17T17:44:01.521678+0000","last_fullsized":"2024-08-17T17:44:01.521678+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":401,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.468184+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:43:51.468184+0000","last_peered":"2024-08-17T17:43:51.468184+0000","last_clean":"2024-08-17T17:43:51.468184+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:43:51.468184+0000","last_undegraded":"2024-08-17T17:43:51.468184+0000","last_fullsized":"2024-08-17T17:43:51.468184+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":351,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459586+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:43:51.459586+0000","last_peered":"2024-08-17T17:43:51.459586+0000","last_clean":"2024-08-17T17:43:51.459586+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:43:51.459586+0000","last_undegraded":"2024-08-17T17:43:51.459586+0000","last_fullsized":"2024-08-17T17:43:51.459586+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":351,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460144+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:43:51.460144+0000","last_peered":"2024-08-17T17:43:51.460144+0000","last_clean":"2024-08-17T17:43:51.460144+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:43:51.460144+0000","last_undegraded":"2024-08-17T17:43:51.460144+0000","last_fullsized":"2024-08-17T17:43:51.460144+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":394,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460139+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:43:51.460139+0000","last_peered":"2024-08-17T17:43:51.460139+0000","last_clean":"2024-08-17T17:43:51.460139+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:43:51.460139+0000","last_undegraded":"2024-08-17T17:43:51.460139+0000","last_fullsized":"2024-08-17T17:43:51.460139+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":637,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.486486+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:44:01.486486+0000","last_peered":"2024-08-17T17:44:01.486486+0000","last_clean":"2024-08-17T17:44:01.486486+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:44:01.486486+0000","last_undegraded":"2024-08-17T17:44:01.486486+0000","last_fullsized":"2024-08-17T17:44:01.486486+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":177,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516871+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:44:01.516871+0000","last_peered":"2024-08-17T17:44:01.516871+0000","last_clean":"2024-08-17T17:44:01.516871+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:44:01.516871+0000","last_undegraded":"2024-08-17T17:44:01.516871+0000","last_fullsized":"2024-08-17T17:44:01.516871+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":301,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.477120+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:43:51.477120+0000","last_peered":"2024-08-17T17:43:51.477120+0000","last_clean":"2024-08-17T17:43:51.477120+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:43:51.477120+0000","last_undegraded":"2024-08-17T17:43:51.477120+0000","last_fullsized":"2024-08-17T17:43:51.477120+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":595,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516250+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:44:01.516250+0000","last_peered":"2024-08-17T17:44:01.516250+0000","last_clean":"2024-08-17T17:44:01.516250+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:44:01.516250+0000","last_undegraded":"2024-08-17T17:44:01.516250+0000","last_fullsized":"2024-08-17T17:44:01.516250+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":445,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.477684+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:43:51.477684+0000","last_peered":"2024-08-17T17:43:51.477684+0000","last_clean":"2024-08-17T17:43:51.477684+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:43:51.477684+0000","last_undegraded":"2024-08-17T17:43:51.477684+0000","last_fullsized":"2024-08-17T17:43:51.477684+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":589,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516125+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:44:01.516125+0000","last_peered":"2024-08-17T17:44:01.516125+0000","last_clean":"2024-08-17T17:44:01.516125+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:44:01.516125+0000","last_undegraded":"2024-08-17T17:44:01.516125+0000","last_fullsized":"2024-08-17T17:44:01.516125+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":368,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.468120+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:43:51.468120+0000","last_peered":"2024-08-17T17:43:51.468120+0000","last_clean":"2024-08-17T17:43:51.468120+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:43:51.468120+0000","last_undegraded":"2024-08-17T17:43:51.468120+0000","last_fullsized":"2024-08-17T17:43:51.468120+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":621,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516373+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:44:01.516373+0000","last_peered":"2024-08-17T17:44:01.516373+0000","last_clean":"2024-08-17T17:44:01.516373+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:44:01.516373+0000","last_undegraded":"2024-08-17T17:44:01.516373+0000","last_fullsized":"2024-08-17T17:44:01.516373+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":337,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.517108+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:44:01.517108+0000","last_peered":"2024-08-17T17:44:01.517108+0000","last_clean":"2024-08-17T17:44:01.517108+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:44:01.517108+0000","last_undegraded":"2024-08-17T17:44:01.517108+0000","last_fullsized":"2024-08-17T17:44:01.517108+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":629,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.486398+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:44:01.486398+0000","last_peered":"2024-08-17T17:44:01.486398+0000","last_clean":"2024-08-17T17:44:01.486398+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:44:01.486398+0000","last_undegraded":"2024-08-17T17:44:01.486398+0000","last_fullsized":"2024-08-17T17:44:01.486398+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":588,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516683+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:44:01.516683+0000","last_peered":"2024-08-17T17:44:01.516683+0000","last_clean":"2024-08-17T17:44:01.516683+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:44:01.516683+0000","last_undegraded":"2024-08-17T17:44:01.516683+0000","last_fullsized":"2024-08-17T17:44:01.516683+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":380,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459866+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:43:51.459866+0000","last_peered":"2024-08-17T17:43:51.459866+0000","last_clean":"2024-08-17T17:43:51.459866+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:43:51.459866+0000","last_undegraded":"2024-08-17T17:43:51.459866+0000","last_fullsized":"2024-08-17T17:43:51.459866+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516849+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:44:01.516849+0000","last_peered":"2024-08-17T17:44:01.516849+0000","last_clean":"2024-08-17T17:44:01.516849+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:44:01.516849+0000","last_undegraded":"2024-08-17T17:44:01.516849+0000","last_fullsized":"2024-08-17T17:44:01.516849+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":333,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.521559+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:44:01.521559+0000","last_peered":"2024-08-17T17:44:01.521559+0000","last_clean":"2024-08-17T17:44:01.521559+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:44:01.521559+0000","last_undegraded":"2024-08-17T17:44:01.521559+0000","last_fullsized":"2024-08-17T17:44:01.521559+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":388,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460079+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:43:51.460079+0000","last_peered":"2024-08-17T17:43:51.460079+0000","last_clean":"2024-08-17T17:43:51.460079+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:43:51.460079+0000","last_undegraded":"2024-08-17T17:43:51.460079+0000","last_fullsized":"2024-08-17T17:43:51.460079+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":251,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460054+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:43:51.460054+0000","last_peered":"2024-08-17T17:43:51.460054+0000","last_clean":"2024-08-17T17:43:51.460054+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:43:51.460054+0000","last_undegraded":"2024-08-17T17:43:51.460054+0000","last_fullsized":"2024-08-17T17:43:51.460054+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":388,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.459734+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:43:51.459734+0000","last_peered":"2024-08-17T17:43:51.459734+0000","last_clean":"2024-08-17T17:43:51.459734+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:43:51.459734+0000","last_undegraded":"2024-08-17T17:43:51.459734+0000","last_fullsized":"2024-08-17T17:43:51.459734+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":251,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460439+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:43:51.460439+0000","last_peered":"2024-08-17T17:43:51.460439+0000","last_clean":"2024-08-17T17:43:51.460439+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:43:51.460439+0000","last_undegraded":"2024-08-17T17:43:51.460439+0000","last_fullsized":"2024-08-17T17:43:51.460439+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":641,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.516450+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:44:01.516450+0000","last_peered":"2024-08-17T17:44:01.516450+0000","last_clean":"2024-08-17T17:44:01.516450+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:44:01.516450+0000","last_undegraded":"2024-08-17T17:44:01.516450+0000","last_fullsized":"2024-08-17T17:44:01.516450+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":614,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.517258+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:44:01.517258+0000","last_peered":"2024-08-17T17:44:01.517258+0000","last_clean":"2024-08-17T17:44:01.517258+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:44:01.517258+0000","last_undegraded":"2024-08-17T17:44:01.517258+0000","last_fullsized":"2024-08-17T17:44:01.517258+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":351,"reported_epoch":396,"state":"active+clean","last_fresh":"2024-08-17T17:43:51.460263+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:43:51.460263+0000","last_peered":"2024-08-17T17:43:51.460263+0000","last_clean":"2024-08-17T17:43:51.460263+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:43:51.460263+0000","last_undegraded":"2024-08-17T17:43:51.460263+0000","last_fullsized":"2024-08-17T17:43:51.460263+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":336,"reported_epoch":397,"state":"active+clean","last_fresh":"2024-08-17T17:44:01.521768+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:44:01.521768+0000","last_peered":"2024-08-17T17:44:01.521768+0000","last_clean":"2024-08-17T17:44:01.521768+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:44:01.521768+0000","last_undegraded":"2024-08-17T17:44:01.521768+0000","last_fullsized":"2024-08-17T17:44:01.521768+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2850029592,"num_objects":43489,"num_object_clones":0,"num_object_copies":130467,"num_objects_missing_on_primary":3347,"num_objects_missing":3347,"num_objects_degraded":49786,"num_objects_misplaced":17812,"num_objects_unfound":0,"num_objects_dirty":43489,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":192901,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34209,"num_bytes_recovered":2119368704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11178553344,"data_stored":11178541128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104720,"ondisk_log_size":104720,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170132,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463502,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2119528,"kb_used_data":1229984,"kb_used_omap":1,"kb_used_meta":889534,"kb_avail":91625624,"statfs":{"total":95995035648,"available":93824638976,"internally_reserved":0,"allocated":1259503616,"data_stored":1257309556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":910883160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492177,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":742156,"kb_used_data":346820,"kb_used_omap":7,"kb_used_meta":395320,"kb_avail":93002996,"statfs":{"total":95995035648,"available":95235067904,"internally_reserved":0,"allocated":355143680,"data_stored":352971920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":404808013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986768,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2185116,"kb_used_data":688916,"kb_used_omap":1,"kb_used_meta":1496190,"kb_avail":91560036,"statfs":{"total":95995035648,"available":93757476864,"internally_reserved":0,"allocated":705449984,"data_stored":703266864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1532098904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225596,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2355084,"kb_used_data":1407428,"kb_used_omap":4,"kb_used_meta":947643,"kb_avail":91390068,"statfs":{"total":95995035648,"available":93583429632,"internally_reserved":0,"allocated":1441206272,"data_stored":1439034810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":970386880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075921,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533814,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2287184,"kb_used_data":1222984,"kb_used_omap":5,"kb_used_meta":1064186,"kb_avail":91457968,"statfs":{"total":95995035648,"available":93652959232,"internally_reserved":0,"allocated":1252335616,"data_stored":1250158832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1089726704},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089791,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2081512,"kb_used_data":1054176,"kb_used_omap":5,"kb_used_meta":1027258,"kb_avail":91663640,"statfs":{"total":95995035648,"available":93863567360,"internally_reserved":0,"allocated":1079476224,"data_stored":1077282223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1051912251},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1071120384,"data_stored":1071120384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1246040064,"data_stored":1246035992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1431830528,"data_stored":1431830528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":697958400,"data_stored":697958400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":349634560,"data_stored":349634560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1249181696,"data_stored":1249181696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1793527808,"data_stored":1793523736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:44:12.992 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:44:13.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:13.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:13.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:13.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:13.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:13.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:13.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:13.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:13.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:13.415 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:13.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:13.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:13.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:13.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:13.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:13.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:44:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.687 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:13.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:13.767 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[106729]: pgmap v1547: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 49789/129738 objects degraded (38.377%); 17752/129738 objects misplaced (13.683%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[103614]: pgmap v1547: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 49789/129738 objects degraded (38.377%); 17752/129738 objects misplaced (13.683%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.768 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:13.806 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:13 smithi152 ceph-mon[93456]: pgmap v1547: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 190 op/s; 49789/129738 objects degraded (38.377%); 17752/129738 objects misplaced (13.683%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:13.806 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.806 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.806 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.807 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:13.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:13.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:13.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:13.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:14.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:14.069 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:14.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:14.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:14.304 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:14.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:14.393 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:14.394 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:14.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:14.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:14.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:14.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:14.679 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:14.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:14.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:14.764 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:14 smithi152 ceph-mon[93456]: pgmap v1548: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 49719/128784 objects degraded (38.607%); 17634/128784 objects misplaced (13.693%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:44:14.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:14.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:14 smithi142 ceph-mon[106729]: pgmap v1548: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 49719/128784 objects degraded (38.607%); 17634/128784 objects misplaced (13.693%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:44:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:14 smithi142 ceph-mon[103614]: pgmap v1548: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 49719/128784 objects degraded (38.607%); 17634/128784 objects misplaced (13.693%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:44:14.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:14.954 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:14.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:15.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:15.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:15.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:15.207 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:15.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:15.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:44:15.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:15.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:15.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:15.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:15.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:15.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:15.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:15.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:15.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:15.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:15.836 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:15.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:15.848 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:44:15.848 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:15.877 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:44:15.878 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:15.883 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:15.899 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:44:15.900 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:15.918 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:44:15.918 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:15.933 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:44:15.934 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:15.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:15.949 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:44:15.949 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:15.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:15.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:15.966 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:44:15.966 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:15.983 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:15 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49719/128784 objects degraded (38.607%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:15.983 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:15 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49719/128784 objects degraded (38.607%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:15.983 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:44:15.983 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:16.029 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:16.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:15 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49719/128784 objects degraded (38.607%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:16.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:16.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:16.213 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:16.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:16.341 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:16.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:16.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:16.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:16.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:16.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:16.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:16.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:16.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:16.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:16.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:16.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:16.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:16.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:17.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1479666833' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:16 smithi152 ceph-mon[93456]: pgmap v1549: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 49661/127830 objects degraded (38.849%); 17469/127830 objects misplaced (13.666%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:17.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:17.118 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1479666833' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:17.119 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:16 smithi142 ceph-mon[106729]: pgmap v1549: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 49661/127830 objects degraded (38.849%); 17469/127830 objects misplaced (13.666%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:17.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1479666833' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:17.119 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:16 smithi142 ceph-mon[103614]: pgmap v1549: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 49661/127830 objects degraded (38.849%); 17469/127830 objects misplaced (13.666%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:17.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:17.213 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:17.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:17.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:17.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:17.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:17.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:17.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:17.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:17.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:17.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:17.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:17.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:17.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:17.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:17.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:17.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:17.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:17.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:18.056 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:18.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:18.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:18.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:18.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:18.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:18.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:18.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:18.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:18.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:18.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:18.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:18.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:18.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:18.707 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:44:18.708 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:44:18.708 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:44:18.708 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:44:18.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:18.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:18.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:18.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:18.895 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:44:18.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:18.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:18.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:18.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:19.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:19.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:19.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:19.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:19.269 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:19.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:19.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:44:19.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:19.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:19.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:19.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:19.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:19.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:19.724 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:19 smithi142 ceph-mon[106729]: pgmap v1550: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49661/126561 objects degraded (39.239%); 17403/126561 objects misplaced (13.751%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:19.725 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:19 smithi142 ceph-mon[103614]: pgmap v1550: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49661/126561 objects degraded (39.239%); 17403/126561 objects misplaced (13.751%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:19.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:19 smithi152 ceph-mon[93456]: pgmap v1550: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49661/126561 objects degraded (39.239%); 17403/126561 objects misplaced (13.751%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:19.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:19.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:19.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:19.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:19.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:20.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:20.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:20.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:20.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:20.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:20.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:20.362 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:20.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:20.498 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:44:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:44:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:20.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:20.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:20.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:20.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:20.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:20.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:20.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:20.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:20.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:20.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:20.961 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:44:20.962 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 49597/125535 objects degraded (39.509%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2341,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":398,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":41845,"data_bytes":2741180389,"bytes_used":11826745344,"bytes_avail":564143468544,"bytes_total":575970213888,"degraded_objects":49597,"degraded_total":125535,"degraded_ratio":0.39508503604572431,"misplaced_objects":17265,"misplaced_total":125535,"misplaced_ratio":0.13753136575457045,"recovering_objects_per_sec":35,"recovering_bytes_per_sec":2332405,"recovering_keys_per_sec":0,"num_objects_recovered":356,"num_bytes_recovered":23330816,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":186},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":35,"modified":"2024-08-17T17:43:38.379260+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:44:20.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:21.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:21.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:21.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:21.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:21.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:21.381 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:21.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:21.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:21.479 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[106729]: pgmap v1551: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 186 op/s; 49597/125535 objects degraded (39.509%); 17265/125535 objects misplaced (13.753%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:21.479 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49597/125535 objects degraded (39.509%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:21.479 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/504686027' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/606998750' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[103614]: pgmap v1551: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 186 op/s; 49597/125535 objects degraded (39.509%); 17265/125535 objects misplaced (13.753%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49597/125535 objects degraded (39.509%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/504686027' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/606998750' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:21.480 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:21.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:21.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:21.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:21 smithi152 ceph-mon[93456]: pgmap v1551: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 186 op/s; 49597/125535 objects degraded (39.509%); 17265/125535 objects misplaced (13.753%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49597/125535 objects degraded (39.509%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/504686027' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:44:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/606998750' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:21.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:21 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:44:21.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:21.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:21.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:21.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:21.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:21.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:21.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:22.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:22.065 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:44:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:44:21] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:44:22.078 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:22.100 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:44:22.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:44:22.174 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:44:22.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:22.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:22.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:22.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:22.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:22.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:22.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:22.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:22 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:22 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:44:22.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:22 smithi152 ceph-mon[93456]: osdmap e399: 8 total, 8 up, 6 in 2024-08-17T17:44:22.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:22.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:22.717 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:22.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:22.839 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:22 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:22.840 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:22 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:44:22.840 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:22 smithi142 ceph-mon[103614]: osdmap e399: 8 total, 8 up, 6 in 2024-08-17T17:44:22.840 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:22 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:22.840 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:22 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:44:22.840 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:22 smithi142 ceph-mon[106729]: osdmap e399: 8 total, 8 up, 6 in 2024-08-17T17:44:22.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:22.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:22.971 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:22.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:23.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:23.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:23.181 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:44:23.181 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:44:23.181 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:44:23.181 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:44:23.181 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:44:23.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:23.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:23.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:23.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:23.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:23.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:23.470 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:23.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:23.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:23.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:23.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:23 smithi142 ceph-mon[106729]: pgmap v1553: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 193 op/s; 49537/123945 objects degraded (39.967%); 17007/123945 objects misplaced (13.721%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:23.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:23 smithi142 ceph-mon[103614]: pgmap v1553: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 193 op/s; 49537/123945 objects degraded (39.967%); 17007/123945 objects misplaced (13.721%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:23.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:23.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:23.768 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:23 smithi152 ceph-mon[93456]: pgmap v1553: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 193 op/s; 49537/123945 objects degraded (39.967%); 17007/123945 objects misplaced (13.721%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:23.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:23.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:23.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:23.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:23.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:23.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:24.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:44:24.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:24.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:24.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:24.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:24.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:24.364 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:24.430 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:44:24.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:24.447 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:44:24.448 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:24.458 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:44:24.463 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:44:24.464 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:24.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:24.488 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:44:24.488 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:24.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:24.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:24.504 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:44:24.504 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:24.521 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:44:24.521 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:24.540 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:44:24.540 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:24.540 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/103734671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:24.541 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/103734671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:24.557 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:44:24.558 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:24.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:24.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:24.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:24.744 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/103734671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:24.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:24.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:24.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:24.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:24.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:25.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:25.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:25.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:25.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:25.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:25.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:25.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:25.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:25.500 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:25.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:25.602 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:25 smithi142 ceph-mon[106729]: pgmap v1554: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 49452/122925 objects degraded (40.229%); 16862/122925 objects misplaced (13.717%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:44:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:25.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:25 smithi142 ceph-mon[103614]: pgmap v1554: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 49452/122925 objects degraded (40.229%); 16862/122925 objects misplaced (13.717%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:44:25.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:25.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:25.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:25.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:25.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:25 smithi152 ceph-mon[93456]: pgmap v1554: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 49452/122925 objects degraded (40.229%); 16862/122925 objects misplaced (13.717%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:44:25.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:25.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:25.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:25.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:25.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:44:25.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:25.880 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:25.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:25.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:26.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:26.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:26.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:26.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:26.257 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:26.292 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:26.292 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:26.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:26.384 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:26.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:26.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:26.508 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49452/122925 objects degraded (40.229%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:26.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:26.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:26.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:26.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:26.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:26.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:26.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49452/122925 objects degraded (40.229%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49452/122925 objects degraded (40.229%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:26.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:26.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:26.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:44:26.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:27.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:27.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:27.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:27.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:27.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:27.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:27.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:27.374 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:27.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:27.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:27.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:27.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:27 smithi142 ceph-mon[103614]: pgmap v1555: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 49396/121656 objects degraded (40.603%); 16650/121656 objects misplaced (13.686%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:27 smithi142 ceph-mon[106729]: pgmap v1555: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 49396/121656 objects degraded (40.603%); 16650/121656 objects misplaced (13.686%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:27.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:27.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:27.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:27.740 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:27.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:27.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:27.790 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:27 smithi152 ceph-mon[93456]: pgmap v1555: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 49396/121656 objects degraded (40.603%); 16650/121656 objects misplaced (13.686%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:27.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:27.883 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:27.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:27.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:44:27.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:28.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:28.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:28.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:28.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:28.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:28.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:28.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:28.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:28.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:28.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:28.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:28.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:28.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:28.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:28.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:28.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:28.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:28.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:28.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:28.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:28.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:29.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:29.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:29.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:29.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:29.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:29.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:29.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:29.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:29.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:29.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:29.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:29.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:29.747 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:29.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:29.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:29 smithi142 ceph-mon[103614]: pgmap v1556: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 49396/120381 objects degraded (41.033%); 16609/120381 objects misplaced (13.797%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:29.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:29 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:29.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:29 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:29.800 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:29 smithi142 ceph-mon[106729]: pgmap v1556: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 49396/120381 objects degraded (41.033%); 16609/120381 objects misplaced (13.797%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:29.800 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:29 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:29.800 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:29 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:29 smithi152 ceph-mon[93456]: pgmap v1556: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 205 op/s; 49396/120381 objects degraded (41.033%); 16609/120381 objects misplaced (13.797%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:29 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:29 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:29.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:29.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:29.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:29.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:29.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:30.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:30.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:30.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:30.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:30.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:30.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:44:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:44:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:30.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:30.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:30.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:30.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:30.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:30.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:30.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:30.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:30.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:30.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:30.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:30.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:30.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:30.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:30.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:31.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:31.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:31.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:31.148 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:44:31.150 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:44:31.155 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:31.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:31.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:31.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:31.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:31.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:31.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-mon[106729]: from='client.37923 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-mon[106729]: pgmap v1557: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 49351/119568 objects degraded (41.274%); 16514/119568 objects misplaced (13.811%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:44:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49396/120381 objects degraded (41.033%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-mon[103614]: from='client.37923 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-mon[103614]: pgmap v1557: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 49351/119568 objects degraded (41.274%); 16514/119568 objects misplaced (13.811%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:44:31.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49396/120381 objects degraded (41.033%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:31.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:31.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:31.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:31.780 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:31.799 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:31 smithi152 ceph-mon[93456]: from='client.37923 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:31 smithi152 ceph-mon[93456]: pgmap v1557: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 49351/119568 objects degraded (41.274%); 16514/119568 objects misplaced (13.811%); 2.3 MiB/s, 37 objects/s recovering 2024-08-17T17:44:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49396/120381 objects degraded (41.033%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:31.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:31.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:31.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:32.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:32.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:32.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:44:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:44:31] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-08-17T17:44:32.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:32.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:32.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:32.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:32.362 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1556,"stamp":"2024-08-17T17:44:28.394011+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2628589541,"num_objects":40127,"num_object_clones":0,"num_object_copies":120381,"num_objects_missing_on_primary":3174,"num_objects_missing":3174,"num_objects_degraded":49396,"num_objects_misplaced":16609,"num_objects_unfound":0,"num_objects_dirty":40127,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":196567,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34818,"num_bytes_recovered":2158961294,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104915,"ondisk_log_size":104915,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11676180,"kb_used_data":5502372,"kb_used_omap":27,"kb_used_meta":6173732,"kb_avail":550794732,"statfs":{"total":575970213888,"available":564013805568,"internally_reserved":0,"allocated":5634428928,"data_stored":5621244225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6321902312},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1,2,1,0,2,1,8,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-135004160,"num_objects":-2060,"num_object_clones":0,"num_object_copies":-6180,"num_objects_missing_on_primary":-117,"num_objects_missing":-117,"num_objects_degraded":-265,"num_objects_misplaced":-794,"num_objects_unfound":0,"num_objects_dirty":-2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2060,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":393,"num_bytes_recovered":25755648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002542"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":642,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454562+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:44:21.454562+0000","last_peered":"2024-08-17T17:44:21.454562+0000","last_clean":"2024-08-17T17:44:21.454562+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:44:21.454562+0000","last_undegraded":"2024-08-17T17:44:21.454562+0000","last_fullsized":"2024-08-17T17:44:21.454562+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":108,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470008+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:44:10.470008+0000","last_peered":"2024-08-17T17:44:10.470008+0000","last_clean":"2024-08-17T17:44:10.470008+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:44:10.470008+0000","last_undegraded":"2024-08-17T17:44:10.470008+0000","last_fullsized":"2024-08-17T17:44:10.470008+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":302,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523495+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:44:10.523495+0000","last_peered":"2024-08-17T17:44:10.523495+0000","last_clean":"2024-08-17T17:44:10.523495+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:44:10.523495+0000","last_undegraded":"2024-08-17T17:44:10.523495+0000","last_fullsized":"2024-08-17T17:44:10.523495+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":340,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.455035+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:44:21.455035+0000","last_peered":"2024-08-17T17:44:21.455035+0000","last_clean":"2024-08-17T17:44:21.455035+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:44:21.455035+0000","last_undegraded":"2024-08-17T17:44:21.455035+0000","last_fullsized":"2024-08-17T17:44:21.455035+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":517,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.461233+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:44:21.461233+0000","last_peered":"2024-08-17T17:44:21.461233+0000","last_clean":"2024-08-17T17:44:21.461233+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:44:21.461233+0000","last_undegraded":"2024-08-17T17:44:21.461233+0000","last_fullsized":"2024-08-17T17:44:21.461233+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":251,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469636+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:44:10.469636+0000","last_peered":"2024-08-17T17:44:10.469636+0000","last_clean":"2024-08-17T17:44:10.469636+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:44:10.469636+0000","last_undegraded":"2024-08-17T17:44:10.469636+0000","last_fullsized":"2024-08-17T17:44:10.469636+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":611,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459937+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:44:21.459937+0000","last_peered":"2024-08-17T17:44:21.459937+0000","last_clean":"2024-08-17T17:44:21.459937+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:44:21.459937+0000","last_undegraded":"2024-08-17T17:44:21.459937+0000","last_fullsized":"2024-08-17T17:44:21.459937+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":419,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.476445+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:44:10.476445+0000","last_peered":"2024-08-17T17:44:10.476445+0000","last_clean":"2024-08-17T17:44:10.476445+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:44:10.476445+0000","last_undegraded":"2024-08-17T17:44:10.476445+0000","last_fullsized":"2024-08-17T17:44:10.476445+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"399'12209","reported_seq":20576,"reported_epoch":399,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:44:25.362840+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:44:25.362840+0000","last_peered":"2024-08-17T17:44:25.362840+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:44:25.362840+0000","last_undegraded":"2024-08-17T17:44:25.362840+0000","last_fullsized":"2024-08-17T17:44:25.362840+0000","mapping_epoch":337,"log_start":"327'5363","ondisk_log_start":"327'5363","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163512320,"num_objects":2495,"num_object_clones":0,"num_object_copies":7485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2495,"num_objects_unfound":0,"num_objects_dirty":2495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12209,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4128,"num_bytes_recovered":270532608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2495}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":104,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.517437+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:44:10.517437+0000","last_peered":"2024-08-17T17:44:10.517437+0000","last_clean":"2024-08-17T17:44:10.517437+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:44:10.517437+0000","last_undegraded":"2024-08-17T17:44:10.517437+0000","last_fullsized":"2024-08-17T17:44:10.517437+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":285,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523137+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:44:10.523137+0000","last_peered":"2024-08-17T17:44:10.523137+0000","last_clean":"2024-08-17T17:44:10.523137+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:44:10.523137+0000","last_undegraded":"2024-08-17T17:44:10.523137+0000","last_fullsized":"2024-08-17T17:44:10.523137+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"399'12463","reported_seq":12560,"reported_epoch":399,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:26.655701+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:44:26.655701+0000","last_peered":"2024-08-17T17:44:26.655701+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:44:26.655701+0000","last_undegraded":"2024-08-17T17:44:26.655701+0000","last_fullsized":"2024-08-17T17:44:26.655701+0000","mapping_epoch":337,"log_start":"327'5882","ondisk_log_start":"327'5882","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6581,"log_dups_size":0,"ondisk_log_size":6581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171114496,"num_objects":2611,"num_object_clones":0,"num_object_copies":7833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2611,"num_objects_unfound":0,"num_objects_dirty":2611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12463,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2611}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":106,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470196+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:44:10.470196+0000","last_peered":"2024-08-17T17:44:10.470196+0000","last_clean":"2024-08-17T17:44:10.470196+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:44:10.470196+0000","last_undegraded":"2024-08-17T17:44:10.470196+0000","last_fullsized":"2024-08-17T17:44:10.470196+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":370,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469557+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:44:10.469557+0000","last_peered":"2024-08-17T17:44:10.469557+0000","last_clean":"2024-08-17T17:44:10.469557+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:44:10.469557+0000","last_undegraded":"2024-08-17T17:44:10.469557+0000","last_fullsized":"2024-08-17T17:44:10.469557+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"399'12242","reported_seq":12342,"reported_epoch":399,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:25.417824+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:44:25.417824+0000","last_peered":"2024-08-17T17:44:25.417824+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:44:25.417824+0000","last_undegraded":"2024-08-17T17:44:25.417824+0000","last_fullsized":"2024-08-17T17:44:25.417824+0000","mapping_epoch":337,"log_start":"327'5353","ondisk_log_start":"327'5353","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6889,"log_dups_size":0,"ondisk_log_size":6889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315712,"num_objects":2492,"num_object_clones":0,"num_object_copies":7476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2492,"num_objects_unfound":0,"num_objects_dirty":2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12242,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2492}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":102,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523434+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:44:10.523434+0000","last_peered":"2024-08-17T17:44:10.523434+0000","last_clean":"2024-08-17T17:44:10.523434+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:44:10.523434+0000","last_undegraded":"2024-08-17T17:44:10.523434+0000","last_fullsized":"2024-08-17T17:44:10.523434+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":380,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.476448+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:44:10.476448+0000","last_peered":"2024-08-17T17:44:10.476448+0000","last_clean":"2024-08-17T17:44:10.476448+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:44:10.476448+0000","last_undegraded":"2024-08-17T17:44:10.476448+0000","last_fullsized":"2024-08-17T17:44:10.476448+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"399'12425","reported_seq":12605,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:24.555124+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:44:24.555124+0000","last_peered":"2024-08-17T17:44:24.555124+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:44:24.555124+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'5413","ondisk_log_start":"327'5413","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166002688,"num_objects":2533,"num_object_clones":0,"num_object_copies":7599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12425,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-4236},{"shards":"0,3,7","objects":2533}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":554,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454728+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:44:21.454728+0000","last_peered":"2024-08-17T17:44:21.454728+0000","last_clean":"2024-08-17T17:44:21.454728+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:44:21.454728+0000","last_undegraded":"2024-08-17T17:44:21.454728+0000","last_fullsized":"2024-08-17T17:44:21.454728+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":320,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458230+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:44:21.458230+0000","last_peered":"2024-08-17T17:44:21.458230+0000","last_clean":"2024-08-17T17:44:21.458230+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:44:21.458230+0000","last_undegraded":"2024-08-17T17:44:21.458230+0000","last_fullsized":"2024-08-17T17:44:21.458230+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"399'12074","reported_seq":12162,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:26.681508+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:44:26.681508+0000","last_peered":"2024-08-17T17:44:26.681508+0000","last_clean":"2024-08-17T17:44:26.681508+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:44:26.681508+0000","last_undegraded":"2024-08-17T17:44:26.681508+0000","last_fullsized":"2024-08-17T17:44:26.681508+0000","mapping_epoch":319,"log_start":"327'5474","ondisk_log_start":"327'5474","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6600,"log_dups_size":0,"ondisk_log_size":6600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165150720,"num_objects":2520,"num_object_clones":0,"num_object_copies":7560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12074,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":644,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459565+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:44:21.459565+0000","last_peered":"2024-08-17T17:44:21.459565+0000","last_clean":"2024-08-17T17:44:21.459565+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:44:21.459565+0000","last_undegraded":"2024-08-17T17:44:21.459565+0000","last_fullsized":"2024-08-17T17:44:21.459565+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":353,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470166+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:44:10.470166+0000","last_peered":"2024-08-17T17:44:10.470166+0000","last_clean":"2024-08-17T17:44:10.470166+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:44:10.470166+0000","last_undegraded":"2024-08-17T17:44:10.470166+0000","last_fullsized":"2024-08-17T17:44:10.470166+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"399'12350","reported_seq":12448,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:26.933942+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:44:26.933942+0000","last_peered":"2024-08-17T17:44:26.933942+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:44:26.933942+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"327'5735","ondisk_log_start":"327'5735","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169213952,"num_objects":2582,"num_object_clones":0,"num_object_copies":7746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12350,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2582},{"shards":"1,6","objects":-4784}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":354,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.476897+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:44:10.476897+0000","last_peered":"2024-08-17T17:44:10.476897+0000","last_clean":"2024-08-17T17:44:10.476897+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:44:10.476897+0000","last_undegraded":"2024-08-17T17:44:10.476897+0000","last_fullsized":"2024-08-17T17:44:10.476897+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":380,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470001+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:44:10.470001+0000","last_peered":"2024-08-17T17:44:10.470001+0000","last_clean":"2024-08-17T17:44:10.470001+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:44:10.470001+0000","last_undegraded":"2024-08-17T17:44:10.470001+0000","last_fullsized":"2024-08-17T17:44:10.470001+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"399'12416","reported_seq":12596,"reported_epoch":399,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:23.717078+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:44:23.717078+0000","last_peered":"2024-08-17T17:44:23.717078+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:44:23.717078+0000","last_undegraded":"2024-08-17T17:44:23.717078+0000","last_fullsized":"2024-08-17T17:44:23.717078+0000","mapping_epoch":337,"log_start":"329'6761","ondisk_log_start":"329'6761","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170000384,"num_objects":2594,"num_object_clones":0,"num_object_copies":7782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2594,"num_objects_unfound":0,"num_objects_dirty":2594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12416,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":2594}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":391,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469781+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:44:10.469781+0000","last_peered":"2024-08-17T17:44:10.469781+0000","last_clean":"2024-08-17T17:44:10.469781+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:44:10.469781+0000","last_undegraded":"2024-08-17T17:44:10.469781+0000","last_fullsized":"2024-08-17T17:44:10.469781+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":353,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470315+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:44:10.470315+0000","last_peered":"2024-08-17T17:44:10.470315+0000","last_clean":"2024-08-17T17:44:10.470315+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:44:10.470315+0000","last_undegraded":"2024-08-17T17:44:10.470315+0000","last_fullsized":"2024-08-17T17:44:10.470315+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"399'12223","reported_seq":20671,"reported_epoch":399,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:25.409442+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:44:25.409442+0000","last_peered":"2024-08-17T17:44:25.409442+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:44:25.409442+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:44:25.409442+0000","mapping_epoch":337,"log_start":"327'5341","ondisk_log_start":"327'5341","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6882,"log_dups_size":0,"ondisk_log_size":6882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":3174,"num_objects_missing":3174,"num_objects_degraded":2481,"num_objects_misplaced":3174,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12223,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4165,"num_bytes_recovered":272891904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":2468}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":613,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.455389+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:44:21.455389+0000","last_peered":"2024-08-17T17:44:21.455389+0000","last_clean":"2024-08-17T17:44:21.455389+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:44:21.455389+0000","last_undegraded":"2024-08-17T17:44:21.455389+0000","last_fullsized":"2024-08-17T17:44:21.455389+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":550,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459947+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:44:21.459947+0000","last_peered":"2024-08-17T17:44:21.459947+0000","last_clean":"2024-08-17T17:44:21.459947+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:44:21.459947+0000","last_undegraded":"2024-08-17T17:44:21.459947+0000","last_fullsized":"2024-08-17T17:44:21.459947+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"399'12403","reported_seq":20982,"reported_epoch":399,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:24.002639+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:44:24.002639+0000","last_peered":"2024-08-17T17:44:24.002639+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:44:24.002639+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:44:24.002639+0000","mapping_epoch":337,"log_start":"329'6844","ondisk_log_start":"329'6844","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315736,"num_objects":2493,"num_object_clones":0,"num_object_copies":7479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2493,"num_objects_misplaced":3243,"num_objects_unfound":0,"num_objects_dirty":2493,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12403,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4205,"num_bytes_recovered":275513344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2493}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":744,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:21.456562+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:44:21.456562+0000","last_peered":"2024-08-17T17:44:21.456562+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:44:21.456562+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":328,"reported_epoch":398,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:44:10.523204+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:44:10.523204+0000","last_peered":"2024-08-17T17:44:10.523204+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:44:10.523204+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:44:10.523204+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":593,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.455198+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:44:21.455198+0000","last_peered":"2024-08-17T17:44:21.455198+0000","last_clean":"2024-08-17T17:44:21.455198+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:44:21.455198+0000","last_undegraded":"2024-08-17T17:44:21.455198+0000","last_fullsized":"2024-08-17T17:44:21.455198+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":363,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.477131+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:44:10.477131+0000","last_peered":"2024-08-17T17:44:10.477131+0000","last_clean":"2024-08-17T17:44:10.477131+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:44:10.477131+0000","last_undegraded":"2024-08-17T17:44:10.477131+0000","last_fullsized":"2024-08-17T17:44:10.477131+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"399'12202","reported_seq":12384,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:24.546745+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:44:24.546745+0000","last_peered":"2024-08-17T17:44:24.546745+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:44:24.546745+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'5221","ondisk_log_start":"327'5221","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166592512,"num_objects":2542,"num_object_clones":0,"num_object_copies":7626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12202,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-4105},{"shards":"0,4,7","objects":2542}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":409,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.453078+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:44:21.453078+0000","last_peered":"2024-08-17T17:44:21.453078+0000","last_clean":"2024-08-17T17:44:21.453078+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:44:21.453078+0000","last_undegraded":"2024-08-17T17:44:21.453078+0000","last_fullsized":"2024-08-17T17:44:21.453078+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":335,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470439+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:44:10.470439+0000","last_peered":"2024-08-17T17:44:10.470439+0000","last_clean":"2024-08-17T17:44:10.470439+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:44:10.470439+0000","last_undegraded":"2024-08-17T17:44:10.470439+0000","last_fullsized":"2024-08-17T17:44:10.470439+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":335,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458361+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:44:21.458361+0000","last_peered":"2024-08-17T17:44:21.458361+0000","last_clean":"2024-08-17T17:44:21.458361+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:44:21.458361+0000","last_undegraded":"2024-08-17T17:44:21.458361+0000","last_fullsized":"2024-08-17T17:44:21.458361+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"399'12390","reported_seq":12488,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:27.028734+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:44:27.028734+0000","last_peered":"2024-08-17T17:44:27.028734+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:44:27.028734+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"327'5817","ondisk_log_start":"327'5817","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12390,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2406},{"shards":"5,6","objects":-4890}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":501,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454127+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:44:21.454127+0000","last_peered":"2024-08-17T17:44:21.454127+0000","last_clean":"2024-08-17T17:44:21.454127+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:44:21.454127+0000","last_undegraded":"2024-08-17T17:44:21.454127+0000","last_fullsized":"2024-08-17T17:44:21.454127+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":135,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.476253+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:44:10.476253+0000","last_peered":"2024-08-17T17:44:10.476253+0000","last_clean":"2024-08-17T17:44:10.476253+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:44:10.476253+0000","last_undegraded":"2024-08-17T17:44:10.476253+0000","last_fullsized":"2024-08-17T17:44:10.476253+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":685,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.452359+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:44:21.452359+0000","last_peered":"2024-08-17T17:44:21.452359+0000","last_clean":"2024-08-17T17:44:21.452359+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:44:21.452359+0000","last_undegraded":"2024-08-17T17:44:21.452359+0000","last_fullsized":"2024-08-17T17:44:21.452359+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"399'12150","reported_seq":13056,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:24.771664+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:44:24.771664+0000","last_peered":"2024-08-17T17:44:24.771664+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:44:24.771664+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"327'5116","ondisk_log_start":"327'5116","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12150,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2496},{"shards":"0,5","objects":-4158}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":477,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.452624+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:44:21.452624+0000","last_peered":"2024-08-17T17:44:21.452624+0000","last_clean":"2024-08-17T17:44:21.452624+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:44:21.452624+0000","last_undegraded":"2024-08-17T17:44:21.452624+0000","last_fullsized":"2024-08-17T17:44:21.452624+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":374,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470398+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:44:10.470398+0000","last_peered":"2024-08-17T17:44:10.470398+0000","last_clean":"2024-08-17T17:44:10.470398+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:44:10.470398+0000","last_undegraded":"2024-08-17T17:44:10.470398+0000","last_fullsized":"2024-08-17T17:44:10.470398+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":272,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454483+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:44:21.454483+0000","last_peered":"2024-08-17T17:44:21.454483+0000","last_clean":"2024-08-17T17:44:21.454483+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:44:21.454483+0000","last_undegraded":"2024-08-17T17:44:21.454483+0000","last_fullsized":"2024-08-17T17:44:21.454483+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"399'12351","reported_seq":27377,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:27.785010+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:44:27.785010+0000","last_peered":"2024-08-17T17:44:27.785010+0000","last_clean":"2024-08-17T17:44:27.785010+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:44:27.785010+0000","last_undegraded":"2024-08-17T17:44:27.785010+0000","last_fullsized":"2024-08-17T17:44:27.785010+0000","mapping_epoch":355,"log_start":"328'6030","ondisk_log_start":"328'6030","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12351,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":313,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523412+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:44:10.523412+0000","last_peered":"2024-08-17T17:44:10.523412+0000","last_clean":"2024-08-17T17:44:10.523412+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:44:10.523412+0000","last_undegraded":"2024-08-17T17:44:10.523412+0000","last_fullsized":"2024-08-17T17:44:10.523412+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454399+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:44:21.454399+0000","last_peered":"2024-08-17T17:44:21.454399+0000","last_clean":"2024-08-17T17:44:21.454399+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:44:21.454399+0000","last_undegraded":"2024-08-17T17:44:21.454399+0000","last_fullsized":"2024-08-17T17:44:21.454399+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":636,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.452859+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:44:21.452859+0000","last_peered":"2024-08-17T17:44:21.452859+0000","last_clean":"2024-08-17T17:44:21.452859+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:44:21.452859+0000","last_undegraded":"2024-08-17T17:44:21.452859+0000","last_fullsized":"2024-08-17T17:44:21.452859+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"399'12046","reported_seq":26639,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:27.684801+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:44:27.684801+0000","last_peered":"2024-08-17T17:44:27.684801+0000","last_clean":"2024-08-17T17:44:27.684801+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:44:27.684801+0000","last_undegraded":"2024-08-17T17:44:27.684801+0000","last_fullsized":"2024-08-17T17:44:27.684801+0000","mapping_epoch":345,"log_start":"327'5772","ondisk_log_start":"327'5772","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6274,"log_dups_size":0,"ondisk_log_size":6274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154533888,"num_objects":2358,"num_object_clones":0,"num_object_copies":7074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12046,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":309,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523260+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:44:10.523260+0000","last_peered":"2024-08-17T17:44:10.523260+0000","last_clean":"2024-08-17T17:44:10.523260+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:44:10.523260+0000","last_undegraded":"2024-08-17T17:44:10.523260+0000","last_fullsized":"2024-08-17T17:44:10.523260+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":392,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457128+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:44:21.457128+0000","last_peered":"2024-08-17T17:44:21.457128+0000","last_clean":"2024-08-17T17:44:21.457128+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:44:21.457128+0000","last_undegraded":"2024-08-17T17:44:21.457128+0000","last_fullsized":"2024-08-17T17:44:21.457128+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":451,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523360+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:44:10.523360+0000","last_peered":"2024-08-17T17:44:10.523360+0000","last_clean":"2024-08-17T17:44:10.523360+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:44:10.523360+0000","last_undegraded":"2024-08-17T17:44:10.523360+0000","last_fullsized":"2024-08-17T17:44:10.523360+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"399'12231","reported_seq":12432,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:27.648516+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:44:27.648516+0000","last_peered":"2024-08-17T17:44:27.648516+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:44:27.648516+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"328'5887","ondisk_log_start":"328'5887","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12231,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":2553},{"shards":"0,3","objects":-4695}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":666,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.460038+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:44:21.460038+0000","last_peered":"2024-08-17T17:44:21.460038+0000","last_clean":"2024-08-17T17:44:21.460038+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:44:21.460038+0000","last_undegraded":"2024-08-17T17:44:21.460038+0000","last_fullsized":"2024-08-17T17:44:21.460038+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":634,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.452675+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:44:21.452675+0000","last_peered":"2024-08-17T17:44:21.452675+0000","last_clean":"2024-08-17T17:44:21.452675+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:44:21.452675+0000","last_undegraded":"2024-08-17T17:44:21.452675+0000","last_fullsized":"2024-08-17T17:44:21.452675+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":652,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.455224+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:44:21.455224+0000","last_peered":"2024-08-17T17:44:21.455224+0000","last_clean":"2024-08-17T17:44:21.455224+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:44:21.455224+0000","last_undegraded":"2024-08-17T17:44:21.455224+0000","last_fullsized":"2024-08-17T17:44:21.455224+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"399'12114","reported_seq":26810,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:27.036390+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:44:27.036390+0000","last_peered":"2024-08-17T17:44:27.036390+0000","last_clean":"2024-08-17T17:44:27.036390+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:44:27.036390+0000","last_undegraded":"2024-08-17T17:44:27.036390+0000","last_fullsized":"2024-08-17T17:44:27.036390+0000","mapping_epoch":347,"log_start":"327'5572","ondisk_log_start":"327'5572","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161218560,"num_objects":2460,"num_object_clones":0,"num_object_copies":7380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12114,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":325,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457376+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:44:21.457376+0000","last_peered":"2024-08-17T17:44:21.457376+0000","last_clean":"2024-08-17T17:44:21.457376+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:44:21.457376+0000","last_undegraded":"2024-08-17T17:44:21.457376+0000","last_fullsized":"2024-08-17T17:44:21.457376+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":403,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.478101+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:44:10.478101+0000","last_peered":"2024-08-17T17:44:10.478101+0000","last_clean":"2024-08-17T17:44:10.478101+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:44:10.478101+0000","last_undegraded":"2024-08-17T17:44:10.478101+0000","last_fullsized":"2024-08-17T17:44:10.478101+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":353,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469707+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:44:10.469707+0000","last_peered":"2024-08-17T17:44:10.469707+0000","last_clean":"2024-08-17T17:44:10.469707+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:44:10.469707+0000","last_undegraded":"2024-08-17T17:44:10.469707+0000","last_fullsized":"2024-08-17T17:44:10.469707+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":353,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470263+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:44:10.470263+0000","last_peered":"2024-08-17T17:44:10.470263+0000","last_clean":"2024-08-17T17:44:10.470263+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:44:10.470263+0000","last_undegraded":"2024-08-17T17:44:10.470263+0000","last_fullsized":"2024-08-17T17:44:10.470263+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":396,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470338+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:44:10.470338+0000","last_peered":"2024-08-17T17:44:10.470338+0000","last_clean":"2024-08-17T17:44:10.470338+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:44:10.470338+0000","last_undegraded":"2024-08-17T17:44:10.470338+0000","last_fullsized":"2024-08-17T17:44:10.470338+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":641,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.452724+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:44:21.452724+0000","last_peered":"2024-08-17T17:44:21.452724+0000","last_clean":"2024-08-17T17:44:21.452724+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:44:21.452724+0000","last_undegraded":"2024-08-17T17:44:21.452724+0000","last_fullsized":"2024-08-17T17:44:21.452724+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":181,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459810+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:44:21.459810+0000","last_peered":"2024-08-17T17:44:21.459810+0000","last_clean":"2024-08-17T17:44:21.459810+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:44:21.459810+0000","last_undegraded":"2024-08-17T17:44:21.459810+0000","last_fullsized":"2024-08-17T17:44:21.459810+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":303,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523524+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:44:10.523524+0000","last_peered":"2024-08-17T17:44:10.523524+0000","last_clean":"2024-08-17T17:44:10.523524+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:44:10.523524+0000","last_undegraded":"2024-08-17T17:44:10.523524+0000","last_fullsized":"2024-08-17T17:44:10.523524+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":599,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.453793+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:44:21.453793+0000","last_peered":"2024-08-17T17:44:21.453793+0000","last_clean":"2024-08-17T17:44:21.453793+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:44:21.453793+0000","last_undegraded":"2024-08-17T17:44:21.453793+0000","last_fullsized":"2024-08-17T17:44:21.453793+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":447,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.523654+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:44:10.523654+0000","last_peered":"2024-08-17T17:44:10.523654+0000","last_clean":"2024-08-17T17:44:10.523654+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:44:10.523654+0000","last_undegraded":"2024-08-17T17:44:10.523654+0000","last_fullsized":"2024-08-17T17:44:10.523654+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":593,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.455135+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:44:21.455135+0000","last_peered":"2024-08-17T17:44:21.455135+0000","last_clean":"2024-08-17T17:44:21.455135+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:44:21.455135+0000","last_undegraded":"2024-08-17T17:44:21.455135+0000","last_fullsized":"2024-08-17T17:44:21.455135+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":370,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.478033+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:44:10.478033+0000","last_peered":"2024-08-17T17:44:10.478033+0000","last_clean":"2024-08-17T17:44:10.478033+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:44:10.478033+0000","last_undegraded":"2024-08-17T17:44:10.478033+0000","last_fullsized":"2024-08-17T17:44:10.478033+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":625,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.455302+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:44:21.455302+0000","last_peered":"2024-08-17T17:44:21.455302+0000","last_clean":"2024-08-17T17:44:21.455302+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:44:21.455302+0000","last_undegraded":"2024-08-17T17:44:21.455302+0000","last_fullsized":"2024-08-17T17:44:21.455302+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":341,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.461142+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:44:21.461142+0000","last_peered":"2024-08-17T17:44:21.461142+0000","last_clean":"2024-08-17T17:44:21.461142+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:44:21.461142+0000","last_undegraded":"2024-08-17T17:44:21.461142+0000","last_fullsized":"2024-08-17T17:44:21.461142+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":633,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.451833+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:44:21.451833+0000","last_peered":"2024-08-17T17:44:21.451833+0000","last_clean":"2024-08-17T17:44:21.451833+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:44:21.451833+0000","last_undegraded":"2024-08-17T17:44:21.451833+0000","last_fullsized":"2024-08-17T17:44:21.451833+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":592,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457765+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:44:21.457765+0000","last_peered":"2024-08-17T17:44:21.457765+0000","last_clean":"2024-08-17T17:44:21.457765+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:44:21.457765+0000","last_undegraded":"2024-08-17T17:44:21.457765+0000","last_fullsized":"2024-08-17T17:44:21.457765+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":382,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469980+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:44:10.469980+0000","last_peered":"2024-08-17T17:44:10.469980+0000","last_clean":"2024-08-17T17:44:10.469980+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:44:10.469980+0000","last_undegraded":"2024-08-17T17:44:10.469980+0000","last_fullsized":"2024-08-17T17:44:10.469980+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":281,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454647+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:44:21.454647+0000","last_peered":"2024-08-17T17:44:21.454647+0000","last_clean":"2024-08-17T17:44:21.454647+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:44:21.454647+0000","last_undegraded":"2024-08-17T17:44:21.454647+0000","last_fullsized":"2024-08-17T17:44:21.454647+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":337,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457020+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:44:21.457020+0000","last_peered":"2024-08-17T17:44:21.457020+0000","last_clean":"2024-08-17T17:44:21.457020+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:44:21.457020+0000","last_undegraded":"2024-08-17T17:44:21.457020+0000","last_fullsized":"2024-08-17T17:44:21.457020+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":390,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470193+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:44:10.470193+0000","last_peered":"2024-08-17T17:44:10.470193+0000","last_clean":"2024-08-17T17:44:10.470193+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:44:10.470193+0000","last_undegraded":"2024-08-17T17:44:10.470193+0000","last_fullsized":"2024-08-17T17:44:10.470193+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":253,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.470246+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:44:10.470246+0000","last_peered":"2024-08-17T17:44:10.470246+0000","last_clean":"2024-08-17T17:44:10.470246+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:44:10.470246+0000","last_undegraded":"2024-08-17T17:44:10.470246+0000","last_fullsized":"2024-08-17T17:44:10.470246+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":390,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469839+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:44:10.469839+0000","last_peered":"2024-08-17T17:44:10.469839+0000","last_clean":"2024-08-17T17:44:10.469839+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:44:10.469839+0000","last_undegraded":"2024-08-17T17:44:10.469839+0000","last_fullsized":"2024-08-17T17:44:10.469839+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":253,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469872+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:44:10.469872+0000","last_peered":"2024-08-17T17:44:10.469872+0000","last_clean":"2024-08-17T17:44:10.469872+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:44:10.469872+0000","last_undegraded":"2024-08-17T17:44:10.469872+0000","last_fullsized":"2024-08-17T17:44:10.469872+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":645,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.454211+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:44:21.454211+0000","last_peered":"2024-08-17T17:44:21.454211+0000","last_clean":"2024-08-17T17:44:21.454211+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:44:21.454211+0000","last_undegraded":"2024-08-17T17:44:21.454211+0000","last_fullsized":"2024-08-17T17:44:21.454211+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":618,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.461315+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:44:21.461315+0000","last_peered":"2024-08-17T17:44:21.461315+0000","last_clean":"2024-08-17T17:44:21.461315+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:44:21.461315+0000","last_undegraded":"2024-08-17T17:44:21.461315+0000","last_fullsized":"2024-08-17T17:44:21.461315+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":353,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-08-17T17:44:10.469518+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:44:10.469518+0000","last_peered":"2024-08-17T17:44:10.469518+0000","last_clean":"2024-08-17T17:44:10.469518+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:44:10.469518+0000","last_undegraded":"2024-08-17T17:44:10.469518+0000","last_fullsized":"2024-08-17T17:44:10.469518+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":340,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457451+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:44:21.457451+0000","last_peered":"2024-08-17T17:44:21.457451+0000","last_clean":"2024-08-17T17:44:21.457451+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:44:21.457451+0000","last_undegraded":"2024-08-17T17:44:21.457451+0000","last_fullsized":"2024-08-17T17:44:21.457451+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2627993624,"num_objects":40101,"num_object_clones":0,"num_object_copies":120303,"num_objects_missing_on_primary":3174,"num_objects_missing":3174,"num_objects_degraded":49393,"num_objects_misplaced":16609,"num_objects_unfound":0,"num_objects_dirty":40101,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196289,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34777,"num_bytes_recovered":2156593152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10634342400,"data_stored":10634330184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104708,"ondisk_log_size":104708,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170136,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463506,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2308744,"kb_used_data":1122496,"kb_used_omap":1,"kb_used_meta":1186238,"kb_avail":91436408,"statfs":{"total":95995035648,"available":93630881792,"internally_reserved":0,"allocated":1149435904,"data_stored":1147226258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1214708056},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492180,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":722348,"kb_used_data":322852,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":93022804,"statfs":{"total":95995035648,"available":95255351296,"internally_reserved":0,"allocated":330600448,"data_stored":328413102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986771,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2144828,"kb_used_data":640308,"kb_used_omap":1,"kb_used_meta":1504510,"kb_avail":91600324,"statfs":{"total":95995035648,"available":93798731776,"internally_reserved":0,"allocated":655675392,"data_stored":653476686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1540618584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,1,0,2,1,8,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225600,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2265260,"kb_used_data":1287076,"kb_used_omap":4,"kb_used_meta":978171,"kb_avail":91479892,"statfs":{"total":95995035648,"available":93675409408,"internally_reserved":0,"allocated":1317965824,"data_stored":1315778776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1001647552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075924,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533817,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2219440,"kb_used_data":1145512,"kb_used_omap":5,"kb_used_meta":1073914,"kb_avail":91525712,"statfs":{"total":95995035648,"available":93722329088,"internally_reserved":0,"allocated":1173004288,"data_stored":1170811918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1099688176},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089794,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2015560,"kb_used_data":984128,"kb_used_omap":5,"kb_used_meta":1031418,"kb_avail":91729592,"statfs":{"total":95995035648,"available":93931102208,"internally_reserved":0,"allocated":1007747072,"data_stored":1005537485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1056172091},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1000210432,"data_stored":1000210432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1164382208,"data_stored":1164378136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1306787840,"data_stored":1306787840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":650182656,"data_stored":650182656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":323616768,"data_stored":323616768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1142095872,"data_stored":1142095872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1707806720,"data_stored":1707802648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:44:32.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:32.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:32.493 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:44:32.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:32.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:32.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/427156418' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:32.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/427156418' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:32.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:32.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:32.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:32.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:32.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/427156418' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:32.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:32.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:32.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:33.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:33.033 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:33.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:33.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:33.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:33.282 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:33.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:33.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:33.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:33.510 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:33.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[106729]: pgmap v1558: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 206 op/s; 49287/118245 objects degraded (41.682%); 16302/118245 objects misplaced (13.787%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:44:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[106729]: osdmap e400: 8 total, 8 up, 6 in 2024-08-17T17:44:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[103614]: pgmap v1558: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 206 op/s; 49287/118245 objects degraded (41.682%); 16302/118245 objects misplaced (13.787%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:44:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:33 smithi142 ceph-mon[103614]: osdmap e400: 8 total, 8 up, 6 in 2024-08-17T17:44:33.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:33.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:33.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:33.780 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:33 smithi152 ceph-mon[93456]: pgmap v1558: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 206 op/s; 49287/118245 objects degraded (41.682%); 16302/118245 objects misplaced (13.787%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:33 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:44:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:33 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:33 smithi152 ceph-mon[93456]: osdmap e400: 8 total, 8 up, 6 in 2024-08-17T17:44:33.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:33.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:33.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:34.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:34.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:34.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:34.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:34.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:34.155 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:34.155 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:34.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:34.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:34.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:34.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:34.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:34.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:34.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:34.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:34.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:34.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:34.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:34.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:34.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:34.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:34.909 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:34.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:34.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:35.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:35.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:35.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:35.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:35.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:35.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:35.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:35.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:44:35.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:35.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:35.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:35.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:35.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:35.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:35.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:35.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:35.725 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:35 smithi142 ceph-mon[106729]: pgmap v1560: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49240/117435 objects degraded (41.930%); 16177/117435 objects misplaced (13.775%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:35.725 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:35 smithi142 ceph-mon[103614]: pgmap v1560: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49240/117435 objects degraded (41.930%); 16177/117435 objects misplaced (13.775%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:35.738 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:44:35.738 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:35.747 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:44:35.748 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:35.763 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:35.766 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:44:35.766 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:35.783 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:44:35.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:35.785 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:35.799 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:44:35.800 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:35.815 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:44:35.815 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:35 smithi152 ceph-mon[93456]: pgmap v1560: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49240/117435 objects degraded (41.930%); 16177/117435 objects misplaced (13.775%); 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:44:35.833 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:44:35.834 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:35.848 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:44:35.849 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:35.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:35.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:35.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:36.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:36.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:36.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:36.162 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:36.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:36.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:36.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:36.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:36.513 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:36.519 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:44:36.535 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:36.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:36.660 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:36.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:44:36.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2548371829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:36.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49240/117435 objects degraded (41.930%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:36.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2548371829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:36.744 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49240/117435 objects degraded (41.930%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:36.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:36.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:36.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:36.798 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:36.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2548371829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49240/117435 objects degraded (41.930%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:36.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:36.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:36.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:37.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:37.040 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:37.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:37.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:37.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:37.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:37.369 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:37.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:37.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:37.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:37.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:37.589 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:37 smithi142 ceph-mon[106729]: pgmap v1561: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 49172/116340 objects degraded (42.266%); 15997/116340 objects misplaced (13.750%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:37.589 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:37 smithi142 ceph-mon[103614]: pgmap v1561: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 49172/116340 objects degraded (42.266%); 15997/116340 objects misplaced (13.750%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:37.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:37.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:37.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:37.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:37 smithi152 ceph-mon[93456]: pgmap v1561: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 49172/116340 objects degraded (42.266%); 15997/116340 objects misplaced (13.750%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:37.890 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:37.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:37.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:37.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:38.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:38.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:38.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:38.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:38.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:38.299 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:38.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:38.427 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:38.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:38.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:38.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:38.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:38.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:38.677 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:38.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:38.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:38.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:38.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:38.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:38.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:39.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:39.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:39.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:39.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:39.238 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:39.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:39.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:39.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:44:39.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:39.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:39.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:39.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:39.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:39 smithi142 ceph-mon[103614]: pgmap v1562: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 178 op/s; 49172/115026 objects degraded (42.749%); 15932/115026 objects misplaced (13.851%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:39 smithi142 ceph-mon[106729]: pgmap v1562: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 178 op/s; 49172/115026 objects degraded (42.749%); 15932/115026 objects misplaced (13.851%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:39.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:39.680 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:39.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:39.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:39.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:39 smithi152 ceph-mon[93456]: pgmap v1562: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 178 op/s; 49172/115026 objects degraded (42.749%); 15932/115026 objects misplaced (13.851%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:44:39.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:39.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:39.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:39.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:40.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:40.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:40.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:40.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:40.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:40.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:40.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:40.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:40.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:40.428 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:40.530 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:40.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:44:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:44:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:40.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:40.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:40.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:40.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:40.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:40.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:40.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:40.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:41.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:41.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:41.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:41.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:41.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:41.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:41.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:41.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:41.513 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:44:41.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:41.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[106729]: pgmap v1563: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49113/114090 objects degraded (43.048%); 15810/114090 objects misplaced (13.857%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 49113/114090 objects degraded (43.048%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2501756189' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:41.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[103614]: pgmap v1563: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49113/114090 objects degraded (43.048%); 15810/114090 objects misplaced (13.857%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 49113/114090 objects degraded (43.048%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2501756189' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:41.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:41.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:41.692 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:41.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:41.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:41.816 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:41 smithi152 ceph-mon[93456]: pgmap v1563: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 182 op/s; 49113/114090 objects degraded (43.048%); 15810/114090 objects misplaced (13.857%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 49113/114090 objects degraded (43.048%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2501756189' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:41 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:44:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:41.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:41.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:41.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:41.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:42.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:42.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:44:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:44:41] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-08-17T17:44:42.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:42.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:42.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:42.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:42.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:42.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:42.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:42.555 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:42 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:42.555 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:42 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:42.555 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:42 smithi142 ceph-mon[106729]: osdmap e401: 8 total, 8 up, 6 in 2024-08-17T17:44:42.555 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:42 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:42.555 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:42 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:42.556 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:42 smithi142 ceph-mon[103614]: osdmap e401: 8 total, 8 up, 6 in 2024-08-17T17:44:42.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:42.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:42.697 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:42.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:42.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:42.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:42 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:42 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:44:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:42 smithi152 ceph-mon[93456]: osdmap e401: 8 total, 8 up, 6 in 2024-08-17T17:44:42.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:42.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:42.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:43.050 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:43.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:43.175 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.202 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:43.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:43.229 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:44:43.230 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:44:43.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:43.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:43.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:43.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:43.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:43.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:43.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:43.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:43.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:43.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:43.763 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:43 smithi142 ceph-mon[106729]: pgmap v1565: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 49044/112599 objects degraded (43.556%); 15559/112599 objects misplaced (13.818%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:43.763 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:43 smithi142 ceph-mon[103614]: pgmap v1565: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 49044/112599 objects degraded (43.556%); 15559/112599 objects misplaced (13.818%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:43.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:43 smithi152 ceph-mon[93456]: pgmap v1565: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 49044/112599 objects degraded (43.556%); 15559/112599 objects misplaced (13.818%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:44:43.829 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:43.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:43.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:43.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:44.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:44.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:44.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:44.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:44.302 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:44.310 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:44:44.311 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:44:44.311 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:44:44.311 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:44:44.311 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:44:44.321 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:44.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:44.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:44.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:44.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:44.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:44.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:44.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:44.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:44.806 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:44 smithi142 ceph-mon[106729]: from='client.37947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:44.807 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:44 smithi142 ceph-mon[103614]: from='client.37947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:44.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:44 smithi152 ceph-mon[93456]: from='client.37947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:44:44.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:44.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:44.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:44.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:45.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:45.076 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:45.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:45.203 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:45.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:45.305 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:45.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:45.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:45.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:44:45.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:45.460 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:45.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:45.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:44:45.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:45.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:45.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:45 smithi142 ceph-mon[103614]: pgmap v1566: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 48972/111585 objects degraded (43.888%); 15416/111585 objects misplaced (13.815%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:44:45.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:45 smithi142 ceph-mon[106729]: pgmap v1566: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 48972/111585 objects degraded (43.888%); 15416/111585 objects misplaced (13.815%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:44:45.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:45.732 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:45.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:45.818 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1565,"stamp":"2024-08-17T17:44:42.398363+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2458589157,"num_objects":37533,"num_object_clones":0,"num_object_copies":112599,"num_objects_missing_on_primary":2973,"num_objects_missing":2973,"num_objects_degraded":49044,"num_objects_misplaced":15559,"num_objects_unfound":0,"num_objects_dirty":37533,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":199161,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35335,"num_bytes_recovered":2192843406,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105009,"ondisk_log_size":105009,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11293640,"kb_used_data":5157268,"kb_used_omap":27,"kb_used_meta":6136292,"kb_avail":551177272,"statfs":{"total":575970213888,"available":564405526528,"internally_reserved":0,"allocated":5281042432,"data_stored":5267803178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6283563752},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,0,3,0,1,2,4,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-123338752,"num_objects":-1882,"num_object_clones":0,"num_object_copies":-5646,"num_objects_missing_on_primary":-136,"num_objects_missing":-136,"num_objects_degraded":-243,"num_objects_misplaced":-743,"num_objects_unfound":0,"num_objects_dirty":-1882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1882,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":354,"num_bytes_recovered":23199744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.908998"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":644,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491206+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:44:32.491206+0000","last_peered":"2024-08-17T17:44:32.491206+0000","last_clean":"2024-08-17T17:44:32.491206+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:44:32.491206+0000","last_undegraded":"2024-08-17T17:44:32.491206+0000","last_fullsized":"2024-08-17T17:44:32.491206+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":109,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459436+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:44:21.459436+0000","last_peered":"2024-08-17T17:44:21.459436+0000","last_clean":"2024-08-17T17:44:21.459436+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:44:21.459436+0000","last_undegraded":"2024-08-17T17:44:21.459436+0000","last_fullsized":"2024-08-17T17:44:21.459436+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":303,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466644+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:44:21.466644+0000","last_peered":"2024-08-17T17:44:21.466644+0000","last_clean":"2024-08-17T17:44:21.466644+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:44:21.466644+0000","last_undegraded":"2024-08-17T17:44:21.466644+0000","last_fullsized":"2024-08-17T17:44:21.466644+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":342,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.495300+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:44:32.495300+0000","last_peered":"2024-08-17T17:44:32.495300+0000","last_clean":"2024-08-17T17:44:32.495300+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:44:32.495300+0000","last_undegraded":"2024-08-17T17:44:32.495300+0000","last_fullsized":"2024-08-17T17:44:32.495300+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":519,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494487+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:44:32.494487+0000","last_peered":"2024-08-17T17:44:32.494487+0000","last_clean":"2024-08-17T17:44:32.494487+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:44:32.494487+0000","last_undegraded":"2024-08-17T17:44:32.494487+0000","last_fullsized":"2024-08-17T17:44:32.494487+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":252,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458699+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:44:21.458699+0000","last_peered":"2024-08-17T17:44:21.458699+0000","last_clean":"2024-08-17T17:44:21.458699+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:44:21.458699+0000","last_undegraded":"2024-08-17T17:44:21.458699+0000","last_fullsized":"2024-08-17T17:44:21.458699+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":613,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491539+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:44:32.491539+0000","last_peered":"2024-08-17T17:44:32.491539+0000","last_clean":"2024-08-17T17:44:32.491539+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:44:32.491539+0000","last_undegraded":"2024-08-17T17:44:32.491539+0000","last_fullsized":"2024-08-17T17:44:32.491539+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":421,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.503510+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:44:32.503510+0000","last_peered":"2024-08-17T17:44:32.503510+0000","last_clean":"2024-08-17T17:44:32.503510+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:44:32.503510+0000","last_undegraded":"2024-08-17T17:44:32.503510+0000","last_fullsized":"2024-08-17T17:44:32.503510+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"400'12372","reported_seq":21068,"reported_epoch":400,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:44:40.183288+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:44:40.183288+0000","last_peered":"2024-08-17T17:44:40.183288+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:44:40.183288+0000","last_undegraded":"2024-08-17T17:44:40.183288+0000","last_fullsized":"2024-08-17T17:44:40.183288+0000","mapping_epoch":337,"log_start":"327'5463","ondisk_log_start":"327'5463","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829952,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2331,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12372,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4292,"num_bytes_recovered":281280512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2331},{"shards":"4,6,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":105,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.499702+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:44:21.499702+0000","last_peered":"2024-08-17T17:44:21.499702+0000","last_clean":"2024-08-17T17:44:21.499702+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:44:21.499702+0000","last_undegraded":"2024-08-17T17:44:21.499702+0000","last_fullsized":"2024-08-17T17:44:21.499702+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":286,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466310+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:44:21.466310+0000","last_peered":"2024-08-17T17:44:21.466310+0000","last_clean":"2024-08-17T17:44:21.466310+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:44:21.466310+0000","last_undegraded":"2024-08-17T17:44:21.466310+0000","last_fullsized":"2024-08-17T17:44:21.466310+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"401'12649","reported_seq":12748,"reported_epoch":401,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:42.191846+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:44:42.191846+0000","last_peered":"2024-08-17T17:44:42.191846+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:44:42.191846+0000","last_undegraded":"2024-08-17T17:44:42.191846+0000","last_fullsized":"2024-08-17T17:44:42.191846+0000","mapping_epoch":337,"log_start":"328'6082","ondisk_log_start":"328'6082","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158924800,"num_objects":2425,"num_object_clones":0,"num_object_copies":7275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2425,"num_objects_unfound":0,"num_objects_dirty":2425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12649,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2425}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":107,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457927+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:44:21.457927+0000","last_peered":"2024-08-17T17:44:21.457927+0000","last_clean":"2024-08-17T17:44:21.457927+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:44:21.457927+0000","last_undegraded":"2024-08-17T17:44:21.457927+0000","last_fullsized":"2024-08-17T17:44:21.457927+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":371,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458635+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:44:21.458635+0000","last_peered":"2024-08-17T17:44:21.458635+0000","last_clean":"2024-08-17T17:44:21.458635+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:44:21.458635+0000","last_undegraded":"2024-08-17T17:44:21.458635+0000","last_fullsized":"2024-08-17T17:44:21.458635+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"400'12399","reported_seq":12500,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:40.494238+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:44:40.494238+0000","last_peered":"2024-08-17T17:44:40.494238+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:44:40.494238+0000","last_undegraded":"2024-08-17T17:44:40.494238+0000","last_fullsized":"2024-08-17T17:44:40.494238+0000","mapping_epoch":337,"log_start":"327'5553","ondisk_log_start":"327'5553","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153026560,"num_objects":2335,"num_object_clones":0,"num_object_copies":7005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2335,"num_objects_unfound":0,"num_objects_dirty":2335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12399,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2335}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":103,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466591+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:44:21.466591+0000","last_peered":"2024-08-17T17:44:21.466591+0000","last_clean":"2024-08-17T17:44:21.466591+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:44:21.466591+0000","last_undegraded":"2024-08-17T17:44:21.466591+0000","last_fullsized":"2024-08-17T17:44:21.466591+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":382,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.502578+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:44:32.502578+0000","last_peered":"2024-08-17T17:44:32.502578+0000","last_clean":"2024-08-17T17:44:32.502578+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:44:32.502578+0000","last_undegraded":"2024-08-17T17:44:32.502578+0000","last_fullsized":"2024-08-17T17:44:32.502578+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"400'12597","reported_seq":12779,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:39.810986+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:44:39.810986+0000","last_peered":"2024-08-17T17:44:39.810986+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:44:39.810986+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'5613","ondisk_log_start":"327'5613","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154730496,"num_objects":2361,"num_object_clones":0,"num_object_copies":7083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12597,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-4408},{"shards":"0,3,7","objects":2361}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":556,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491334+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:44:32.491334+0000","last_peered":"2024-08-17T17:44:32.491334+0000","last_clean":"2024-08-17T17:44:32.491334+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:44:32.491334+0000","last_undegraded":"2024-08-17T17:44:32.491334+0000","last_fullsized":"2024-08-17T17:44:32.491334+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":322,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.500118+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:44:32.500118+0000","last_peered":"2024-08-17T17:44:32.500118+0000","last_clean":"2024-08-17T17:44:32.500118+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:44:32.500118+0000","last_undegraded":"2024-08-17T17:44:32.500118+0000","last_fullsized":"2024-08-17T17:44:32.500118+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"401'12252","reported_seq":12342,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:42.131017+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:44:42.131017+0000","last_peered":"2024-08-17T17:44:42.131017+0000","last_clean":"2024-08-17T17:44:42.131017+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:44:42.131017+0000","last_undegraded":"2024-08-17T17:44:42.131017+0000","last_fullsized":"2024-08-17T17:44:42.131017+0000","mapping_epoch":319,"log_start":"327'5674","ondisk_log_start":"327'5674","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153485312,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12252,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":646,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.493613+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:44:32.493613+0000","last_peered":"2024-08-17T17:44:32.493613+0000","last_clean":"2024-08-17T17:44:32.493613+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:44:32.493613+0000","last_undegraded":"2024-08-17T17:44:32.493613+0000","last_fullsized":"2024-08-17T17:44:32.493613+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457547+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:44:21.457547+0000","last_peered":"2024-08-17T17:44:21.457547+0000","last_clean":"2024-08-17T17:44:21.457547+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:44:21.457547+0000","last_undegraded":"2024-08-17T17:44:21.457547+0000","last_fullsized":"2024-08-17T17:44:21.457547+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"401'12517","reported_seq":12617,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:42.187994+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:44:42.187994+0000","last_peered":"2024-08-17T17:44:42.187994+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:44:42.187994+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"328'5935","ondisk_log_start":"328'5935","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158269440,"num_objects":2415,"num_object_clones":0,"num_object_copies":7245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12517,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2415},{"shards":"1,6","objects":-4951}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":356,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.502730+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:44:32.502730+0000","last_peered":"2024-08-17T17:44:32.502730+0000","last_clean":"2024-08-17T17:44:32.502730+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:44:32.502730+0000","last_undegraded":"2024-08-17T17:44:32.502730+0000","last_fullsized":"2024-08-17T17:44:32.502730+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":381,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457775+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:44:21.457775+0000","last_peered":"2024-08-17T17:44:21.457775+0000","last_clean":"2024-08-17T17:44:21.457775+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:44:21.457775+0000","last_undegraded":"2024-08-17T17:44:21.457775+0000","last_fullsized":"2024-08-17T17:44:21.457775+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"400'12607","reported_seq":12790,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:44:39.027109+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:44:39.027109+0000","last_peered":"2024-08-17T17:44:39.027109+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:44:39.027109+0000","last_undegraded":"2024-08-17T17:44:39.027109+0000","last_fullsized":"2024-08-17T17:44:39.027109+0000","mapping_epoch":337,"log_start":"329'6961","ondisk_log_start":"329'6961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157483008,"num_objects":2403,"num_object_clones":0,"num_object_copies":7209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2403,"num_objects_unfound":0,"num_objects_dirty":2403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12607,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":2403}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":392,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458824+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:44:21.458824+0000","last_peered":"2024-08-17T17:44:21.458824+0000","last_clean":"2024-08-17T17:44:21.458824+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:44:21.458824+0000","last_undegraded":"2024-08-17T17:44:21.458824+0000","last_fullsized":"2024-08-17T17:44:21.458824+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458044+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:44:21.458044+0000","last_peered":"2024-08-17T17:44:21.458044+0000","last_clean":"2024-08-17T17:44:21.458044+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:44:21.458044+0000","last_undegraded":"2024-08-17T17:44:21.458044+0000","last_fullsized":"2024-08-17T17:44:21.458044+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"400'12424","reported_seq":21275,"reported_epoch":400,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:40.486816+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:44:40.486816+0000","last_peered":"2024-08-17T17:44:40.486816+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:44:40.486816+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:44:40.486816+0000","mapping_epoch":337,"log_start":"327'5541","ondisk_log_start":"327'5541","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149422080,"num_objects":2280,"num_object_clones":0,"num_object_copies":6840,"num_objects_missing_on_primary":2973,"num_objects_missing":2973,"num_objects_degraded":2280,"num_objects_misplaced":2973,"num_objects_unfound":0,"num_objects_dirty":2280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12424,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4367,"num_bytes_recovered":286130176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":2267},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":615,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494237+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:44:32.494237+0000","last_peered":"2024-08-17T17:44:32.494237+0000","last_clean":"2024-08-17T17:44:32.494237+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:44:32.494237+0000","last_undegraded":"2024-08-17T17:44:32.494237+0000","last_fullsized":"2024-08-17T17:44:32.494237+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":552,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491603+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:44:32.491603+0000","last_peered":"2024-08-17T17:44:32.491603+0000","last_clean":"2024-08-17T17:44:32.491603+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:44:32.491603+0000","last_undegraded":"2024-08-17T17:44:32.491603+0000","last_fullsized":"2024-08-17T17:44:32.491603+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"400'12554","reported_seq":21437,"reported_epoch":400,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:44:39.032075+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:44:39.032075+0000","last_peered":"2024-08-17T17:44:39.032075+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:44:39.032075+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:44:39.032075+0000","mapping_epoch":337,"log_start":"329'6944","ondisk_log_start":"329'6944","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419800,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2342,"num_objects_misplaced":3092,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12554,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":285409280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2342}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":746,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:32.496933+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:44:32.496933+0000","last_peered":"2024-08-17T17:44:32.496933+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:44:32.496933+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":329,"reported_epoch":399,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:44:21.466425+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:44:21.466425+0000","last_peered":"2024-08-17T17:44:21.466425+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:44:21.466425+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:44:21.466425+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":595,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.495302+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:44:32.495302+0000","last_peered":"2024-08-17T17:44:32.495302+0000","last_clean":"2024-08-17T17:44:32.495302+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:44:32.495302+0000","last_undegraded":"2024-08-17T17:44:32.495302+0000","last_fullsized":"2024-08-17T17:44:32.495302+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":365,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.504364+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:44:32.504364+0000","last_peered":"2024-08-17T17:44:32.504364+0000","last_clean":"2024-08-17T17:44:32.504364+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:44:32.504364+0000","last_undegraded":"2024-08-17T17:44:32.504364+0000","last_fullsized":"2024-08-17T17:44:32.504364+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"400'12397","reported_seq":12581,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:39.947776+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:44:39.947776+0000","last_peered":"2024-08-17T17:44:39.947776+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:44:39.947776+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'5321","ondisk_log_start":"327'5321","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153812992,"num_objects":2347,"num_object_clones":0,"num_object_copies":7041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12397,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-4300},{"shards":"0,4,7","objects":2347}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":411,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.490703+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:44:32.490703+0000","last_peered":"2024-08-17T17:44:32.490703+0000","last_clean":"2024-08-17T17:44:32.490703+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:44:32.490703+0000","last_undegraded":"2024-08-17T17:44:32.490703+0000","last_fullsized":"2024-08-17T17:44:32.490703+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":336,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458526+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:44:21.458526+0000","last_peered":"2024-08-17T17:44:21.458526+0000","last_clean":"2024-08-17T17:44:21.458526+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:44:21.458526+0000","last_undegraded":"2024-08-17T17:44:21.458526+0000","last_fullsized":"2024-08-17T17:44:21.458526+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":337,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.501140+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:44:32.501140+0000","last_peered":"2024-08-17T17:44:32.501140+0000","last_clean":"2024-08-17T17:44:32.501140+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:44:32.501140+0000","last_undegraded":"2024-08-17T17:44:32.501140+0000","last_fullsized":"2024-08-17T17:44:32.501140+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"401'12559","reported_seq":12656,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:42.115961+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:44:42.115961+0000","last_peered":"2024-08-17T17:44:42.115961+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:44:42.115961+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"328'6017","ondisk_log_start":"328'6017","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146604032,"num_objects":2237,"num_object_clones":0,"num_object_copies":6711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12559,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2237},{"shards":"5,6","objects":-5059}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":503,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.492137+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:44:32.492137+0000","last_peered":"2024-08-17T17:44:32.492137+0000","last_clean":"2024-08-17T17:44:32.492137+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:44:32.492137+0000","last_undegraded":"2024-08-17T17:44:32.492137+0000","last_fullsized":"2024-08-17T17:44:32.492137+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":137,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.503332+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:44:32.503332+0000","last_peered":"2024-08-17T17:44:32.503332+0000","last_clean":"2024-08-17T17:44:32.503332+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:44:32.503332+0000","last_undegraded":"2024-08-17T17:44:32.503332+0000","last_fullsized":"2024-08-17T17:44:32.503332+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":687,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.492187+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:44:32.492187+0000","last_peered":"2024-08-17T17:44:32.492187+0000","last_clean":"2024-08-17T17:44:32.492187+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:44:32.492187+0000","last_undegraded":"2024-08-17T17:44:32.492187+0000","last_fullsized":"2024-08-17T17:44:32.492187+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"400'12333","reported_seq":13241,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:40.015991+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:44:40.015991+0000","last_peered":"2024-08-17T17:44:40.015991+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:44:40.015991+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"327'5316","ondisk_log_start":"327'5316","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584768,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12333,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2313},{"shards":"0,5","objects":-4341}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":479,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491668+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:44:32.491668+0000","last_peered":"2024-08-17T17:44:32.491668+0000","last_clean":"2024-08-17T17:44:32.491668+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:44:32.491668+0000","last_undegraded":"2024-08-17T17:44:32.491668+0000","last_fullsized":"2024-08-17T17:44:32.491668+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":375,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459619+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:44:21.459619+0000","last_peered":"2024-08-17T17:44:21.459619+0000","last_clean":"2024-08-17T17:44:21.459619+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:44:21.459619+0000","last_undegraded":"2024-08-17T17:44:21.459619+0000","last_fullsized":"2024-08-17T17:44:21.459619+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491130+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:44:32.491130+0000","last_peered":"2024-08-17T17:44:32.491130+0000","last_clean":"2024-08-17T17:44:32.491130+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:44:32.491130+0000","last_undegraded":"2024-08-17T17:44:32.491130+0000","last_fullsized":"2024-08-17T17:44:32.491130+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"400'12462","reported_seq":27490,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:37.395510+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:44:37.395510+0000","last_peered":"2024-08-17T17:44:37.395510+0000","last_clean":"2024-08-17T17:44:37.395510+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:44:37.395510+0000","last_undegraded":"2024-08-17T17:44:37.395510+0000","last_fullsized":"2024-08-17T17:44:37.395510+0000","mapping_epoch":355,"log_start":"328'6130","ondisk_log_start":"328'6130","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6332,"log_dups_size":0,"ondisk_log_size":6332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155582464,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12462,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466503+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:44:21.466503+0000","last_peered":"2024-08-17T17:44:21.466503+0000","last_clean":"2024-08-17T17:44:21.466503+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:44:21.466503+0000","last_undegraded":"2024-08-17T17:44:21.466503+0000","last_fullsized":"2024-08-17T17:44:21.466503+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":143,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491035+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:44:32.491035+0000","last_peered":"2024-08-17T17:44:32.491035+0000","last_clean":"2024-08-17T17:44:32.491035+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:44:32.491035+0000","last_undegraded":"2024-08-17T17:44:32.491035+0000","last_fullsized":"2024-08-17T17:44:32.491035+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":638,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.493522+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:44:32.493522+0000","last_peered":"2024-08-17T17:44:32.493522+0000","last_clean":"2024-08-17T17:44:32.493522+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:44:32.493522+0000","last_undegraded":"2024-08-17T17:44:32.493522+0000","last_fullsized":"2024-08-17T17:44:32.493522+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"400'12141","reported_seq":26736,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:37.676163+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:44:37.676163+0000","last_peered":"2024-08-17T17:44:37.676163+0000","last_clean":"2024-08-17T17:44:37.676163+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:44:37.676163+0000","last_undegraded":"2024-08-17T17:44:37.676163+0000","last_fullsized":"2024-08-17T17:44:37.676163+0000","mapping_epoch":345,"log_start":"328'5872","ondisk_log_start":"328'5872","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6269,"log_dups_size":0,"ondisk_log_size":6269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148307968,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12141,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":310,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466588+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:44:21.466588+0000","last_peered":"2024-08-17T17:44:21.466588+0000","last_clean":"2024-08-17T17:44:21.466588+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:44:21.466588+0000","last_undegraded":"2024-08-17T17:44:21.466588+0000","last_fullsized":"2024-08-17T17:44:21.466588+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":394,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.497229+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:44:32.497229+0000","last_peered":"2024-08-17T17:44:32.497229+0000","last_clean":"2024-08-17T17:44:32.497229+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:44:32.497229+0000","last_undegraded":"2024-08-17T17:44:32.497229+0000","last_fullsized":"2024-08-17T17:44:32.497229+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":452,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466360+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:44:21.466360+0000","last_peered":"2024-08-17T17:44:21.466360+0000","last_clean":"2024-08-17T17:44:21.466360+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:44:21.466360+0000","last_undegraded":"2024-08-17T17:44:21.466360+0000","last_fullsized":"2024-08-17T17:44:21.466360+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"400'12333","reported_seq":12536,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:37.799308+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:44:37.799308+0000","last_peered":"2024-08-17T17:44:37.799308+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:44:37.799308+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"328'5987","ondisk_log_start":"328'5987","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6346,"log_dups_size":0,"ondisk_log_size":6346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12333,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":2451},{"shards":"0,3","objects":-4797}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":668,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491677+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:44:32.491677+0000","last_peered":"2024-08-17T17:44:32.491677+0000","last_clean":"2024-08-17T17:44:32.491677+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:44:32.491677+0000","last_undegraded":"2024-08-17T17:44:32.491677+0000","last_fullsized":"2024-08-17T17:44:32.491677+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":637,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.492842+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:44:32.492842+0000","last_peered":"2024-08-17T17:44:32.492842+0000","last_clean":"2024-08-17T17:44:32.492842+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:44:32.492842+0000","last_undegraded":"2024-08-17T17:44:32.492842+0000","last_fullsized":"2024-08-17T17:44:32.492842+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":654,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494066+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:44:32.494066+0000","last_peered":"2024-08-17T17:44:32.494066+0000","last_clean":"2024-08-17T17:44:32.494066+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:44:32.494066+0000","last_undegraded":"2024-08-17T17:44:32.494066+0000","last_fullsized":"2024-08-17T17:44:32.494066+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"401'12287","reported_seq":26985,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:42.211475+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:44:42.211475+0000","last_peered":"2024-08-17T17:44:42.211475+0000","last_clean":"2024-08-17T17:44:42.211475+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:44:42.211475+0000","last_undegraded":"2024-08-17T17:44:42.211475+0000","last_fullsized":"2024-08-17T17:44:42.211475+0000","mapping_epoch":347,"log_start":"327'5672","ondisk_log_start":"327'5672","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149880832,"num_objects":2287,"num_object_clones":0,"num_object_copies":6861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12287,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":327,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.500789+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:44:32.500789+0000","last_peered":"2024-08-17T17:44:32.500789+0000","last_clean":"2024-08-17T17:44:32.500789+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:44:32.500789+0000","last_undegraded":"2024-08-17T17:44:32.500789+0000","last_fullsized":"2024-08-17T17:44:32.500789+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":405,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.504506+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:44:32.504506+0000","last_peered":"2024-08-17T17:44:32.504506+0000","last_clean":"2024-08-17T17:44:32.504506+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:44:32.504506+0000","last_undegraded":"2024-08-17T17:44:32.504506+0000","last_fullsized":"2024-08-17T17:44:32.504506+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458759+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:44:21.458759+0000","last_peered":"2024-08-17T17:44:21.458759+0000","last_clean":"2024-08-17T17:44:21.458759+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:44:21.458759+0000","last_undegraded":"2024-08-17T17:44:21.458759+0000","last_fullsized":"2024-08-17T17:44:21.458759+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459550+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:44:21.459550+0000","last_peered":"2024-08-17T17:44:21.459550+0000","last_clean":"2024-08-17T17:44:21.459550+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:44:21.459550+0000","last_undegraded":"2024-08-17T17:44:21.459550+0000","last_fullsized":"2024-08-17T17:44:21.459550+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":397,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457778+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:44:21.457778+0000","last_peered":"2024-08-17T17:44:21.457778+0000","last_clean":"2024-08-17T17:44:21.457778+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:44:21.457778+0000","last_undegraded":"2024-08-17T17:44:21.457778+0000","last_fullsized":"2024-08-17T17:44:21.457778+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":643,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491755+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:44:32.491755+0000","last_peered":"2024-08-17T17:44:32.491755+0000","last_clean":"2024-08-17T17:44:32.491755+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:44:32.491755+0000","last_undegraded":"2024-08-17T17:44:32.491755+0000","last_fullsized":"2024-08-17T17:44:32.491755+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":183,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494375+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:44:32.494375+0000","last_peered":"2024-08-17T17:44:32.494375+0000","last_clean":"2024-08-17T17:44:32.494375+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:44:32.494375+0000","last_undegraded":"2024-08-17T17:44:32.494375+0000","last_fullsized":"2024-08-17T17:44:32.494375+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":304,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.466537+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:44:21.466537+0000","last_peered":"2024-08-17T17:44:21.466537+0000","last_clean":"2024-08-17T17:44:21.466537+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:44:21.466537+0000","last_undegraded":"2024-08-17T17:44:21.466537+0000","last_fullsized":"2024-08-17T17:44:21.466537+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":601,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491905+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:44:32.491905+0000","last_peered":"2024-08-17T17:44:32.491905+0000","last_clean":"2024-08-17T17:44:32.491905+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:44:32.491905+0000","last_undegraded":"2024-08-17T17:44:32.491905+0000","last_fullsized":"2024-08-17T17:44:32.491905+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":448,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.467927+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:44:21.467927+0000","last_peered":"2024-08-17T17:44:21.467927+0000","last_clean":"2024-08-17T17:44:21.467927+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:44:21.467927+0000","last_undegraded":"2024-08-17T17:44:21.467927+0000","last_fullsized":"2024-08-17T17:44:21.467927+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":595,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.492363+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:44:32.492363+0000","last_peered":"2024-08-17T17:44:32.492363+0000","last_clean":"2024-08-17T17:44:32.492363+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:44:32.492363+0000","last_undegraded":"2024-08-17T17:44:32.492363+0000","last_fullsized":"2024-08-17T17:44:32.492363+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":372,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.504444+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:44:32.504444+0000","last_peered":"2024-08-17T17:44:32.504444+0000","last_clean":"2024-08-17T17:44:32.504444+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:44:32.504444+0000","last_undegraded":"2024-08-17T17:44:32.504444+0000","last_fullsized":"2024-08-17T17:44:32.504444+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":627,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494152+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:44:32.494152+0000","last_peered":"2024-08-17T17:44:32.494152+0000","last_clean":"2024-08-17T17:44:32.494152+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:44:32.494152+0000","last_undegraded":"2024-08-17T17:44:32.494152+0000","last_fullsized":"2024-08-17T17:44:32.494152+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":343,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491852+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:44:32.491852+0000","last_peered":"2024-08-17T17:44:32.491852+0000","last_clean":"2024-08-17T17:44:32.491852+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:44:32.491852+0000","last_undegraded":"2024-08-17T17:44:32.491852+0000","last_fullsized":"2024-08-17T17:44:32.491852+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":635,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.492322+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:44:32.492322+0000","last_peered":"2024-08-17T17:44:32.492322+0000","last_clean":"2024-08-17T17:44:32.492322+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:44:32.492322+0000","last_undegraded":"2024-08-17T17:44:32.492322+0000","last_fullsized":"2024-08-17T17:44:32.492322+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":594,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494254+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:44:32.494254+0000","last_peered":"2024-08-17T17:44:32.494254+0000","last_clean":"2024-08-17T17:44:32.494254+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:44:32.494254+0000","last_undegraded":"2024-08-17T17:44:32.494254+0000","last_fullsized":"2024-08-17T17:44:32.494254+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":383,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458418+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:44:21.458418+0000","last_peered":"2024-08-17T17:44:21.458418+0000","last_clean":"2024-08-17T17:44:21.458418+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:44:21.458418+0000","last_undegraded":"2024-08-17T17:44:21.458418+0000","last_fullsized":"2024-08-17T17:44:21.458418+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":283,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.491270+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:44:32.491270+0000","last_peered":"2024-08-17T17:44:32.491270+0000","last_clean":"2024-08-17T17:44:32.491270+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:44:32.491270+0000","last_undegraded":"2024-08-17T17:44:32.491270+0000","last_fullsized":"2024-08-17T17:44:32.491270+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":340,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.497390+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:44:32.497390+0000","last_peered":"2024-08-17T17:44:32.497390+0000","last_clean":"2024-08-17T17:44:32.497390+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:44:32.497390+0000","last_undegraded":"2024-08-17T17:44:32.497390+0000","last_fullsized":"2024-08-17T17:44:32.497390+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":391,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.459522+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:44:21.459522+0000","last_peered":"2024-08-17T17:44:21.459522+0000","last_clean":"2024-08-17T17:44:21.459522+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:44:21.459522+0000","last_undegraded":"2024-08-17T17:44:21.459522+0000","last_fullsized":"2024-08-17T17:44:21.459522+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":254,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457624+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:44:21.457624+0000","last_peered":"2024-08-17T17:44:21.457624+0000","last_clean":"2024-08-17T17:44:21.457624+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:44:21.457624+0000","last_undegraded":"2024-08-17T17:44:21.457624+0000","last_fullsized":"2024-08-17T17:44:21.457624+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":391,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.458194+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:44:21.458194+0000","last_peered":"2024-08-17T17:44:21.458194+0000","last_clean":"2024-08-17T17:44:21.458194+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:44:21.458194+0000","last_undegraded":"2024-08-17T17:44:21.458194+0000","last_fullsized":"2024-08-17T17:44:21.458194+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":254,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457877+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:44:21.457877+0000","last_peered":"2024-08-17T17:44:21.457877+0000","last_clean":"2024-08-17T17:44:21.457877+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:44:21.457877+0000","last_undegraded":"2024-08-17T17:44:21.457877+0000","last_fullsized":"2024-08-17T17:44:21.457877+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":647,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.492202+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:44:32.492202+0000","last_peered":"2024-08-17T17:44:32.492202+0000","last_clean":"2024-08-17T17:44:32.492202+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:44:32.492202+0000","last_undegraded":"2024-08-17T17:44:32.492202+0000","last_fullsized":"2024-08-17T17:44:32.492202+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":620,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.494570+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:44:32.494570+0000","last_peered":"2024-08-17T17:44:32.494570+0000","last_clean":"2024-08-17T17:44:32.494570+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:44:32.494570+0000","last_undegraded":"2024-08-17T17:44:32.494570+0000","last_fullsized":"2024-08-17T17:44:32.494570+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":354,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-08-17T17:44:21.457391+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:44:21.457391+0000","last_peered":"2024-08-17T17:44:21.457391+0000","last_clean":"2024-08-17T17:44:21.457391+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:44:21.457391+0000","last_undegraded":"2024-08-17T17:44:21.457391+0000","last_fullsized":"2024-08-17T17:44:21.457391+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":342,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-08-17T17:44:32.497618+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:44:32.497618+0000","last_peered":"2024-08-17T17:44:32.497618+0000","last_clean":"2024-08-17T17:44:32.497618+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:44:32.497618+0000","last_undegraded":"2024-08-17T17:44:32.497618+0000","last_fullsized":"2024-08-17T17:44:32.497618+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2457993240,"num_objects":37507,"num_object_clones":0,"num_object_copies":112521,"num_objects_missing_on_primary":2973,"num_objects_missing":2973,"num_objects_degraded":49041,"num_objects_misplaced":15559,"num_objects_unfound":0,"num_objects_dirty":37507,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":198883,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35294,"num_bytes_recovered":2190475264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10222055424,"data_stored":10222043208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104802,"ondisk_log_size":104802,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170139,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463509,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2225452,"kb_used_data":1047520,"kb_used_omap":1,"kb_used_meta":1177918,"kb_avail":91519700,"statfs":{"total":95995035648,"available":93716172800,"internally_reserved":0,"allocated":1072660480,"data_stored":1070435248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1206188376},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492183,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":699324,"kb_used_data":299828,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":93045828,"statfs":{"total":95995035648,"available":95278927872,"internally_reserved":0,"allocated":307023872,"data_stored":304828733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986774,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2093964,"kb_used_data":597764,"kb_used_omap":1,"kb_used_meta":1496190,"kb_avail":91651188,"statfs":{"total":95995035648,"available":93850816512,"internally_reserved":0,"allocated":612110336,"data_stored":609903837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1532098904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,3,0,1,2,4,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225602,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2195708,"kb_used_data":1230004,"kb_used_omap":4,"kb_used_meta":965691,"kb_avail":91549444,"statfs":{"total":95995035648,"available":93746630656,"internally_reserved":0,"allocated":1259524096,"data_stored":1257329255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":988868032},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075927,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533820,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2145280,"kb_used_data":1067192,"kb_used_omap":5,"kb_used_meta":1078074,"kb_avail":91599872,"statfs":{"total":95995035648,"available":93798268928,"internally_reserved":0,"allocated":1092804608,"data_stored":1090604445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1103948016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089797,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1933912,"kb_used_data":914960,"kb_used_omap":5,"kb_used_meta":1018938,"kb_avail":91811240,"statfs":{"total":95995035648,"available":94014709760,"internally_reserved":0,"allocated":936919040,"data_stored":934701660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1043392571},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":928841728,"data_stored":928841728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1083838464,"data_stored":1083834392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1249312768,"data_stored":1249312768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":606339072,"data_stored":606339072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":300548096,"data_stored":300548096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1063256064,"data_stored":1063256064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1650659328,"data_stored":1650655256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:44:45.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:44:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:45 smithi152 ceph-mon[93456]: pgmap v1566: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 48972/111585 objects degraded (43.888%); 15416/111585 objects misplaced (13.815%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T17:44:45.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:45.880 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:45.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:45.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:46.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:46.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:46.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:46.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:46.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:46.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:46.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:46.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:46.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:46.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:46.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:46.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:46.749 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:46.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48972/111585 objects degraded (43.888%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:46.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:46 smithi142 ceph-mon[106729]: pgmap v1567: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 48901/110424 objects degraded (44.285%); 15228/110424 objects misplaced (13.790%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:44:46.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48972/111585 objects degraded (43.888%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:46.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:46 smithi142 ceph-mon[103614]: pgmap v1567: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 48901/110424 objects degraded (44.285%); 15228/110424 objects misplaced (13.790%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:44:46.781 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:46.784 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48972/111585 objects degraded (43.888%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:46 smithi152 ceph-mon[93456]: pgmap v1567: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 48901/110424 objects degraded (44.285%); 15228/110424 objects misplaced (13.790%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:44:46.841 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:46.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:46.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:46.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:46.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:46.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:47.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:47.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:47.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:47.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:47.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:47.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:47.512 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:47.614 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:47.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:47.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:47.885 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:47.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:47.987 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:48.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:48.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:48.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:48.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:48.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:48.384 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:48.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:48.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:48.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:48.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:48.610 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:48.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:48.733 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:48.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:48.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:48.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:48.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:48.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:49.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:49.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:49.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:49.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:49.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:49.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:49.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:49.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:49.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:49.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:49.587 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:49 smithi142 ceph-mon[106729]: pgmap v1568: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 48901/108807 objects degraded (44.943%); 15159/108807 objects misplaced (13.932%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:44:49.588 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:49 smithi142 ceph-mon[103614]: pgmap v1568: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 48901/108807 objects degraded (44.943%); 15159/108807 objects misplaced (13.932%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:44:49.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:49.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:49.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:49.770 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:49 smithi152 ceph-mon[93456]: pgmap v1568: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 48901/108807 objects degraded (44.943%); 15159/108807 objects misplaced (13.932%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:44:49.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:49.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:49.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:49.997 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:50.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:50.057 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:44:50.057 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:44:50.076 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:44:50.077 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:44:50.095 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:44:50.095 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:44:50.117 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:44:50.117 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:44:50.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:50.133 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:44:50.133 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:44:50.145 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:50.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:50.149 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:44:50.149 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:44:50.164 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:44:50.164 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:44:50.180 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:44:50.180 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:44:50.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:50.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:50.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:50.398 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:50.465 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:44:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:44:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:50.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:50.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:50.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:50.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:50.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:50.780 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2246099819' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2246099819' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2246099819' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:44:50.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:50.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:50.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:51.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:51.034 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:51.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:51.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:51.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:51.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:51.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:51.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:51.411 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:51.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:51.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:51.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:51.590 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:51 smithi142 ceph-mon[106729]: pgmap v1569: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 48832/107790 objects degraded (45.303%); 15024/107790 objects misplaced (13.938%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:51.590 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48832/107790 objects degraded (45.303%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:51.590 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:51 smithi142 ceph-mon[103614]: pgmap v1569: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 48832/107790 objects degraded (45.303%); 15024/107790 objects misplaced (13.938%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:51.590 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48832/107790 objects degraded (45.303%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:51.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:51.658 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:51.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:51.782 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:51 smithi152 ceph-mon[93456]: pgmap v1569: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 209 op/s; 48832/107790 objects degraded (45.303%); 15024/107790 objects misplaced (13.938%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48832/107790 objects degraded (45.303%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:51.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:44:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:44:51] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-08-17T17:44:51.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:51.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:51.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:52.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:52.032 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:52.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:52.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:52.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.163 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:52.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:52.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:52.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:52.389 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:52.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:52.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:52.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:52.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:52.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:52.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:52.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:52.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:52.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:52.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:52.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:52.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:53.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:53.049 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:53.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:53.174 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:53.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:53.299 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:53.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:53.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:53.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:53.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:53.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:53 smithi142 ceph-mon[106729]: pgmap v1570: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 48762/106062 objects degraded (45.975%); 14744/106062 objects misplaced (13.901%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:44:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:53 smithi142 ceph-mon[103614]: pgmap v1570: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 48762/106062 objects degraded (45.975%); 14744/106062 objects misplaced (13.901%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:44:53.660 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:53.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:53.778 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:44:53.778 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:44:53.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:53.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:53 smithi152 ceph-mon[93456]: pgmap v1570: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 48762/106062 objects degraded (45.975%); 14744/106062 objects misplaced (13.901%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:44:53.914 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:53.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:53.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:53.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:54.038 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:54.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:54.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:54.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:54.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:54.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:54.410 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:54.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:54.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:54.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:54.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:54.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:54.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:44:54.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:54.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:54.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:54.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:54.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:55.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:55.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:55.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:44:55.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:55.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:55.241 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:44:55.241 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 48685/104871 objects degraded (46.424%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2375,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":401,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":34957,"data_bytes":2289768421,"bytes_used":11198222336,"bytes_avail":564771991552,"bytes_total":575970213888,"degraded_objects":48685,"degraded_total":104871,"degraded_ratio":0.46423701499938019,"misplaced_objects":14580,"misplaced_total":104871,"misplaced_ratio":0.13902794862259346,"recovering_objects_per_sec":39,"recovering_bytes_per_sec":2609771,"recovering_keys_per_sec":0,"num_objects_recovered":478,"num_bytes_recovered":31326208,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":214},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":35,"modified":"2024-08-17T17:43:38.379260+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:44:55.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:55.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:55.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:55.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:55.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:55.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:55.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:55.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:55.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:55.810 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:55 smithi152 ceph-mon[93456]: pgmap v1571: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 48685/104871 objects degraded (46.424%); 14580/104871 objects misplaced (13.903%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3020928636' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:44:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:55 smithi142 ceph-mon[103614]: pgmap v1571: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 48685/104871 objects degraded (46.424%); 14580/104871 objects misplaced (13.903%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3020928636' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:44:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:55 smithi142 ceph-mon[106729]: pgmap v1571: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 48685/104871 objects degraded (46.424%); 14580/104871 objects misplaced (13.903%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:44:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3020928636' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:44:55.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:55.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:55.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:56.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:56.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:56.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:56.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:56.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:56.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:56.393 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:44:56.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:44:56.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:56.449 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:56.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:56.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48685/104871 objects degraded (46.424%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:56.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48685/104871 objects degraded (46.424%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:56.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:56.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:56.692 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:56.705 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:56.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48685/104871 objects degraded (46.424%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:44:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:44:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:56.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:56.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:56.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:56.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:56.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:56.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:57.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:57.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:57.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:57.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:57.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:57.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:57.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:57.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:57.483 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:57.485 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:44:57.585 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:44:57.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:44:57.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:57.730 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:57 smithi152 ceph-mon[93456]: pgmap v1572: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 48614/103509 objects degraded (46.966%); 14341/103509 objects misplaced (13.855%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:44:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1596216594' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:57 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:57.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:57.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:57 smithi142 ceph-mon[106729]: pgmap v1572: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 48614/103509 objects degraded (46.966%); 14341/103509 objects misplaced (13.855%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:44:57.846 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1596216594' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:57.846 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:57 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:57.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:57 smithi142 ceph-mon[103614]: pgmap v1572: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 48614/103509 objects degraded (46.966%); 14341/103509 objects misplaced (13.855%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:44:57.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1596216594' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:57.846 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:57 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:44:57.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:57.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:57.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:57.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:57.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:58.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:58.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:58.209 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:58.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:58.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:58.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:58.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:58.488 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:58.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:58.613 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:58.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:58 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:58 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:58 smithi142 ceph-mon[106729]: osdmap e402: 8 total, 8 up, 6 in 2024-08-17T17:44:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:58 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:58 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:58 smithi142 ceph-mon[103614]: osdmap e402: 8 total, 8 up, 6 in 2024-08-17T17:44:58.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:58.742 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:44:58.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:58 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:44:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:58 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:44:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:58 smithi152 ceph-mon[93456]: osdmap e402: 8 total, 8 up, 6 in 2024-08-17T17:44:58.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:58.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:58.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:44:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:58.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:58.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:44:59.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:44:59.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:44:59.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:59.239 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:59.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:44:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.371 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:59.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:44:59.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:44:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:59.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:44:59.610 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:44:59.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:44:59 smithi142 ceph-mon[106729]: pgmap v1574: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 242 op/s; 48614/101544 objects degraded (47.875%); 14248/101544 objects misplaced (14.031%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:44:59.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:44:59 smithi142 ceph-mon[103614]: pgmap v1574: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 242 op/s; 48614/101544 objects degraded (47.875%); 14248/101544 objects misplaced (14.031%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:44:59.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:44:59.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:44:59.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:44:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:44:59.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:44:59 smithi152 ceph-mon[93456]: pgmap v1574: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 242 op/s; 48614/101544 objects degraded (47.875%); 14248/101544 objects misplaced (14.031%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:44:59.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:44:59.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:44:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:44:59.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:44:59.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:00.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:00.109 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:00.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:00.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:00.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:00.357 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:00.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:00.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:00.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:00.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:00.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:45:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:45:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:00.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:00.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:00.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:00.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:00.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:00.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:00.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:00.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:00.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:00.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:01.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:01.110 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:01.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:01.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:01.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:01.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:01.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:01.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:01.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:01.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:01.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:01.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:01.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:01.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:01.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:01.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:01 smithi142 ceph-mon[106729]: pgmap v1575: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 48538/100425 objects degraded (48.333%); 14100/100425 objects misplaced (14.040%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:45:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48538/100425 objects degraded (48.333%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:01 smithi142 ceph-mon[103614]: pgmap v1575: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 48538/100425 objects degraded (48.333%); 14100/100425 objects misplaced (14.040%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:45:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48538/100425 objects degraded (48.333%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:01.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:45:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:01 smithi152 ceph-mon[93456]: pgmap v1575: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 48538/100425 objects degraded (48.333%); 14100/100425 objects misplaced (14.040%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T17:45:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48538/100425 objects degraded (48.333%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:01.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:01.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:01.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:01.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:02.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:02.087 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:45:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:45:01] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-17T17:45:02.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:02.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:02.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:02.272 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:02.323 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:02.324 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:02.324 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:02.324 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:02.324 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:02.324 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:02.324 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:45:02.337 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:45:02.338 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:02.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:02.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:02.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:02.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:02.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:02.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:02.566 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:45:02.566 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:45:02.566 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:45:02.566 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:45:02.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1112563717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:02.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1112563717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:02.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:02.644 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:02.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:02.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1112563717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:02.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:02.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:02.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:02.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:03.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:03.024 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:03.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:03.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:03.249 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:03.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:03.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:03.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:03.397 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:03.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:03.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:03.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:03.624 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:03.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:03.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:03.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:03 smithi152 ceph-mon[93456]: pgmap v1576: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 251 op/s; 48464/98511 objects degraded (49.197%); 13777/98511 objects misplaced (13.985%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:45:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:03 smithi142 ceph-mon[106729]: pgmap v1576: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 251 op/s; 48464/98511 objects degraded (49.197%); 13777/98511 objects misplaced (13.985%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:45:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:03 smithi142 ceph-mon[103614]: pgmap v1576: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 251 op/s; 48464/98511 objects degraded (49.197%); 13777/98511 objects misplaced (13.985%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:45:03.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:03.897 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:03.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:03.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:04.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:04.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:04.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:04.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:04.274 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:04.375 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:04.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:04.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:04.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:04.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:04.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.649 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:45:04.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:04.662 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:04.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:04.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:04.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:04.900 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:04.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:05.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:05.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:05.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:05.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:05.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:05.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:05.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:05.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:05.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:05.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:05.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:05.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:05.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:05.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:05.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:05.663 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:45:05.666 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:45:05.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:05.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:05.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:05 smithi152 ceph-mon[93456]: pgmap v1577: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 48389/97257 objects degraded (49.754%); 13611/97257 objects misplaced (13.995%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:45:05.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:05 smithi142 ceph-mon[106729]: pgmap v1577: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 48389/97257 objects degraded (49.754%); 13611/97257 objects misplaced (13.995%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:45:05.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:05 smithi142 ceph-mon[103614]: pgmap v1577: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 48389/97257 objects degraded (49.754%); 13611/97257 objects misplaced (13.995%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T17:45:05.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:05.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:05.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:06.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:06.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:06.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:06.150 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:06.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:45:06.172 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:06.274 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:06.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:06.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:06.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:06.419 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:06.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:06.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:06.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:06.680 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:06.686 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1577,"stamp":"2024-08-17T17:45:04.404667+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2123438053,"num_objects":32419,"num_object_clones":0,"num_object_copies":97257,"num_objects_missing_on_primary":2688,"num_objects_missing":2688,"num_objects_degraded":48389,"num_objects_misplaced":13611,"num_objects_unfound":0,"num_objects_dirty":32419,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":204275,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36263,"num_bytes_recovered":2253660814,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105023,"ondisk_log_size":105023,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10612728,"kb_used_data":4451396,"kb_used_omap":27,"kb_used_meta":6161252,"kb_avail":551858184,"statfs":{"total":575970213888,"available":565102780416,"internally_reserved":0,"allocated":4558229504,"data_stored":4544904527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6309122792},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-166330368,"num_objects":-2538,"num_object_clones":0,"num_object_copies":-7614,"num_objects_missing_on_primary":-147,"num_objects_missing":-147,"num_objects_degraded":-296,"num_objects_misplaced":-969,"num_objects_unfound":0,"num_objects_dirty":-2538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2538,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":450,"num_bytes_recovered":29491200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002873"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":648,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491775+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:44:57.491775+0000","last_peered":"2024-08-17T17:44:57.491775+0000","last_clean":"2024-08-17T17:44:57.491775+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:44:57.491775+0000","last_undegraded":"2024-08-17T17:44:57.491775+0000","last_fullsized":"2024-08-17T17:44:57.491775+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":111,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516769+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:44:41.516769+0000","last_peered":"2024-08-17T17:44:41.516769+0000","last_clean":"2024-08-17T17:44:41.516769+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:44:41.516769+0000","last_undegraded":"2024-08-17T17:44:41.516769+0000","last_fullsized":"2024-08-17T17:44:41.516769+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512117+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:44:41.512117+0000","last_peered":"2024-08-17T17:44:41.512117+0000","last_clean":"2024-08-17T17:44:41.512117+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:44:41.512117+0000","last_undegraded":"2024-08-17T17:44:41.512117+0000","last_fullsized":"2024-08-17T17:44:41.512117+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":346,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491435+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:44:57.491435+0000","last_peered":"2024-08-17T17:44:57.491435+0000","last_clean":"2024-08-17T17:44:57.491435+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:44:57.491435+0000","last_undegraded":"2024-08-17T17:44:57.491435+0000","last_fullsized":"2024-08-17T17:44:57.491435+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":523,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492607+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:44:57.492607+0000","last_peered":"2024-08-17T17:44:57.492607+0000","last_clean":"2024-08-17T17:44:57.492607+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:44:57.492607+0000","last_undegraded":"2024-08-17T17:44:57.492607+0000","last_fullsized":"2024-08-17T17:44:57.492607+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":254,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516689+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:44:41.516689+0000","last_peered":"2024-08-17T17:44:41.516689+0000","last_clean":"2024-08-17T17:44:41.516689+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:44:41.516689+0000","last_undegraded":"2024-08-17T17:44:41.516689+0000","last_fullsized":"2024-08-17T17:44:41.516689+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":617,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492855+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:44:57.492855+0000","last_peered":"2024-08-17T17:44:57.492855+0000","last_clean":"2024-08-17T17:44:57.492855+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:44:57.492855+0000","last_undegraded":"2024-08-17T17:44:57.492855+0000","last_fullsized":"2024-08-17T17:44:57.492855+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":422,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.524532+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:44:41.524532+0000","last_peered":"2024-08-17T17:44:41.524532+0000","last_clean":"2024-08-17T17:44:41.524532+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:44:41.524532+0000","last_undegraded":"2024-08-17T17:44:41.524532+0000","last_fullsized":"2024-08-17T17:44:41.524532+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"402'12647","reported_seq":21893,"reported_epoch":402,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:45:00.412469+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:45:00.412469+0000","last_peered":"2024-08-17T17:45:00.412469+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:45:00.412469+0000","last_undegraded":"2024-08-17T17:45:00.412469+0000","last_fullsized":"2024-08-17T17:45:00.412469+0000","mapping_epoch":337,"log_start":"327'5763","ondisk_log_start":"327'5763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134807552,"num_objects":2057,"num_object_clones":0,"num_object_copies":6171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2057,"num_objects_unfound":0,"num_objects_dirty":2057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12647,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4566,"num_bytes_recovered":299237376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2057}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":107,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.541794+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:44:41.541794+0000","last_peered":"2024-08-17T17:44:41.541794+0000","last_clean":"2024-08-17T17:44:41.541794+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:44:41.541794+0000","last_undegraded":"2024-08-17T17:44:41.541794+0000","last_fullsized":"2024-08-17T17:44:41.541794+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":288,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511798+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:44:41.511798+0000","last_peered":"2024-08-17T17:44:41.511798+0000","last_clean":"2024-08-17T17:44:41.511798+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:44:41.511798+0000","last_undegraded":"2024-08-17T17:44:41.511798+0000","last_fullsized":"2024-08-17T17:44:41.511798+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"402'12979","reported_seq":13079,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:02.023274+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:45:02.023274+0000","last_peered":"2024-08-17T17:45:02.023274+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:45:02.023274+0000","last_undegraded":"2024-08-17T17:45:02.023274+0000","last_fullsized":"2024-08-17T17:45:02.023274+0000","mapping_epoch":337,"log_start":"328'6382","ondisk_log_start":"328'6382","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137297920,"num_objects":2095,"num_object_clones":0,"num_object_copies":6285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2095,"num_objects_unfound":0,"num_objects_dirty":2095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12979,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":2095}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":109,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517283+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:44:41.517283+0000","last_peered":"2024-08-17T17:44:41.517283+0000","last_clean":"2024-08-17T17:44:41.517283+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:44:41.517283+0000","last_undegraded":"2024-08-17T17:44:41.517283+0000","last_fullsized":"2024-08-17T17:44:41.517283+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":373,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516615+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:44:41.516615+0000","last_peered":"2024-08-17T17:44:41.516615+0000","last_clean":"2024-08-17T17:44:41.516615+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:44:41.516615+0000","last_undegraded":"2024-08-17T17:44:41.516615+0000","last_fullsized":"2024-08-17T17:44:41.516615+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"402'12702","reported_seq":12805,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:00.195063+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:45:00.195063+0000","last_peered":"2024-08-17T17:45:00.195063+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:45:00.195063+0000","last_undegraded":"2024-08-17T17:45:00.195063+0000","last_fullsized":"2024-08-17T17:45:00.195063+0000","mapping_epoch":337,"log_start":"327'5853","ondisk_log_start":"327'5853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6849,"log_dups_size":0,"ondisk_log_size":6849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133169152,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2032,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12702,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2032}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":105,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512074+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:44:41.512074+0000","last_peered":"2024-08-17T17:44:41.512074+0000","last_clean":"2024-08-17T17:44:41.512074+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:44:41.512074+0000","last_undegraded":"2024-08-17T17:44:41.512074+0000","last_fullsized":"2024-08-17T17:44:41.512074+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":383,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523143+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:44:41.523143+0000","last_peered":"2024-08-17T17:44:41.523143+0000","last_clean":"2024-08-17T17:44:41.523143+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:44:41.523143+0000","last_undegraded":"2024-08-17T17:44:41.523143+0000","last_fullsized":"2024-08-17T17:44:41.523143+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"402'12870","reported_seq":13056,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:59.999624+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:44:59.999624+0000","last_peered":"2024-08-17T17:44:59.999624+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:44:59.999624+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"327'5813","ondisk_log_start":"327'5813","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136839168,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12870,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-4681},{"shards":"0,3,7","objects":2088}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":560,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491936+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:44:57.491936+0000","last_peered":"2024-08-17T17:44:57.491936+0000","last_clean":"2024-08-17T17:44:57.491936+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:44:57.491936+0000","last_undegraded":"2024-08-17T17:44:57.491936+0000","last_fullsized":"2024-08-17T17:44:57.491936+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":327,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.636011+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:44:57.636011+0000","last_peered":"2024-08-17T17:44:57.636011+0000","last_clean":"2024-08-17T17:44:57.636011+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:44:57.636011+0000","last_undegraded":"2024-08-17T17:44:57.636011+0000","last_fullsized":"2024-08-17T17:44:57.636011+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"402'12532","reported_seq":12623,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:02.128019+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:45:02.128019+0000","last_peered":"2024-08-17T17:45:02.128019+0000","last_clean":"2024-08-17T17:45:02.128019+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:45:02.128019+0000","last_undegraded":"2024-08-17T17:45:02.128019+0000","last_fullsized":"2024-08-17T17:45:02.128019+0000","mapping_epoch":319,"log_start":"328'5974","ondisk_log_start":"328'5974","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135135232,"num_objects":2062,"num_object_clones":0,"num_object_copies":6186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12532,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":650,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492233+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:44:57.492233+0000","last_peered":"2024-08-17T17:44:57.492233+0000","last_clean":"2024-08-17T17:44:57.492233+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:44:57.492233+0000","last_undegraded":"2024-08-17T17:44:57.492233+0000","last_fullsized":"2024-08-17T17:44:57.492233+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516951+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:44:41.516951+0000","last_peered":"2024-08-17T17:44:41.516951+0000","last_clean":"2024-08-17T17:44:41.516951+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:44:41.516951+0000","last_undegraded":"2024-08-17T17:44:41.516951+0000","last_fullsized":"2024-08-17T17:44:41.516951+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"402'12829","reported_seq":12930,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:02.123030+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:45:02.123030+0000","last_peered":"2024-08-17T17:45:02.123030+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:45:02.123030+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"328'6235","ondisk_log_start":"328'6235","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137822208,"num_objects":2103,"num_object_clones":0,"num_object_copies":6309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12829,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2103},{"shards":"1,6","objects":-5263}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":357,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523221+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:44:41.523221+0000","last_peered":"2024-08-17T17:44:41.523221+0000","last_clean":"2024-08-17T17:44:41.523221+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:44:41.523221+0000","last_undegraded":"2024-08-17T17:44:41.523221+0000","last_fullsized":"2024-08-17T17:44:41.523221+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":383,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517158+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:44:41.517158+0000","last_peered":"2024-08-17T17:44:41.517158+0000","last_clean":"2024-08-17T17:44:41.517158+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:44:41.517158+0000","last_undegraded":"2024-08-17T17:44:41.517158+0000","last_fullsized":"2024-08-17T17:44:41.517158+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"402'12994","reported_seq":13181,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:03.968017+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:45:03.968017+0000","last_peered":"2024-08-17T17:45:03.968017+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:45:03.968017+0000","last_undegraded":"2024-08-17T17:45:03.968017+0000","last_fullsized":"2024-08-17T17:45:03.968017+0000","mapping_epoch":337,"log_start":"330'7361","ondisk_log_start":"330'7361","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5633,"log_dups_size":0,"ondisk_log_size":5633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132120576,"num_objects":2016,"num_object_clones":0,"num_object_copies":6048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2016,"num_objects_unfound":0,"num_objects_dirty":2016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12994,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":2016}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":394,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516829+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:44:41.516829+0000","last_peered":"2024-08-17T17:44:41.516829+0000","last_clean":"2024-08-17T17:44:41.516829+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:44:41.516829+0000","last_undegraded":"2024-08-17T17:44:41.516829+0000","last_fullsized":"2024-08-17T17:44:41.516829+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517396+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:44:41.517396+0000","last_peered":"2024-08-17T17:44:41.517396+0000","last_clean":"2024-08-17T17:44:41.517396+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:44:41.517396+0000","last_undegraded":"2024-08-17T17:44:41.517396+0000","last_fullsized":"2024-08-17T17:44:41.517396+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"402'12710","reported_seq":22133,"reported_epoch":402,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:00.583643+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:45:00.583643+0000","last_peered":"2024-08-17T17:45:00.583643+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:45:00.583643+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:45:00.583643+0000","mapping_epoch":337,"log_start":"327'5841","ondisk_log_start":"327'5841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130678784,"num_objects":1994,"num_object_clones":0,"num_object_copies":5982,"num_objects_missing_on_primary":2688,"num_objects_missing":2688,"num_objects_degraded":1994,"num_objects_misplaced":2688,"num_objects_unfound":0,"num_objects_dirty":1994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12710,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4652,"num_bytes_recovered":304807936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":12},{"shards":"1,2,7","objects":1982}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":619,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491694+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:44:57.491694+0000","last_peered":"2024-08-17T17:44:57.491694+0000","last_clean":"2024-08-17T17:44:57.491694+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:44:57.491694+0000","last_undegraded":"2024-08-17T17:44:57.491694+0000","last_fullsized":"2024-08-17T17:44:57.491694+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":556,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492855+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:44:57.492855+0000","last_peered":"2024-08-17T17:44:57.492855+0000","last_clean":"2024-08-17T17:44:57.492855+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:44:57.492855+0000","last_undegraded":"2024-08-17T17:44:57.492855+0000","last_fullsized":"2024-08-17T17:44:57.492855+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"402'12923","reported_seq":22549,"reported_epoch":402,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:04.018310+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:45:04.018310+0000","last_peered":"2024-08-17T17:45:04.018310+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:45:04.018310+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:45:04.018310+0000","mapping_epoch":337,"log_start":"330'7344","ondisk_log_start":"330'7344","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5579,"log_dups_size":0,"ondisk_log_size":5579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129237016,"num_objects":1973,"num_object_clones":0,"num_object_copies":5919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1973,"num_objects_misplaced":2723,"num_objects_unfound":0,"num_objects_dirty":1973,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12923,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4725,"num_bytes_recovered":309592064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1973}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":750,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:57.635699+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:44:57.635699+0000","last_peered":"2024-08-17T17:44:57.635699+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:44:57.635699+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":331,"reported_epoch":401,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:44:41.511961+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:44:41.511961+0000","last_peered":"2024-08-17T17:44:41.511961+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:44:41.511961+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:44:41.511961+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":599,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491331+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:44:57.491331+0000","last_peered":"2024-08-17T17:44:57.491331+0000","last_clean":"2024-08-17T17:44:57.491331+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:44:57.491331+0000","last_undegraded":"2024-08-17T17:44:57.491331+0000","last_fullsized":"2024-08-17T17:44:57.491331+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":366,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523680+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:44:41.523680+0000","last_peered":"2024-08-17T17:44:41.523680+0000","last_clean":"2024-08-17T17:44:41.523680+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:44:41.523680+0000","last_undegraded":"2024-08-17T17:44:41.523680+0000","last_fullsized":"2024-08-17T17:44:41.523680+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"402'12669","reported_seq":12857,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:59.534023+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:44:59.534023+0000","last_peered":"2024-08-17T17:44:59.534023+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:44:59.534023+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'5621","ondisk_log_start":"327'5621","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135987200,"num_objects":2075,"num_object_clones":0,"num_object_copies":6225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12669,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-4572},{"shards":"0,4,7","objects":2075}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":415,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.483252+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:44:57.483252+0000","last_peered":"2024-08-17T17:44:57.483252+0000","last_clean":"2024-08-17T17:44:57.483252+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:44:57.483252+0000","last_undegraded":"2024-08-17T17:44:57.483252+0000","last_fullsized":"2024-08-17T17:44:57.483252+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":338,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516493+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:44:41.516493+0000","last_peered":"2024-08-17T17:44:41.516493+0000","last_clean":"2024-08-17T17:44:41.516493+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:44:41.516493+0000","last_undegraded":"2024-08-17T17:44:41.516493+0000","last_fullsized":"2024-08-17T17:44:41.516493+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":341,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.637122+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:44:57.637122+0000","last_peered":"2024-08-17T17:44:57.637122+0000","last_clean":"2024-08-17T17:44:57.637122+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:44:57.637122+0000","last_undegraded":"2024-08-17T17:44:57.637122+0000","last_fullsized":"2024-08-17T17:44:57.637122+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"402'12888","reported_seq":12989,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:02.196153+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:45:02.196153+0000","last_peered":"2024-08-17T17:45:02.196153+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:45:02.196153+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"328'6317","ondisk_log_start":"328'6317","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125042688,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12888,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1908},{"shards":"5,6","objects":-5388}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":507,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491607+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:44:57.491607+0000","last_peered":"2024-08-17T17:44:57.491607+0000","last_clean":"2024-08-17T17:44:57.491607+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:44:57.491607+0000","last_undegraded":"2024-08-17T17:44:57.491607+0000","last_fullsized":"2024-08-17T17:44:57.491607+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":138,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523350+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:44:41.523350+0000","last_peered":"2024-08-17T17:44:41.523350+0000","last_clean":"2024-08-17T17:44:41.523350+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:44:41.523350+0000","last_undegraded":"2024-08-17T17:44:41.523350+0000","last_fullsized":"2024-08-17T17:44:41.523350+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":692,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.483890+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:44:57.483890+0000","last_peered":"2024-08-17T17:44:57.483890+0000","last_clean":"2024-08-17T17:44:57.483890+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:44:57.483890+0000","last_undegraded":"2024-08-17T17:44:57.483890+0000","last_fullsized":"2024-08-17T17:44:57.483890+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"402'12630","reported_seq":13542,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:59.899115+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:44:59.899115+0000","last_peered":"2024-08-17T17:44:59.899115+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:44:59.899115+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"327'5616","ondisk_log_start":"327'5616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132120576,"num_objects":2016,"num_object_clones":0,"num_object_copies":6048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12630,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":2016},{"shards":"0,5","objects":-4638}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":483,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.483491+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:44:57.483491+0000","last_peered":"2024-08-17T17:44:57.483491+0000","last_clean":"2024-08-17T17:44:57.483491+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:44:57.483491+0000","last_undegraded":"2024-08-17T17:44:57.483491+0000","last_fullsized":"2024-08-17T17:44:57.483491+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":377,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517157+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:44:41.517157+0000","last_peered":"2024-08-17T17:44:41.517157+0000","last_clean":"2024-08-17T17:44:41.517157+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:44:41.517157+0000","last_undegraded":"2024-08-17T17:44:41.517157+0000","last_fullsized":"2024-08-17T17:44:41.517157+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491437+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:44:57.491437+0000","last_peered":"2024-08-17T17:44:57.491437+0000","last_clean":"2024-08-17T17:44:57.491437+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:44:57.491437+0000","last_undegraded":"2024-08-17T17:44:57.491437+0000","last_fullsized":"2024-08-17T17:44:57.491437+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"402'12837","reported_seq":27869,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:02.736336+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:45:02.736336+0000","last_peered":"2024-08-17T17:45:02.736336+0000","last_clean":"2024-08-17T17:45:02.736336+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:45:02.736336+0000","last_undegraded":"2024-08-17T17:45:02.736336+0000","last_fullsized":"2024-08-17T17:45:02.736336+0000","mapping_epoch":355,"log_start":"328'6530","ondisk_log_start":"328'6530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12837,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":316,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511981+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:44:41.511981+0000","last_peered":"2024-08-17T17:44:41.511981+0000","last_clean":"2024-08-17T17:44:41.511981+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:44:41.511981+0000","last_undegraded":"2024-08-17T17:44:41.511981+0000","last_fullsized":"2024-08-17T17:44:41.511981+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":147,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491352+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:44:57.491352+0000","last_peered":"2024-08-17T17:44:57.491352+0000","last_clean":"2024-08-17T17:44:57.491352+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:44:57.491352+0000","last_undegraded":"2024-08-17T17:44:57.491352+0000","last_fullsized":"2024-08-17T17:44:57.491352+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":642,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.483629+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:44:57.483629+0000","last_peered":"2024-08-17T17:44:57.483629+0000","last_clean":"2024-08-17T17:44:57.483629+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:44:57.483629+0000","last_undegraded":"2024-08-17T17:44:57.483629+0000","last_fullsized":"2024-08-17T17:44:57.483629+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"402'12510","reported_seq":27109,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:02.842636+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:45:02.842636+0000","last_peered":"2024-08-17T17:45:02.842636+0000","last_clean":"2024-08-17T17:45:02.842636+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:45:02.842636+0000","last_undegraded":"2024-08-17T17:45:02.842636+0000","last_fullsized":"2024-08-17T17:45:02.842636+0000","mapping_epoch":345,"log_start":"328'6172","ondisk_log_start":"328'6172","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6338,"log_dups_size":0,"ondisk_log_size":6338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124125184,"num_objects":1894,"num_object_clones":0,"num_object_copies":5682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12510,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":312,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512095+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:44:41.512095+0000","last_peered":"2024-08-17T17:44:41.512095+0000","last_clean":"2024-08-17T17:44:41.512095+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:44:41.512095+0000","last_undegraded":"2024-08-17T17:44:41.512095+0000","last_fullsized":"2024-08-17T17:44:41.512095+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":398,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.636006+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:44:57.636006+0000","last_peered":"2024-08-17T17:44:57.636006+0000","last_clean":"2024-08-17T17:44:57.636006+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:44:57.636006+0000","last_undegraded":"2024-08-17T17:44:57.636006+0000","last_fullsized":"2024-08-17T17:44:57.636006+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":454,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511865+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:44:41.511865+0000","last_peered":"2024-08-17T17:44:41.511865+0000","last_clean":"2024-08-17T17:44:41.511865+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:44:41.511865+0000","last_undegraded":"2024-08-17T17:44:41.511865+0000","last_fullsized":"2024-08-17T17:44:41.511865+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"402'12712","reported_seq":12920,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:02.839327+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:45:02.839327+0000","last_peered":"2024-08-17T17:45:02.839327+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:45:02.839327+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"328'6387","ondisk_log_start":"328'6387","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6325,"log_dups_size":0,"ondisk_log_size":6325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135790592,"num_objects":2072,"num_object_clones":0,"num_object_copies":6216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12712,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":2072},{"shards":"0,3","objects":-5176}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":672,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492962+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:44:57.492962+0000","last_peered":"2024-08-17T17:44:57.492962+0000","last_clean":"2024-08-17T17:44:57.492962+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:44:57.492962+0000","last_undegraded":"2024-08-17T17:44:57.492962+0000","last_fullsized":"2024-08-17T17:44:57.492962+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":641,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.484126+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:44:57.484126+0000","last_peered":"2024-08-17T17:44:57.484126+0000","last_clean":"2024-08-17T17:44:57.484126+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:44:57.484126+0000","last_undegraded":"2024-08-17T17:44:57.484126+0000","last_fullsized":"2024-08-17T17:44:57.484126+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":659,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491438+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:44:57.491438+0000","last_peered":"2024-08-17T17:44:57.491438+0000","last_clean":"2024-08-17T17:44:57.491438+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:44:57.491438+0000","last_undegraded":"2024-08-17T17:44:57.491438+0000","last_fullsized":"2024-08-17T17:44:57.491438+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"402'12565","reported_seq":27264,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:02.132198+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:45:02.132198+0000","last_peered":"2024-08-17T17:45:02.132198+0000","last_clean":"2024-08-17T17:45:02.132198+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:45:02.132198+0000","last_undegraded":"2024-08-17T17:45:02.132198+0000","last_fullsized":"2024-08-17T17:45:02.132198+0000","mapping_epoch":347,"log_start":"328'5972","ondisk_log_start":"328'5972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131661824,"num_objects":2009,"num_object_clones":0,"num_object_copies":6027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":331,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.637030+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:44:57.637030+0000","last_peered":"2024-08-17T17:44:57.637030+0000","last_clean":"2024-08-17T17:44:57.637030+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:44:57.637030+0000","last_undegraded":"2024-08-17T17:44:57.637030+0000","last_fullsized":"2024-08-17T17:44:57.637030+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":406,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523700+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:44:41.523700+0000","last_peered":"2024-08-17T17:44:41.523700+0000","last_clean":"2024-08-17T17:44:41.523700+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:44:41.523700+0000","last_undegraded":"2024-08-17T17:44:41.523700+0000","last_fullsized":"2024-08-17T17:44:41.523700+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516754+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:44:41.516754+0000","last_peered":"2024-08-17T17:44:41.516754+0000","last_clean":"2024-08-17T17:44:41.516754+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:44:41.516754+0000","last_undegraded":"2024-08-17T17:44:41.516754+0000","last_fullsized":"2024-08-17T17:44:41.516754+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517027+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:44:41.517027+0000","last_peered":"2024-08-17T17:44:41.517027+0000","last_clean":"2024-08-17T17:44:41.517027+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:44:41.517027+0000","last_undegraded":"2024-08-17T17:44:41.517027+0000","last_fullsized":"2024-08-17T17:44:41.517027+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":399,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517114+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:44:41.517114+0000","last_peered":"2024-08-17T17:44:41.517114+0000","last_clean":"2024-08-17T17:44:41.517114+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:44:41.517114+0000","last_undegraded":"2024-08-17T17:44:41.517114+0000","last_fullsized":"2024-08-17T17:44:41.517114+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":647,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.483455+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:44:57.483455+0000","last_peered":"2024-08-17T17:44:57.483455+0000","last_clean":"2024-08-17T17:44:57.483455+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:44:57.483455+0000","last_undegraded":"2024-08-17T17:44:57.483455+0000","last_fullsized":"2024-08-17T17:44:57.483455+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":187,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492452+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:44:57.492452+0000","last_peered":"2024-08-17T17:44:57.492452+0000","last_clean":"2024-08-17T17:44:57.492452+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:44:57.492452+0000","last_undegraded":"2024-08-17T17:44:57.492452+0000","last_fullsized":"2024-08-17T17:44:57.492452+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511931+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:44:41.511931+0000","last_peered":"2024-08-17T17:44:41.511931+0000","last_clean":"2024-08-17T17:44:41.511931+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:44:41.511931+0000","last_undegraded":"2024-08-17T17:44:41.511931+0000","last_fullsized":"2024-08-17T17:44:41.511931+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":605,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491019+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:44:57.491019+0000","last_peered":"2024-08-17T17:44:57.491019+0000","last_clean":"2024-08-17T17:44:57.491019+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:44:57.491019+0000","last_undegraded":"2024-08-17T17:44:57.491019+0000","last_fullsized":"2024-08-17T17:44:57.491019+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":450,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512295+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:44:41.512295+0000","last_peered":"2024-08-17T17:44:41.512295+0000","last_clean":"2024-08-17T17:44:41.512295+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:44:41.512295+0000","last_undegraded":"2024-08-17T17:44:41.512295+0000","last_fullsized":"2024-08-17T17:44:41.512295+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":599,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491076+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:44:57.491076+0000","last_peered":"2024-08-17T17:44:57.491076+0000","last_clean":"2024-08-17T17:44:57.491076+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:44:57.491076+0000","last_undegraded":"2024-08-17T17:44:57.491076+0000","last_fullsized":"2024-08-17T17:44:57.491076+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":373,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523619+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:44:41.523619+0000","last_peered":"2024-08-17T17:44:41.523619+0000","last_clean":"2024-08-17T17:44:41.523619+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:44:41.523619+0000","last_undegraded":"2024-08-17T17:44:41.523619+0000","last_fullsized":"2024-08-17T17:44:41.523619+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":631,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491626+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:44:57.491626+0000","last_peered":"2024-08-17T17:44:57.491626+0000","last_clean":"2024-08-17T17:44:57.491626+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:44:57.491626+0000","last_undegraded":"2024-08-17T17:44:57.491626+0000","last_fullsized":"2024-08-17T17:44:57.491626+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":347,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492587+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:44:57.492587+0000","last_peered":"2024-08-17T17:44:57.492587+0000","last_clean":"2024-08-17T17:44:57.492587+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:44:57.492587+0000","last_undegraded":"2024-08-17T17:44:57.492587+0000","last_fullsized":"2024-08-17T17:44:57.492587+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":639,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.483453+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:44:57.483453+0000","last_peered":"2024-08-17T17:44:57.483453+0000","last_clean":"2024-08-17T17:44:57.483453+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:44:57.483453+0000","last_undegraded":"2024-08-17T17:44:57.483453+0000","last_fullsized":"2024-08-17T17:44:57.483453+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":598,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492333+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:44:57.492333+0000","last_peered":"2024-08-17T17:44:57.492333+0000","last_clean":"2024-08-17T17:44:57.492333+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:44:57.492333+0000","last_undegraded":"2024-08-17T17:44:57.492333+0000","last_fullsized":"2024-08-17T17:44:57.492333+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":385,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516836+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:44:41.516836+0000","last_peered":"2024-08-17T17:44:41.516836+0000","last_clean":"2024-08-17T17:44:41.516836+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:44:41.516836+0000","last_undegraded":"2024-08-17T17:44:41.516836+0000","last_fullsized":"2024-08-17T17:44:41.516836+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":287,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491847+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:44:57.491847+0000","last_peered":"2024-08-17T17:44:57.491847+0000","last_clean":"2024-08-17T17:44:57.491847+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:44:57.491847+0000","last_undegraded":"2024-08-17T17:44:57.491847+0000","last_fullsized":"2024-08-17T17:44:57.491847+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":344,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.635852+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:44:57.635852+0000","last_peered":"2024-08-17T17:44:57.635852+0000","last_clean":"2024-08-17T17:44:57.635852+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:44:57.635852+0000","last_undegraded":"2024-08-17T17:44:57.635852+0000","last_fullsized":"2024-08-17T17:44:57.635852+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":393,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516957+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:44:41.516957+0000","last_peered":"2024-08-17T17:44:41.516957+0000","last_clean":"2024-08-17T17:44:41.516957+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:44:41.516957+0000","last_undegraded":"2024-08-17T17:44:41.516957+0000","last_fullsized":"2024-08-17T17:44:41.516957+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":256,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517034+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:44:41.517034+0000","last_peered":"2024-08-17T17:44:41.517034+0000","last_clean":"2024-08-17T17:44:41.517034+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:44:41.517034+0000","last_undegraded":"2024-08-17T17:44:41.517034+0000","last_fullsized":"2024-08-17T17:44:41.517034+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":393,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516694+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:44:41.516694+0000","last_peered":"2024-08-17T17:44:41.516694+0000","last_clean":"2024-08-17T17:44:41.516694+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:44:41.516694+0000","last_undegraded":"2024-08-17T17:44:41.516694+0000","last_fullsized":"2024-08-17T17:44:41.516694+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":256,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517035+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:44:41.517035+0000","last_peered":"2024-08-17T17:44:41.517035+0000","last_clean":"2024-08-17T17:44:41.517035+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:44:41.517035+0000","last_undegraded":"2024-08-17T17:44:41.517035+0000","last_fullsized":"2024-08-17T17:44:41.517035+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":651,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491682+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:44:57.491682+0000","last_peered":"2024-08-17T17:44:57.491682+0000","last_clean":"2024-08-17T17:44:57.491682+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:44:57.491682+0000","last_undegraded":"2024-08-17T17:44:57.491682+0000","last_fullsized":"2024-08-17T17:44:57.491682+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":625,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492681+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:44:57.492681+0000","last_peered":"2024-08-17T17:44:57.492681+0000","last_clean":"2024-08-17T17:44:57.492681+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:44:57.492681+0000","last_undegraded":"2024-08-17T17:44:57.492681+0000","last_fullsized":"2024-08-17T17:44:57.492681+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.518921+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:44:41.518921+0000","last_peered":"2024-08-17T17:44:41.518921+0000","last_clean":"2024-08-17T17:44:41.518921+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:44:41.518921+0000","last_undegraded":"2024-08-17T17:44:41.518921+0000","last_fullsized":"2024-08-17T17:44:41.518921+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":346,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.636820+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:44:57.636820+0000","last_peered":"2024-08-17T17:44:57.636820+0000","last_clean":"2024-08-17T17:44:57.636820+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:44:57.636820+0000","last_undegraded":"2024-08-17T17:44:57.636820+0000","last_fullsized":"2024-08-17T17:44:57.636820+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2122842136,"num_objects":32393,"num_object_clones":0,"num_object_copies":97179,"num_objects_missing_on_primary":2688,"num_objects_missing":2688,"num_objects_degraded":48386,"num_objects_misplaced":13611,"num_objects_unfound":0,"num_objects_dirty":32393,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":203997,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36222,"num_bytes_recovered":2251292672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9400168448,"data_stored":9400156232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104816,"ondisk_log_size":104816,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170143,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463513,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2101944,"kb_used_data":911536,"kb_used_omap":1,"kb_used_meta":1190398,"kb_avail":91643208,"statfs":{"total":95995035648,"available":93842644992,"internally_reserved":0,"allocated":933412864,"data_stored":931179839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1218967896},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492187,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":652576,"kb_used_data":261396,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":93092576,"statfs":{"total":95995035648,"available":95326797824,"internally_reserved":0,"allocated":267669504,"data_stored":265458779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986778,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2034028,"kb_used_data":525348,"kb_used_omap":1,"kb_used_meta":1508670,"kb_avail":91711124,"statfs":{"total":95995035648,"available":93912190976,"internally_reserved":0,"allocated":537956352,"data_stored":535734267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1544878424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225607,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2012252,"kb_used_data":1042388,"kb_used_omap":4,"kb_used_meta":969851,"kb_avail":91732900,"statfs":{"total":95995035648,"available":93934489600,"internally_reserved":0,"allocated":1067405312,"data_stored":1065194885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":993127872},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075931,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533825,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1981600,"kb_used_data":907672,"kb_used_omap":5,"kb_used_meta":1073914,"kb_avail":91763552,"statfs":{"total":95995035648,"available":93965877248,"internally_reserved":0,"allocated":929456128,"data_stored":927240379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1099688176},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089801,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830328,"kb_used_data":803056,"kb_used_omap":5,"kb_used_meta":1027258,"kb_avail":91914824,"statfs":{"total":95995035648,"available":94120779776,"internally_reserved":0,"allocated":822329344,"data_stored":820096378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1051912251},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":815136768,"data_stored":815136768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":919146496,"data_stored":919142424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1058275328,"data_stored":1058275328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":531628032,"data_stored":531628032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":260243456,"data_stored":260243456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":923598848,"data_stored":923598848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1552879616,"data_stored":1552875544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:45:06.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:06.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48389/97257 objects degraded (49.754%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:06.871 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48389/97257 objects degraded (49.754%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48389/97257 objects degraded (49.754%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:06.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:06.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:06.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:07.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:07.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:07.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:07.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:07.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:07.222 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:07.225 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:07.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:07.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:07.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:07.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:07.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:07.563 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:07.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:07.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:07.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:07.786 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:07 smithi142 ceph-mon[106729]: from='client.37974 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:07.786 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:07 smithi142 ceph-mon[106729]: pgmap v1578: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 48316/95859 objects degraded (50.403%); 13352/95859 objects misplaced (13.929%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:45:07.786 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:07 smithi142 ceph-mon[103614]: from='client.37974 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:07.786 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:07 smithi142 ceph-mon[103614]: pgmap v1578: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 48316/95859 objects degraded (50.403%); 13352/95859 objects misplaced (13.929%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:45:07.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:07.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:07 smithi152 ceph-mon[93456]: from='client.37974 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:07 smithi152 ceph-mon[93456]: pgmap v1578: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 48316/95859 objects degraded (50.403%); 13352/95859 objects misplaced (13.929%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:45:07.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:07.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:07.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:07.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:08.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:08.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:08.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:08.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:08.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:08.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:08.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:08.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:08.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:08.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:08.541 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:08.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:08.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:08.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:08.694 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:08.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:08.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:08.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:08 smithi152 ceph-mon[93456]: pgmap v1579: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 48316/93840 objects degraded (51.488%); 13274/93840 objects misplaced (14.145%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:45:08.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:08 smithi142 ceph-mon[106729]: pgmap v1579: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 48316/93840 objects degraded (51.488%); 13274/93840 objects misplaced (14.145%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:45:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:08 smithi142 ceph-mon[103614]: pgmap v1579: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 48316/93840 objects degraded (51.488%); 13274/93840 objects misplaced (14.145%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:45:08.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:08.948 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:08.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:09.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:09.070 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:09.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:09.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:09.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:09.295 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:09.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:09.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:09.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:09.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:09.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:09.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:09.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:09.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:45:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.694 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:09.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:09.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:09.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:09.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:09.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:09.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:10.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:10.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:10.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:10.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:10.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:10.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:10.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:10.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:10.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:10.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:10.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:10.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:45:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:45:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:10.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:10.614 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:45:10.615 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:10.627 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:45:10.627 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:10.651 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:45:10.651 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:10.671 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:45:10.671 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:10.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:10.690 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:45:10.690 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:10.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:10.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:10.712 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:45:10.712 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:10.736 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:45:10.737 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:10.756 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:45:10.756 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:10.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:10.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:10.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:10.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:10.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:10.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:10.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:11.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:11.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:11.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:11.208 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:11.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:11.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:11.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:11.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:11.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:11.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:11.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:11.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:11.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:11.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[106729]: pgmap v1580: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 250 op/s; 48221/92541 objects degraded (52.108%); 13101/92541 objects misplaced (14.157%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3500289423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48221/92541 objects degraded (52.108%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[103614]: pgmap v1580: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 250 op/s; 48221/92541 objects degraded (52.108%); 13101/92541 objects misplaced (14.157%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3500289423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48221/92541 objects degraded (52.108%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:11.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:11.789 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:11.789 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:11.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:11 smithi152 ceph-mon[93456]: pgmap v1580: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 250 op/s; 48221/92541 objects degraded (52.108%); 13101/92541 objects misplaced (14.157%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:11 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3500289423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48221/92541 objects degraded (52.108%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:11.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:11.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:11.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:11.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:11.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:12.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:12.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:45:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:45:11] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-08-17T17:45:12.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:12.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:12.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:12.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:12.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:12.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:12.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:12.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:12.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:12.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:12.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:45:12.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:12.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:12.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:12.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:12.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:12.928 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:12.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:12.992 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:12.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:13.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:13.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:13.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:13.243 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:13.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:13.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:13.462 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:45:13.468 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:13.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:13.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:13.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:13.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:45:13.616 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:13.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:13.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:13 smithi152 ceph-mon[93456]: pgmap v1581: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 48142/90666 objects degraded (53.098%); 12794/90666 objects misplaced (14.111%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3925260406' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:45:13.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:13.864 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:13.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:13 smithi142 ceph-mon[106729]: pgmap v1581: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 48142/90666 objects degraded (53.098%); 12794/90666 objects misplaced (14.111%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3925260406' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:45:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:13 smithi142 ceph-mon[103614]: pgmap v1581: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 48142/90666 objects degraded (53.098%); 12794/90666 objects misplaced (14.111%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3925260406' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:45:13.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:13.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:13.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:14.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:14.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:14.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:14.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:14.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:14.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:14.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:14.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:14.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:14.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:14.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:14 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3925260406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:45:14.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:14 smithi142 ceph-mon[106729]: osdmap e403: 8 total, 8 up, 6 in 2024-08-17T17:45:14.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:14 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:14.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3925260406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:45:14.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:14 smithi142 ceph-mon[103614]: osdmap e403: 8 total, 8 up, 6 in 2024-08-17T17:45:14.626 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:14.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:14.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:14 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3925260406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:45:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:14 smithi152 ceph-mon[93456]: osdmap e403: 8 total, 8 up, 6 in 2024-08-17T17:45:14.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:14.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:14.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:14.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:14.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:15.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:15.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:15.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:15.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:15.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:15.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:15.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:15.468 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:15.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:15.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:15.574 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:15 smithi142 ceph-mon[106729]: pgmap v1583: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 48068/89508 objects degraded (53.702%); 12641/89508 objects misplaced (14.123%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:45:15.574 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:15 smithi142 ceph-mon[103614]: pgmap v1583: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 48068/89508 objects degraded (53.702%); 12641/89508 objects misplaced (14.123%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:45:15.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:15.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:15.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:15.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:15.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:15 smithi152 ceph-mon[93456]: pgmap v1583: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 48068/89508 objects degraded (53.702%); 12641/89508 objects misplaced (14.123%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:45:15.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:15.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:15.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:15.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:15.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:15.996 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:45:15.997 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:45:16.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:16.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:16.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:16.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:16.234 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:16.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:16.361 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:16.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:16.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:16.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:16.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 48068/89508 objects degraded (53.702%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:16.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 48068/89508 objects degraded (53.702%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:16.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:16.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:16.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:16.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:16.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:16.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 48068/89508 objects degraded (53.702%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:16.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:16.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:16.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:16.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.015 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:17.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.057 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1583,"stamp":"2024-08-17T17:45:14.407480+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1954158565,"num_objects":29836,"num_object_clones":0,"num_object_copies":89508,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":48068,"num_objects_misplaced":12641,"num_objects_unfound":0,"num_objects_dirty":29836,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":206858,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36738,"num_bytes_recovered":2284790414,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104806,"ondisk_log_size":104806,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10258484,"kb_used_data":4101316,"kb_used_omap":27,"kb_used_meta":6157092,"kb_avail":552212428,"statfs":{"total":575970213888,"available":565465526272,"internally_reserved":0,"allocated":4199747584,"data_stored":4186422607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6304862952},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,3,1,0,1,3,3,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-169279488,"num_objects":-2583,"num_object_clones":0,"num_object_copies":-7749,"num_objects_missing_on_primary":-150,"num_objects_missing":-150,"num_objects_degraded":-321,"num_objects_misplaced":-970,"num_objects_unfound":0,"num_objects_dirty":-2583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2583,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":475,"num_bytes_recovered":31129600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002812"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":648,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491775+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:44:57.491775+0000","last_peered":"2024-08-17T17:44:57.491775+0000","last_clean":"2024-08-17T17:44:57.491775+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:44:57.491775+0000","last_undegraded":"2024-08-17T17:44:57.491775+0000","last_fullsized":"2024-08-17T17:44:57.491775+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":111,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516769+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:44:41.516769+0000","last_peered":"2024-08-17T17:44:41.516769+0000","last_clean":"2024-08-17T17:44:41.516769+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:44:41.516769+0000","last_undegraded":"2024-08-17T17:44:41.516769+0000","last_fullsized":"2024-08-17T17:44:41.516769+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512117+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:44:41.512117+0000","last_peered":"2024-08-17T17:44:41.512117+0000","last_clean":"2024-08-17T17:44:41.512117+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:44:41.512117+0000","last_undegraded":"2024-08-17T17:44:41.512117+0000","last_fullsized":"2024-08-17T17:44:41.512117+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":346,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491435+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:44:57.491435+0000","last_peered":"2024-08-17T17:44:57.491435+0000","last_clean":"2024-08-17T17:44:57.491435+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:44:57.491435+0000","last_undegraded":"2024-08-17T17:44:57.491435+0000","last_fullsized":"2024-08-17T17:44:57.491435+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":523,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492607+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:44:57.492607+0000","last_peered":"2024-08-17T17:44:57.492607+0000","last_clean":"2024-08-17T17:44:57.492607+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:44:57.492607+0000","last_undegraded":"2024-08-17T17:44:57.492607+0000","last_fullsized":"2024-08-17T17:44:57.492607+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":254,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516689+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:44:41.516689+0000","last_peered":"2024-08-17T17:44:41.516689+0000","last_clean":"2024-08-17T17:44:41.516689+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:44:41.516689+0000","last_undegraded":"2024-08-17T17:44:41.516689+0000","last_fullsized":"2024-08-17T17:44:41.516689+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":617,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492855+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:44:57.492855+0000","last_peered":"2024-08-17T17:44:57.492855+0000","last_clean":"2024-08-17T17:44:57.492855+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:44:57.492855+0000","last_undegraded":"2024-08-17T17:44:57.492855+0000","last_fullsized":"2024-08-17T17:44:57.492855+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":422,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.524532+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:44:41.524532+0000","last_peered":"2024-08-17T17:44:41.524532+0000","last_clean":"2024-08-17T17:44:41.524532+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:44:41.524532+0000","last_undegraded":"2024-08-17T17:44:41.524532+0000","last_fullsized":"2024-08-17T17:44:41.524532+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"402'12800","reported_seq":22354,"reported_epoch":402,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:45:10.649331+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:45:10.649331+0000","last_peered":"2024-08-17T17:45:10.649331+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:45:10.649331+0000","last_undegraded":"2024-08-17T17:45:10.649331+0000","last_fullsized":"2024-08-17T17:45:10.649331+0000","mapping_epoch":337,"log_start":"328'5963","ondisk_log_start":"328'5963","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124780544,"num_objects":1904,"num_object_clones":0,"num_object_copies":5712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1903,"num_objects_unfound":0,"num_objects_dirty":1904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12800,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4720,"num_bytes_recovered":309329920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":1903},{"shards":"4,6,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":107,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.541794+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:44:41.541794+0000","last_peered":"2024-08-17T17:44:41.541794+0000","last_clean":"2024-08-17T17:44:41.541794+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:44:41.541794+0000","last_undegraded":"2024-08-17T17:44:41.541794+0000","last_fullsized":"2024-08-17T17:44:41.541794+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":288,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511798+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:44:41.511798+0000","last_peered":"2024-08-17T17:44:41.511798+0000","last_clean":"2024-08-17T17:44:41.511798+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:44:41.511798+0000","last_undegraded":"2024-08-17T17:44:41.511798+0000","last_fullsized":"2024-08-17T17:44:41.511798+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"402'13152","reported_seq":13252,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:12.165804+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:45:12.165804+0000","last_peered":"2024-08-17T17:45:12.165804+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:45:12.165804+0000","last_undegraded":"2024-08-17T17:45:12.165804+0000","last_fullsized":"2024-08-17T17:45:12.165804+0000","mapping_epoch":337,"log_start":"328'6582","ondisk_log_start":"328'6582","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125960192,"num_objects":1922,"num_object_clones":0,"num_object_copies":5766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1922,"num_objects_unfound":0,"num_objects_dirty":1922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13152,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":1922}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":109,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517283+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:44:41.517283+0000","last_peered":"2024-08-17T17:44:41.517283+0000","last_clean":"2024-08-17T17:44:41.517283+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:44:41.517283+0000","last_undegraded":"2024-08-17T17:44:41.517283+0000","last_fullsized":"2024-08-17T17:44:41.517283+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":373,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516615+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:44:41.516615+0000","last_peered":"2024-08-17T17:44:41.516615+0000","last_clean":"2024-08-17T17:44:41.516615+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:44:41.516615+0000","last_undegraded":"2024-08-17T17:44:41.516615+0000","last_fullsized":"2024-08-17T17:44:41.516615+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"402'12869","reported_seq":12972,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:10.384638+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:45:10.384638+0000","last_peered":"2024-08-17T17:45:10.384638+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:45:10.384638+0000","last_undegraded":"2024-08-17T17:45:10.384638+0000","last_fullsized":"2024-08-17T17:45:10.384638+0000","mapping_epoch":337,"log_start":"328'5953","ondisk_log_start":"328'5953","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6916,"log_dups_size":0,"ondisk_log_size":6916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122224640,"num_objects":1865,"num_object_clones":0,"num_object_copies":5595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1865,"num_objects_unfound":0,"num_objects_dirty":1865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12869,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1865}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":105,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512074+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:44:41.512074+0000","last_peered":"2024-08-17T17:44:41.512074+0000","last_clean":"2024-08-17T17:44:41.512074+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:44:41.512074+0000","last_undegraded":"2024-08-17T17:44:41.512074+0000","last_fullsized":"2024-08-17T17:44:41.512074+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":383,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523143+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:44:41.523143+0000","last_peered":"2024-08-17T17:44:41.523143+0000","last_clean":"2024-08-17T17:44:41.523143+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:44:41.523143+0000","last_undegraded":"2024-08-17T17:44:41.523143+0000","last_fullsized":"2024-08-17T17:44:41.523143+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"402'13006","reported_seq":13192,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:09.857717+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:45:09.857717+0000","last_peered":"2024-08-17T17:45:09.857717+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:45:09.857717+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"328'6013","ondisk_log_start":"328'6013","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127926272,"num_objects":1952,"num_object_clones":0,"num_object_copies":5856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13006,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-4817},{"shards":"0,3,7","objects":1952}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":560,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491936+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:44:57.491936+0000","last_peered":"2024-08-17T17:44:57.491936+0000","last_clean":"2024-08-17T17:44:57.491936+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:44:57.491936+0000","last_undegraded":"2024-08-17T17:44:57.491936+0000","last_fullsized":"2024-08-17T17:44:57.491936+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":327,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.636011+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:44:57.636011+0000","last_peered":"2024-08-17T17:44:57.636011+0000","last_clean":"2024-08-17T17:44:57.636011+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:44:57.636011+0000","last_undegraded":"2024-08-17T17:44:57.636011+0000","last_fullsized":"2024-08-17T17:44:57.636011+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"402'12694","reported_seq":12785,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:12.234270+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:45:12.234270+0000","last_peered":"2024-08-17T17:45:12.234270+0000","last_clean":"2024-08-17T17:45:12.234270+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:45:12.234270+0000","last_undegraded":"2024-08-17T17:45:12.234270+0000","last_fullsized":"2024-08-17T17:45:12.234270+0000","mapping_epoch":319,"log_start":"328'6074","ondisk_log_start":"328'6074","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124518400,"num_objects":1900,"num_object_clones":0,"num_object_copies":5700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12694,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":650,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492233+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:44:57.492233+0000","last_peered":"2024-08-17T17:44:57.492233+0000","last_clean":"2024-08-17T17:44:57.492233+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:44:57.492233+0000","last_undegraded":"2024-08-17T17:44:57.492233+0000","last_fullsized":"2024-08-17T17:44:57.492233+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516951+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:44:41.516951+0000","last_peered":"2024-08-17T17:44:41.516951+0000","last_clean":"2024-08-17T17:44:41.516951+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:44:41.516951+0000","last_undegraded":"2024-08-17T17:44:41.516951+0000","last_fullsized":"2024-08-17T17:44:41.516951+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"402'12991","reported_seq":13092,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:12.140798+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:45:12.140798+0000","last_peered":"2024-08-17T17:45:12.140798+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:45:12.140798+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"328'6435","ondisk_log_start":"328'6435","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127205376,"num_objects":1941,"num_object_clones":0,"num_object_copies":5823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12991,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":1941},{"shards":"1,6","objects":-5425}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":357,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523221+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:44:41.523221+0000","last_peered":"2024-08-17T17:44:41.523221+0000","last_clean":"2024-08-17T17:44:41.523221+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:44:41.523221+0000","last_undegraded":"2024-08-17T17:44:41.523221+0000","last_fullsized":"2024-08-17T17:44:41.523221+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":383,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517158+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:44:41.517158+0000","last_peered":"2024-08-17T17:44:41.517158+0000","last_clean":"2024-08-17T17:44:41.517158+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:44:41.517158+0000","last_undegraded":"2024-08-17T17:44:41.517158+0000","last_fullsized":"2024-08-17T17:44:41.517158+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"403'13151","reported_seq":13340,"reported_epoch":403,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:13.697142+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:45:13.697142+0000","last_peered":"2024-08-17T17:45:13.697142+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:45:13.697142+0000","last_undegraded":"2024-08-17T17:45:13.697142+0000","last_fullsized":"2024-08-17T17:45:13.697142+0000","mapping_epoch":337,"log_start":"330'7561","ondisk_log_start":"330'7561","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5590,"log_dups_size":0,"ondisk_log_size":5590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121831424,"num_objects":1859,"num_object_clones":0,"num_object_copies":5577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1859,"num_objects_unfound":0,"num_objects_dirty":1859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13151,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1859}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":394,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516829+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:44:41.516829+0000","last_peered":"2024-08-17T17:44:41.516829+0000","last_clean":"2024-08-17T17:44:41.516829+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:44:41.516829+0000","last_undegraded":"2024-08-17T17:44:41.516829+0000","last_fullsized":"2024-08-17T17:44:41.516829+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517396+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:44:41.517396+0000","last_peered":"2024-08-17T17:44:41.517396+0000","last_clean":"2024-08-17T17:44:41.517396+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:44:41.517396+0000","last_undegraded":"2024-08-17T17:44:41.517396+0000","last_fullsized":"2024-08-17T17:44:41.517396+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"402'12862","reported_seq":22588,"reported_epoch":402,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:10.633165+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:45:10.633165+0000","last_peered":"2024-08-17T17:45:10.633165+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:45:10.633165+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:45:10.633165+0000","mapping_epoch":337,"log_start":"328'6041","ondisk_log_start":"328'6041","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120717312,"num_objects":1842,"num_object_clones":0,"num_object_copies":5526,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":1842,"num_objects_misplaced":2538,"num_objects_unfound":0,"num_objects_dirty":1842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12862,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4804,"num_bytes_recovered":314769408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":10},{"shards":"1,2,7","objects":1832}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":620,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491717+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:44:57.491717+0000","last_peered":"2024-08-17T17:44:57.491717+0000","last_clean":"2024-08-17T17:44:57.491717+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:44:57.491717+0000","last_undegraded":"2024-08-17T17:44:57.491717+0000","last_fullsized":"2024-08-17T17:44:57.491717+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":556,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492855+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:44:57.492855+0000","last_peered":"2024-08-17T17:44:57.492855+0000","last_clean":"2024-08-17T17:44:57.492855+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:44:57.492855+0000","last_undegraded":"2024-08-17T17:44:57.492855+0000","last_fullsized":"2024-08-17T17:44:57.492855+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"403'13092","reported_seq":23058,"reported_epoch":403,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:14.047329+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:45:14.047329+0000","last_peered":"2024-08-17T17:45:14.047329+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:45:14.047329+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:45:14.047329+0000","mapping_epoch":337,"log_start":"330'7444","ondisk_log_start":"330'7444","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5648,"log_dups_size":0,"ondisk_log_size":5648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118161432,"num_objects":1804,"num_object_clones":0,"num_object_copies":5412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1804,"num_objects_misplaced":2554,"num_objects_unfound":0,"num_objects_dirty":1804,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13092,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4894,"num_bytes_recovered":320667648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1804}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":750,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:44:57.635699+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:44:57.635699+0000","last_peered":"2024-08-17T17:44:57.635699+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:44:57.635699+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":331,"reported_epoch":401,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:44:41.511961+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:44:41.511961+0000","last_peered":"2024-08-17T17:44:41.511961+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:44:41.511961+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:44:41.511961+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":599,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491331+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:44:57.491331+0000","last_peered":"2024-08-17T17:44:57.491331+0000","last_clean":"2024-08-17T17:44:57.491331+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:44:57.491331+0000","last_undegraded":"2024-08-17T17:44:57.491331+0000","last_fullsized":"2024-08-17T17:44:57.491331+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":366,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523680+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:44:41.523680+0000","last_peered":"2024-08-17T17:44:41.523680+0000","last_clean":"2024-08-17T17:44:41.523680+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:44:41.523680+0000","last_undegraded":"2024-08-17T17:44:41.523680+0000","last_fullsized":"2024-08-17T17:44:41.523680+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"402'12849","reported_seq":13038,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:09.990683+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:45:09.990683+0000","last_peered":"2024-08-17T17:45:09.990683+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:45:09.990683+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"327'5821","ondisk_log_start":"327'5821","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124190720,"num_objects":1895,"num_object_clones":0,"num_object_copies":5685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12849,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-4752},{"shards":"0,4,7","objects":1895}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":417,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.468854+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:45:13.468854+0000","last_peered":"2024-08-17T17:45:13.468854+0000","last_clean":"2024-08-17T17:45:13.468854+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:45:13.468854+0000","last_undegraded":"2024-08-17T17:45:13.468854+0000","last_fullsized":"2024-08-17T17:45:13.468854+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":338,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516493+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:44:41.516493+0000","last_peered":"2024-08-17T17:44:41.516493+0000","last_clean":"2024-08-17T17:44:41.516493+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:44:41.516493+0000","last_undegraded":"2024-08-17T17:44:41.516493+0000","last_fullsized":"2024-08-17T17:44:41.516493+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":341,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.637122+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:44:57.637122+0000","last_peered":"2024-08-17T17:44:57.637122+0000","last_clean":"2024-08-17T17:44:57.637122+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:44:57.637122+0000","last_undegraded":"2024-08-17T17:44:57.637122+0000","last_fullsized":"2024-08-17T17:44:57.637122+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"402'13047","reported_seq":13148,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:12.128824+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:45:12.128824+0000","last_peered":"2024-08-17T17:45:12.128824+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:45:12.128824+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"328'6517","ondisk_log_start":"328'6517","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114622464,"num_objects":1749,"num_object_clones":0,"num_object_copies":5247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13047,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1749},{"shards":"5,6","objects":-5547}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":507,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491607+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:44:57.491607+0000","last_peered":"2024-08-17T17:44:57.491607+0000","last_clean":"2024-08-17T17:44:57.491607+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:44:57.491607+0000","last_undegraded":"2024-08-17T17:44:57.491607+0000","last_fullsized":"2024-08-17T17:44:57.491607+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":138,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523350+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:44:41.523350+0000","last_peered":"2024-08-17T17:44:41.523350+0000","last_clean":"2024-08-17T17:44:41.523350+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:44:41.523350+0000","last_undegraded":"2024-08-17T17:44:41.523350+0000","last_fullsized":"2024-08-17T17:44:41.523350+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":694,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.473401+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:45:13.473401+0000","last_peered":"2024-08-17T17:45:13.473401+0000","last_clean":"2024-08-17T17:45:13.473401+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:45:13.473401+0000","last_undegraded":"2024-08-17T17:45:13.473401+0000","last_fullsized":"2024-08-17T17:45:13.473401+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"402'12780","reported_seq":13693,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:09.987503+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:45:09.987503+0000","last_peered":"2024-08-17T17:45:09.987503+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:45:09.987503+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"327'5716","ondisk_log_start":"327'5716","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122290176,"num_objects":1866,"num_object_clones":0,"num_object_copies":5598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12780,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":1866},{"shards":"0,5","objects":-4788}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":485,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.469128+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:45:13.469128+0000","last_peered":"2024-08-17T17:45:13.469128+0000","last_clean":"2024-08-17T17:45:13.469128+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:45:13.469128+0000","last_undegraded":"2024-08-17T17:45:13.469128+0000","last_fullsized":"2024-08-17T17:45:13.469128+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":377,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517157+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:44:41.517157+0000","last_peered":"2024-08-17T17:44:41.517157+0000","last_clean":"2024-08-17T17:44:41.517157+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:44:41.517157+0000","last_undegraded":"2024-08-17T17:44:41.517157+0000","last_fullsized":"2024-08-17T17:44:41.517157+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491437+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:44:57.491437+0000","last_peered":"2024-08-17T17:44:57.491437+0000","last_clean":"2024-08-17T17:44:57.491437+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:44:57.491437+0000","last_undegraded":"2024-08-17T17:44:57.491437+0000","last_fullsized":"2024-08-17T17:44:57.491437+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"402'12993","reported_seq":28025,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:12.867666+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:45:12.867666+0000","last_peered":"2024-08-17T17:45:12.867666+0000","last_clean":"2024-08-17T17:45:12.867666+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:45:12.867666+0000","last_undegraded":"2024-08-17T17:45:12.867666+0000","last_fullsized":"2024-08-17T17:45:12.867666+0000","mapping_epoch":355,"log_start":"329'6730","ondisk_log_start":"329'6730","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6263,"log_dups_size":0,"ondisk_log_size":6263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120782848,"num_objects":1843,"num_object_clones":0,"num_object_copies":5529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12993,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":316,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511981+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:44:41.511981+0000","last_peered":"2024-08-17T17:44:41.511981+0000","last_clean":"2024-08-17T17:44:41.511981+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:44:41.511981+0000","last_undegraded":"2024-08-17T17:44:41.511981+0000","last_fullsized":"2024-08-17T17:44:41.511981+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":147,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491352+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:44:57.491352+0000","last_peered":"2024-08-17T17:44:57.491352+0000","last_clean":"2024-08-17T17:44:57.491352+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:44:57.491352+0000","last_undegraded":"2024-08-17T17:44:57.491352+0000","last_fullsized":"2024-08-17T17:44:57.491352+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":644,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.469300+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:45:13.469300+0000","last_peered":"2024-08-17T17:45:13.469300+0000","last_clean":"2024-08-17T17:45:13.469300+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:45:13.469300+0000","last_undegraded":"2024-08-17T17:45:13.469300+0000","last_fullsized":"2024-08-17T17:45:13.469300+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"402'12673","reported_seq":27273,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:12.830912+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:45:12.830912+0000","last_peered":"2024-08-17T17:45:12.830912+0000","last_clean":"2024-08-17T17:45:12.830912+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:45:12.830912+0000","last_undegraded":"2024-08-17T17:45:12.830912+0000","last_fullsized":"2024-08-17T17:45:12.830912+0000","mapping_epoch":345,"log_start":"328'6372","ondisk_log_start":"328'6372","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113442816,"num_objects":1731,"num_object_clones":0,"num_object_copies":5193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12673,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":312,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512095+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:44:41.512095+0000","last_peered":"2024-08-17T17:44:41.512095+0000","last_clean":"2024-08-17T17:44:41.512095+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:44:41.512095+0000","last_undegraded":"2024-08-17T17:44:41.512095+0000","last_fullsized":"2024-08-17T17:44:41.512095+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":398,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.636006+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:44:57.636006+0000","last_peered":"2024-08-17T17:44:57.636006+0000","last_clean":"2024-08-17T17:44:57.636006+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:44:57.636006+0000","last_undegraded":"2024-08-17T17:44:57.636006+0000","last_fullsized":"2024-08-17T17:44:57.636006+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":454,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511865+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:44:41.511865+0000","last_peered":"2024-08-17T17:44:41.511865+0000","last_clean":"2024-08-17T17:44:41.511865+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:44:41.511865+0000","last_undegraded":"2024-08-17T17:44:41.511865+0000","last_fullsized":"2024-08-17T17:44:41.511865+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"402'12888","reported_seq":13096,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:12.853357+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:45:12.853357+0000","last_peered":"2024-08-17T17:45:12.853357+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:45:12.853357+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"329'6587","ondisk_log_start":"329'6587","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124256256,"num_objects":1896,"num_object_clones":0,"num_object_copies":5688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12888,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1896},{"shards":"0,3","objects":-5352}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":673,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.493002+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:44:57.493002+0000","last_peered":"2024-08-17T17:44:57.493002+0000","last_clean":"2024-08-17T17:44:57.493002+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:44:57.493002+0000","last_undegraded":"2024-08-17T17:44:57.493002+0000","last_fullsized":"2024-08-17T17:44:57.493002+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":643,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.470526+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:45:13.470526+0000","last_peered":"2024-08-17T17:45:13.470526+0000","last_clean":"2024-08-17T17:45:13.470526+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:45:13.470526+0000","last_undegraded":"2024-08-17T17:45:13.470526+0000","last_fullsized":"2024-08-17T17:45:13.470526+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":659,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491438+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:44:57.491438+0000","last_peered":"2024-08-17T17:44:57.491438+0000","last_clean":"2024-08-17T17:44:57.491438+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:44:57.491438+0000","last_undegraded":"2024-08-17T17:44:57.491438+0000","last_fullsized":"2024-08-17T17:44:57.491438+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"402'12733","reported_seq":27432,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:45:12.137528+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:45:12.137528+0000","last_peered":"2024-08-17T17:45:12.137528+0000","last_clean":"2024-08-17T17:45:12.137528+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:45:12.137528+0000","last_undegraded":"2024-08-17T17:45:12.137528+0000","last_fullsized":"2024-08-17T17:45:12.137528+0000","mapping_epoch":347,"log_start":"328'6172","ondisk_log_start":"328'6172","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120651776,"num_objects":1841,"num_object_clones":0,"num_object_copies":5523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12733,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":331,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.637030+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:44:57.637030+0000","last_peered":"2024-08-17T17:44:57.637030+0000","last_clean":"2024-08-17T17:44:57.637030+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:44:57.637030+0000","last_undegraded":"2024-08-17T17:44:57.637030+0000","last_fullsized":"2024-08-17T17:44:57.637030+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":406,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523700+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:44:41.523700+0000","last_peered":"2024-08-17T17:44:41.523700+0000","last_clean":"2024-08-17T17:44:41.523700+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:44:41.523700+0000","last_undegraded":"2024-08-17T17:44:41.523700+0000","last_fullsized":"2024-08-17T17:44:41.523700+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516754+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:44:41.516754+0000","last_peered":"2024-08-17T17:44:41.516754+0000","last_clean":"2024-08-17T17:44:41.516754+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:44:41.516754+0000","last_undegraded":"2024-08-17T17:44:41.516754+0000","last_fullsized":"2024-08-17T17:44:41.516754+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517027+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:44:41.517027+0000","last_peered":"2024-08-17T17:44:41.517027+0000","last_clean":"2024-08-17T17:44:41.517027+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:44:41.517027+0000","last_undegraded":"2024-08-17T17:44:41.517027+0000","last_fullsized":"2024-08-17T17:44:41.517027+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":399,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517114+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:44:41.517114+0000","last_peered":"2024-08-17T17:44:41.517114+0000","last_clean":"2024-08-17T17:44:41.517114+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:44:41.517114+0000","last_undegraded":"2024-08-17T17:44:41.517114+0000","last_fullsized":"2024-08-17T17:44:41.517114+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":650,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.469171+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:45:13.469171+0000","last_peered":"2024-08-17T17:45:13.469171+0000","last_clean":"2024-08-17T17:45:13.469171+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:45:13.469171+0000","last_undegraded":"2024-08-17T17:45:13.469171+0000","last_fullsized":"2024-08-17T17:45:13.469171+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":187,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492452+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:44:57.492452+0000","last_peered":"2024-08-17T17:44:57.492452+0000","last_clean":"2024-08-17T17:44:57.492452+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:44:57.492452+0000","last_undegraded":"2024-08-17T17:44:57.492452+0000","last_fullsized":"2024-08-17T17:44:57.492452+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.511931+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:44:41.511931+0000","last_peered":"2024-08-17T17:44:41.511931+0000","last_clean":"2024-08-17T17:44:41.511931+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:44:41.511931+0000","last_undegraded":"2024-08-17T17:44:41.511931+0000","last_fullsized":"2024-08-17T17:44:41.511931+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":605,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491019+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:44:57.491019+0000","last_peered":"2024-08-17T17:44:57.491019+0000","last_clean":"2024-08-17T17:44:57.491019+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:44:57.491019+0000","last_undegraded":"2024-08-17T17:44:57.491019+0000","last_fullsized":"2024-08-17T17:44:57.491019+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":450,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.512295+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:44:41.512295+0000","last_peered":"2024-08-17T17:44:41.512295+0000","last_clean":"2024-08-17T17:44:41.512295+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:44:41.512295+0000","last_undegraded":"2024-08-17T17:44:41.512295+0000","last_fullsized":"2024-08-17T17:44:41.512295+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":599,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491076+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:44:57.491076+0000","last_peered":"2024-08-17T17:44:57.491076+0000","last_clean":"2024-08-17T17:44:57.491076+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:44:57.491076+0000","last_undegraded":"2024-08-17T17:44:57.491076+0000","last_fullsized":"2024-08-17T17:44:57.491076+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":373,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.523619+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:44:41.523619+0000","last_peered":"2024-08-17T17:44:41.523619+0000","last_clean":"2024-08-17T17:44:41.523619+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:44:41.523619+0000","last_undegraded":"2024-08-17T17:44:41.523619+0000","last_fullsized":"2024-08-17T17:44:41.523619+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":631,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491626+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:44:57.491626+0000","last_peered":"2024-08-17T17:44:57.491626+0000","last_clean":"2024-08-17T17:44:57.491626+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:44:57.491626+0000","last_undegraded":"2024-08-17T17:44:57.491626+0000","last_fullsized":"2024-08-17T17:44:57.491626+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":347,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492587+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:44:57.492587+0000","last_peered":"2024-08-17T17:44:57.492587+0000","last_clean":"2024-08-17T17:44:57.492587+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:44:57.492587+0000","last_undegraded":"2024-08-17T17:44:57.492587+0000","last_fullsized":"2024-08-17T17:44:57.492587+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":641,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.469144+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:45:13.469144+0000","last_peered":"2024-08-17T17:45:13.469144+0000","last_clean":"2024-08-17T17:45:13.469144+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:45:13.469144+0000","last_undegraded":"2024-08-17T17:45:13.469144+0000","last_fullsized":"2024-08-17T17:45:13.469144+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":599,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492370+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:44:57.492370+0000","last_peered":"2024-08-17T17:44:57.492370+0000","last_clean":"2024-08-17T17:44:57.492370+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:44:57.492370+0000","last_undegraded":"2024-08-17T17:44:57.492370+0000","last_fullsized":"2024-08-17T17:44:57.492370+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":385,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516836+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:44:41.516836+0000","last_peered":"2024-08-17T17:44:41.516836+0000","last_clean":"2024-08-17T17:44:41.516836+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:44:41.516836+0000","last_undegraded":"2024-08-17T17:44:41.516836+0000","last_fullsized":"2024-08-17T17:44:41.516836+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":287,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491847+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:44:57.491847+0000","last_peered":"2024-08-17T17:44:57.491847+0000","last_clean":"2024-08-17T17:44:57.491847+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:44:57.491847+0000","last_undegraded":"2024-08-17T17:44:57.491847+0000","last_fullsized":"2024-08-17T17:44:57.491847+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":344,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.635852+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:44:57.635852+0000","last_peered":"2024-08-17T17:44:57.635852+0000","last_clean":"2024-08-17T17:44:57.635852+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:44:57.635852+0000","last_undegraded":"2024-08-17T17:44:57.635852+0000","last_fullsized":"2024-08-17T17:44:57.635852+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":393,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516957+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:44:41.516957+0000","last_peered":"2024-08-17T17:44:41.516957+0000","last_clean":"2024-08-17T17:44:41.516957+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:44:41.516957+0000","last_undegraded":"2024-08-17T17:44:41.516957+0000","last_fullsized":"2024-08-17T17:44:41.516957+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":256,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517034+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:44:41.517034+0000","last_peered":"2024-08-17T17:44:41.517034+0000","last_clean":"2024-08-17T17:44:41.517034+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:44:41.517034+0000","last_undegraded":"2024-08-17T17:44:41.517034+0000","last_fullsized":"2024-08-17T17:44:41.517034+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":393,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.516694+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:44:41.516694+0000","last_peered":"2024-08-17T17:44:41.516694+0000","last_clean":"2024-08-17T17:44:41.516694+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:44:41.516694+0000","last_undegraded":"2024-08-17T17:44:41.516694+0000","last_fullsized":"2024-08-17T17:44:41.516694+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":256,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.517035+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:44:41.517035+0000","last_peered":"2024-08-17T17:44:41.517035+0000","last_clean":"2024-08-17T17:44:41.517035+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:44:41.517035+0000","last_undegraded":"2024-08-17T17:44:41.517035+0000","last_fullsized":"2024-08-17T17:44:41.517035+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":651,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.491682+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:44:57.491682+0000","last_peered":"2024-08-17T17:44:57.491682+0000","last_clean":"2024-08-17T17:44:57.491682+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:44:57.491682+0000","last_undegraded":"2024-08-17T17:44:57.491682+0000","last_fullsized":"2024-08-17T17:44:57.491682+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":625,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.492681+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:44:57.492681+0000","last_peered":"2024-08-17T17:44:57.492681+0000","last_clean":"2024-08-17T17:44:57.492681+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:44:57.492681+0000","last_undegraded":"2024-08-17T17:44:57.492681+0000","last_fullsized":"2024-08-17T17:44:57.492681+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-08-17T17:44:41.518921+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:44:41.518921+0000","last_peered":"2024-08-17T17:44:41.518921+0000","last_clean":"2024-08-17T17:44:41.518921+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:44:41.518921+0000","last_undegraded":"2024-08-17T17:44:41.518921+0000","last_fullsized":"2024-08-17T17:44:41.518921+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":346,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-08-17T17:44:57.636820+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:44:57.636820+0000","last_peered":"2024-08-17T17:44:57.636820+0000","last_clean":"2024-08-17T17:44:57.636820+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:44:57.636820+0000","last_undegraded":"2024-08-17T17:44:57.636820+0000","last_fullsized":"2024-08-17T17:44:57.636820+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1953562648,"num_objects":29810,"num_object_clones":0,"num_object_copies":89430,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":48065,"num_objects_misplaced":12641,"num_objects_unfound":0,"num_objects_dirty":29810,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":206580,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36697,"num_bytes_recovered":2282422272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8981852160,"data_stored":8981839944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104599,"ondisk_log_size":104599,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170145,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463515,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2023352,"kb_used_data":841264,"kb_used_omap":1,"kb_used_meta":1182078,"kb_avail":91721800,"statfs":{"total":95995035648,"available":93923123200,"internally_reserved":0,"allocated":861454336,"data_stored":859221311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1210448216},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492189,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":631452,"kb_used_data":240276,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":93113700,"statfs":{"total":95995035648,"available":95348428800,"internally_reserved":0,"allocated":246042624,"data_stored":243831899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986780,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1977836,"kb_used_data":481636,"kb_used_omap":1,"kb_used_meta":1496190,"kb_avail":91767316,"statfs":{"total":95995035648,"available":93969731584,"internally_reserved":0,"allocated":493195264,"data_stored":490973179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1532098904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1,1,0,1,3,3,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225609,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1940572,"kb_used_data":962388,"kb_used_omap":4,"kb_used_meta":978171,"kb_avail":91804580,"statfs":{"total":95995035648,"available":94007889920,"internally_reserved":0,"allocated":985485312,"data_stored":983274885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1001647552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075933,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533827,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1911776,"kb_used_data":833688,"kb_used_omap":5,"kb_used_meta":1078074,"kb_avail":91833376,"statfs":{"total":95995035648,"available":94037377024,"internally_reserved":0,"allocated":853696512,"data_stored":851480763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1103948016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089803,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1773496,"kb_used_data":742064,"kb_used_omap":5,"kb_used_meta":1031418,"kb_avail":91971656,"statfs":{"total":95995035648,"available":94178975744,"internally_reserved":0,"allocated":759873536,"data_stored":757640570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1056172091},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":751828992,"data_stored":751828992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":844304384,"data_stored":844300312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":970981376,"data_stored":970981376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":486932480,"data_stored":486932480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":239140864,"data_stored":239140864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":850264064,"data_stored":850264064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1499140096,"data_stored":1499136024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:45:17.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:45:17.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:45:17.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:17.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:17.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:17.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:17.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.546 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:17.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:17.649 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:17.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:17.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:17.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:17 smithi152 ceph-mon[93456]: from='client.37989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:17 smithi152 ceph-mon[93456]: pgmap v1584: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 47988/88062 objects degraded (54.493%); 12398/88062 objects misplaced (14.079%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:17.849 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:17 smithi142 ceph-mon[103614]: from='client.37989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:17.849 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:17 smithi142 ceph-mon[103614]: pgmap v1584: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 47988/88062 objects degraded (54.493%); 12398/88062 objects misplaced (14.079%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:17.849 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:17 smithi142 ceph-mon[106729]: from='client.37989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:17.849 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:17 smithi142 ceph-mon[106729]: pgmap v1584: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 259 op/s; 47988/88062 objects degraded (54.493%); 12398/88062 objects misplaced (14.079%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:17.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:17.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:17.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:17.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:18.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:18.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:18.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:18.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:18.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:18.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:18.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:18.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:18.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:18.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:18.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:18.662 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:18.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:18.768 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:18.769 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:18.790 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:18.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:18.892 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:18.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:18.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:19.010 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:19.010 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:19.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:19.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:19.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:19.173 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:19.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:19.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:19.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:19.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:19.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:19.552 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:19.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:19.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:19.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:19.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:19.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:19.797 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:19 smithi142 ceph-mon[106729]: pgmap v1585: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 47988/86124 objects degraded (55.720%); 12298/86124 objects misplaced (14.279%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:19.797 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:19 smithi142 ceph-mon[103614]: pgmap v1585: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 47988/86124 objects degraded (55.720%); 12298/86124 objects misplaced (14.279%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:19.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:19.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:19 smithi152 ceph-mon[93456]: pgmap v1585: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 257 op/s; 47988/86124 objects degraded (55.720%); 12298/86124 objects misplaced (14.279%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:19.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:19.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:19.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:19.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:20.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:20.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:20.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:20.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:20.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:20.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:20.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:20.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:20.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:20.558 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:45:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:45:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:20.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:20.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:20.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:20.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:20.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:20.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:20.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:20.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:20.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:21.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:21.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:21.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:21.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:21.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:21.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:21.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:21.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:21.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:21.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:21 smithi142 ceph-mon[106729]: pgmap v1586: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 47896/84780 objects degraded (56.494%); 12116/84780 objects misplaced (14.291%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:21.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47896/84780 objects degraded (56.494%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:21 smithi142 ceph-mon[103614]: pgmap v1586: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 47896/84780 objects degraded (56.494%); 12116/84780 objects misplaced (14.291%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:21.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47896/84780 objects degraded (56.494%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:21.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:21.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:21.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:21.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:21 smithi152 ceph-mon[93456]: pgmap v1586: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 258 op/s; 47896/84780 objects degraded (56.494%); 12116/84780 objects misplaced (14.291%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T17:45:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47896/84780 objects degraded (56.494%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:21.833 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:21.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:21.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:21.947 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:45:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:45:21] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-08-17T17:45:21.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:21.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:21.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:45:22.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:22.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:22.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:22.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:22.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:22.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:22.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:22.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:22.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:22.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:22.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:22.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:22.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:22.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:22.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:22.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:22.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:22.912 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:45:22.912 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:22.913 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:22.914 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:22.914 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:22.914 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:22.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:22.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:22.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:23.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:23.120 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:23.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:23.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:23.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:23.472 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:23.503 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:45:23.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:23.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:23.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:23.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:23 smithi152 ceph-mon[93456]: pgmap v1587: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 47814/82623 objects degraded (57.870%); 11747/82623 objects misplaced (14.218%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:45:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3195770222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/271530802' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:23.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:23 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:23.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:23.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:23.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[106729]: pgmap v1587: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 47814/82623 objects degraded (57.870%); 11747/82623 objects misplaced (14.218%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3195770222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/271530802' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[103614]: pgmap v1587: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 47814/82623 objects degraded (57.870%); 11747/82623 objects misplaced (14.218%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3195770222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/271530802' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:23.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:23 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:23.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:23.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:23.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:24.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:24.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:24.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:24.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:24.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:24.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:24.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:24.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:24.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:24.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:24.582 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:24.607 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:24.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:24.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:24.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:24.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:24 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:45:24.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:24 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:24.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:24 smithi142 ceph-mon[106729]: osdmap e404: 8 total, 8 up, 6 in 2024-08-17T17:45:24.744 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:24 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:45:24.744 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:24 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:24.744 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:24 smithi142 ceph-mon[103614]: osdmap e404: 8 total, 8 up, 6 in 2024-08-17T17:45:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:24 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:45:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:24 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:24 smithi152 ceph-mon[93456]: osdmap e404: 8 total, 8 up, 6 in 2024-08-17T17:45:24.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:24.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:24.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:24.997 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:24.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:25.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:25.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:25.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:25.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:25.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:25.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:25.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:25.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:25.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:25.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:25.596 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:25.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:25.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:25 smithi142 ceph-mon[103614]: pgmap v1589: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 47738/81363 objects degraded (58.673%); 11581/81363 objects misplaced (14.234%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:45:25.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:25 smithi142 ceph-mon[106729]: pgmap v1589: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 47738/81363 objects degraded (58.673%); 11581/81363 objects misplaced (14.234%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:45:25.636 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:25.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:25.761 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:25.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:25 smithi152 ceph-mon[93456]: pgmap v1589: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 271 op/s; 47738/81363 objects degraded (58.673%); 11581/81363 objects misplaced (14.234%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:45:25.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:25.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:25.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:25.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:25.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:26.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:26.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:26.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:26.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:26.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:26.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:26.392 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:26.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:26.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:26.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:26.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:26.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:45:26.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:26.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47738/81363 objects degraded (58.673%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:26.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:26.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:26.776 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47738/81363 objects degraded (58.673%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:26.776 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:26.776 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:26.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47738/81363 objects degraded (58.673%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:26.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:26.924 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:26.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:27.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:27.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:27.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:27.173 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:27.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:27.300 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:27.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:27.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:27.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:27.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:27.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:27.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:27.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:27.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:27.765 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:27 smithi142 ceph-mon[106729]: pgmap v1590: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 47654/79713 objects degraded (59.782%); 11310/79713 objects misplaced (14.188%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:27.765 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:27 smithi142 ceph-mon[103614]: pgmap v1590: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 47654/79713 objects degraded (59.782%); 11310/79713 objects misplaced (14.188%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:27.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:27.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:27 smithi152 ceph-mon[93456]: pgmap v1590: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 278 op/s; 47654/79713 objects degraded (59.782%); 11310/79713 objects misplaced (14.188%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:27.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:27.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:27.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:28.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:28.051 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:45:28.051 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 47654/79713 objects degraded (59.782%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2408,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":404,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":26571,"data_bytes":1740183525,"bytes_used":10046615552,"bytes_avail":565923598336,"bytes_total":575970213888,"degraded_objects":47654,"degraded_total":79713,"degraded_ratio":0.59781967809516645,"misplaced_objects":11310,"misplaced_total":79713,"misplaced_ratio":0.1418840088818637,"recovering_objects_per_sec":50,"recovering_bytes_per_sec":3321768,"recovering_keys_per_sec":0,"num_objects_recovered":507,"num_bytes_recovered":33226752,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":278},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":35,"modified":"2024-08-17T17:43:38.379260+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:45:28.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:28.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:28.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:28.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:45:28.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:28.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:28.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:28.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:28.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:28.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:28.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:28.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:28.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:28.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:28.684 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:28.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:28.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:28.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:28.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1748589464' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:45:28.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1748589464' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:45:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1748589464' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:45:28.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:28.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:28.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:29.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:29.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:29.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:29.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:29.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:29.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:29.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:29.440 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:29.542 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.576 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:29.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:29.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:29.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:29.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:29.734 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:45:29.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:45:29.759 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:29.789 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:29 smithi142 ceph-mon[103614]: pgmap v1591: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 47654/77475 objects degraded (61.509%); 11208/77475 objects misplaced (14.467%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:29.789 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:29 smithi142 ceph-mon[106729]: pgmap v1591: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 47654/77475 objects degraded (61.509%); 11208/77475 objects misplaced (14.467%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:29.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:29 smithi152 ceph-mon[93456]: pgmap v1591: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 47654/77475 objects degraded (61.509%); 11208/77475 objects misplaced (14.467%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:29.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:29.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:29.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:29.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:30.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:30.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:30.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:30.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:30.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:30.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:30.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:30.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:30.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:30.522 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:45:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:45:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:30.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:30.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:30.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:30.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:30.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:30 smithi142 ceph-mon[106729]: pgmap v1592: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 289 op/s; 47559/76095 objects degraded (62.500%); 11021/76095 objects misplaced (14.483%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:30.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:30.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:30 smithi142 ceph-mon[103614]: pgmap v1592: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 289 op/s; 47559/76095 objects degraded (62.500%); 11021/76095 objects misplaced (14.483%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:30.611 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:45:30.611 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:45:30.611 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:45:30.611 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:45:30.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:30.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:30.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:30 smithi152 ceph-mon[93456]: pgmap v1592: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 289 op/s; 47559/76095 objects degraded (62.500%); 11021/76095 objects misplaced (14.483%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:45:30.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:30.848 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:30.950 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:30.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:30.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:31.075 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:31.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:31.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:31.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:31.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:31.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:31.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:31.379 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:31.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:45:31.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:31.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:31.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:31.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:31.606 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47559/76095 objects degraded (62.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:31.606 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47559/76095 objects degraded (62.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:31.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:31.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47559/76095 objects degraded (62.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:31.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:31.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:31.867 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:45:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:45:31] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-17T17:45:31.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:31.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:31.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:31.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:32.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:32.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:32.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:32.228 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:32.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:32.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:32.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:32.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:32.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:32.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:32.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:32.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:32.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:32 smithi152 ceph-mon[93456]: pgmap v1593: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 285 op/s; 47472/74040 objects degraded (64.117%); 10659/74040 objects misplaced (14.396%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:32.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:32.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:32 smithi142 ceph-mon[103614]: pgmap v1593: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 285 op/s; 47472/74040 objects degraded (64.117%); 10659/74040 objects misplaced (14.396%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:32.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:32 smithi142 ceph-mon[106729]: pgmap v1593: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 285 op/s; 47472/74040 objects degraded (64.117%); 10659/74040 objects misplaced (14.396%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:32.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:32.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:32.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:33.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:33.109 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:33.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:33.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:33.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:33.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:33.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:33.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:33.511 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:33.539 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:33.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:33.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:33.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:33.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:33.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:33.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:33.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:33.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:33.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:34.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:34.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:34.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:34.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:34.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:34.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:34.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:34.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:34.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:34.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:34.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:34.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:34.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:34.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:34.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:34.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:34.778 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:34.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:34.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:34.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:34.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:35.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:35.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:35.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:35.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:35.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:35.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:35.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:35.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:35.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:35.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:45:35.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:35.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:35.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:35.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:35.659 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:35.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:35.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:35.805 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:35 smithi152 ceph-mon[93456]: pgmap v1594: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 47382/72657 objects degraded (65.213%); 10486/72657 objects misplaced (14.432%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:45:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3347116990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:35.856 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:35 smithi142 ceph-mon[106729]: pgmap v1594: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 47382/72657 objects degraded (65.213%); 10486/72657 objects misplaced (14.432%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:45:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3347116990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:35 smithi142 ceph-mon[103614]: pgmap v1594: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 47382/72657 objects degraded (65.213%); 10486/72657 objects misplaced (14.432%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:45:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3347116990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:35.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:35.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:35.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:36.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:36.057 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:36.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:36.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:36.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:36.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:36.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:36.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:36.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:36.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:36.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:36.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:36.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47382/72657 objects degraded (65.213%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[106729]: pgmap v1595: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 47288/71280 objects degraded (66.341%); 10252/71280 objects misplaced (14.383%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47382/72657 objects degraded (65.213%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:36.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:36 smithi142 ceph-mon[103614]: pgmap v1595: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 47288/71280 objects degraded (66.341%); 10252/71280 objects misplaced (14.383%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:45:36.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:36.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:36.810 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47382/72657 objects degraded (65.213%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:36 smithi152 ceph-mon[93456]: pgmap v1595: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 279 op/s; 47288/71280 objects degraded (66.341%); 10252/71280 objects misplaced (14.383%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T17:45:36.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:36.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:36.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:36.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:37.039 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:37.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:37.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:37.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:37.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:37.314 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:37.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:37.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:37.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:37.541 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:37.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:37.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:37.671 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:45:37.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:37.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:37.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:37.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:37.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:37.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:37.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:38.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:38.048 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3453260843' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:38.048 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:37 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:38.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3453260843' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:38.049 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:37 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:38.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:38.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3453260843' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:38.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:37 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:45:38.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:38.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:38.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:38.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:38.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:45:38.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:38.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:38.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:38.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:38.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:38.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:38.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:38.718 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:38.718 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:38.718 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[106729]: osdmap e405: 8 total, 8 up, 6 in 2024-08-17T17:45:38.718 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[106729]: pgmap v1597: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 272 op/s; 47288/69297 objects degraded (68.240%); 10156/69297 objects misplaced (14.656%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:45:38.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:38.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:38.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[103614]: osdmap e405: 8 total, 8 up, 6 in 2024-08-17T17:45:38.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:38 smithi142 ceph-mon[103614]: pgmap v1597: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 272 op/s; 47288/69297 objects degraded (68.240%); 10156/69297 objects misplaced (14.656%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:45:38.817 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:38.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:38.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:38.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:38.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:38 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:38 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:45:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:38 smithi152 ceph-mon[93456]: osdmap e405: 8 total, 8 up, 6 in 2024-08-17T17:45:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:38 smithi152 ceph-mon[93456]: pgmap v1597: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 272 op/s; 47288/69297 objects degraded (68.240%); 10156/69297 objects misplaced (14.656%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:45:39.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:39.100 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:39.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:39.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:39.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:39.349 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:39.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:39.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:39.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:39.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:39.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:39.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:39.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:39.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:39.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:39.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:39.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:39.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:39.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:40.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:40.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:40.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:40.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:40.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:40.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:40.462 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:40.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:40.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:45:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:45:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:45:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:40.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:40.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:40.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:40.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:40.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:40.891 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:40.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:40.936 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:45:40.936 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:45:40.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:41.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:41.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:41.088 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:45:41.089 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:45:41.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:41.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:41.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:41.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:41.272 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:41.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:41.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:41.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:41.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:41.522 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:41.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:41.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:41.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:41.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:41.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[106729]: pgmap v1598: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 275 op/s; 47199/67842 objects degraded (69.572%); 9957/67842 objects misplaced (14.677%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 47199/67842 objects degraded (69.572%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[103614]: pgmap v1598: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 275 op/s; 47199/67842 objects degraded (69.572%); 9957/67842 objects misplaced (14.677%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 47199/67842 objects degraded (69.572%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:41.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:41.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:41 smithi152 ceph-mon[93456]: pgmap v1598: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 275 op/s; 47199/67842 objects degraded (69.572%); 9957/67842 objects misplaced (14.677%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:45:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 47199/67842 objects degraded (69.572%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:41.873 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:41.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:41.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:41.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:41.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:42.023 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:42.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:42.058 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:45:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:45:41] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-08-17T17:45:42.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:45:42.126 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:42.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:42.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:42.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:42.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:42.407 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:42.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:42.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:42.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:42.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:42.666 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1598,"stamp":"2024-08-17T17:45:40.415707+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1480857573,"num_objects":22614,"num_object_clones":0,"num_object_copies":67842,"num_objects_missing_on_primary":2107,"num_objects_missing":2107,"num_objects_degraded":47199,"num_objects_misplaced":9957,"num_objects_unfound":0,"num_objects_dirty":22614,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":214080,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38012,"num_bytes_recovered":2368283278,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104928,"ondisk_log_size":104928,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9282732,"kb_used_data":3111668,"kb_used_omap":27,"kb_used_meta":6170980,"kb_avail":553188180,"statfs":{"total":575970213888,"available":566464696320,"internally_reserved":0,"allocated":3186348032,"data_stored":3172906160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6319084264},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-180289536,"num_objects":-2751,"num_object_clones":0,"num_object_copies":-8253,"num_objects_missing_on_primary":-185,"num_objects_missing":-185,"num_objects_degraded":-360,"num_objects_misplaced":-1064,"num_objects_unfound":0,"num_objects_dirty":-2751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2751,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":514,"num_bytes_recovered":33685504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002754"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":655,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670530+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:45:37.670530+0000","last_peered":"2024-08-17T17:45:37.670530+0000","last_clean":"2024-08-17T17:45:37.670530+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:45:37.670530+0000","last_undegraded":"2024-08-17T17:45:37.670530+0000","last_fullsized":"2024-08-17T17:45:37.670530+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":113,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475568+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:45:13.475568+0000","last_peered":"2024-08-17T17:45:13.475568+0000","last_clean":"2024-08-17T17:45:13.475568+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:45:13.475568+0000","last_undegraded":"2024-08-17T17:45:13.475568+0000","last_fullsized":"2024-08-17T17:45:13.475568+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":308,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.512338+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:45:23.512338+0000","last_peered":"2024-08-17T17:45:23.512338+0000","last_clean":"2024-08-17T17:45:23.512338+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:45:23.512338+0000","last_undegraded":"2024-08-17T17:45:23.512338+0000","last_fullsized":"2024-08-17T17:45:23.512338+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":353,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670595+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:45:37.670595+0000","last_peered":"2024-08-17T17:45:37.670595+0000","last_clean":"2024-08-17T17:45:37.670595+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:45:37.670595+0000","last_undegraded":"2024-08-17T17:45:37.670595+0000","last_fullsized":"2024-08-17T17:45:37.670595+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":530,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670991+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:45:37.670991+0000","last_peered":"2024-08-17T17:45:37.670991+0000","last_clean":"2024-08-17T17:45:37.670991+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:45:37.670991+0000","last_undegraded":"2024-08-17T17:45:37.670991+0000","last_fullsized":"2024-08-17T17:45:37.670991+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":256,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475087+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:45:13.475087+0000","last_peered":"2024-08-17T17:45:13.475087+0000","last_clean":"2024-08-17T17:45:13.475087+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:45:13.475087+0000","last_undegraded":"2024-08-17T17:45:13.475087+0000","last_fullsized":"2024-08-17T17:45:13.475087+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":623,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670653+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:45:37.670653+0000","last_peered":"2024-08-17T17:45:37.670653+0000","last_clean":"2024-08-17T17:45:37.670653+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:45:37.670653+0000","last_undegraded":"2024-08-17T17:45:37.670653+0000","last_fullsized":"2024-08-17T17:45:37.670653+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":424,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.479236+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:45:13.479236+0000","last_peered":"2024-08-17T17:45:13.479236+0000","last_clean":"2024-08-17T17:45:13.479236+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:45:13.479236+0000","last_undegraded":"2024-08-17T17:45:13.479236+0000","last_fullsized":"2024-08-17T17:45:13.479236+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"404'13206","reported_seq":23572,"reported_epoch":404,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:45:35.650824+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:45:35.650824+0000","last_peered":"2024-08-17T17:45:35.650824+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:45:35.650824+0000","last_undegraded":"2024-08-17T17:45:35.650824+0000","last_fullsized":"2024-08-17T17:45:35.650824+0000","mapping_epoch":337,"log_start":"328'6363","ondisk_log_start":"328'6363","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1498,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13206,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5125,"num_bytes_recovered":335872000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":1498}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":109,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.489210+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:45:13.489210+0000","last_peered":"2024-08-17T17:45:13.489210+0000","last_clean":"2024-08-17T17:45:13.489210+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:45:13.489210+0000","last_undegraded":"2024-08-17T17:45:13.489210+0000","last_fullsized":"2024-08-17T17:45:13.489210+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":291,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.511639+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:45:23.511639+0000","last_peered":"2024-08-17T17:45:23.511639+0000","last_clean":"2024-08-17T17:45:23.511639+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:45:23.511639+0000","last_undegraded":"2024-08-17T17:45:23.511639+0000","last_fullsized":"2024-08-17T17:45:23.511639+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"404'13648","reported_seq":13750,"reported_epoch":404,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:37.151488+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:45:37.151488+0000","last_peered":"2024-08-17T17:45:37.151488+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:45:37.151488+0000","last_undegraded":"2024-08-17T17:45:37.151488+0000","last_fullsized":"2024-08-17T17:45:37.151488+0000","mapping_epoch":337,"log_start":"329'7082","ondisk_log_start":"329'7082","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6566,"log_dups_size":0,"ondisk_log_size":6566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93454336,"num_objects":1426,"num_object_clones":0,"num_object_copies":4278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1426,"num_objects_unfound":0,"num_objects_dirty":1426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13648,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":1426}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475559+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:45:13.475559+0000","last_peered":"2024-08-17T17:45:13.475559+0000","last_clean":"2024-08-17T17:45:13.475559+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:45:13.475559+0000","last_undegraded":"2024-08-17T17:45:13.475559+0000","last_fullsized":"2024-08-17T17:45:13.475559+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":375,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475016+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:45:13.475016+0000","last_peered":"2024-08-17T17:45:13.475016+0000","last_clean":"2024-08-17T17:45:13.475016+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:45:13.475016+0000","last_undegraded":"2024-08-17T17:45:13.475016+0000","last_fullsized":"2024-08-17T17:45:13.475016+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"404'13314","reported_seq":13419,"reported_epoch":404,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:35.694580+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:45:35.694580+0000","last_peered":"2024-08-17T17:45:35.694580+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:45:35.694580+0000","last_undegraded":"2024-08-17T17:45:35.694580+0000","last_fullsized":"2024-08-17T17:45:35.694580+0000","mapping_epoch":337,"log_start":"328'6453","ondisk_log_start":"328'6453","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6861,"log_dups_size":0,"ondisk_log_size":6861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93061120,"num_objects":1420,"num_object_clones":0,"num_object_copies":4260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1420,"num_objects_unfound":0,"num_objects_dirty":1420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13314,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1420}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":108,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.512274+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:45:23.512274+0000","last_peered":"2024-08-17T17:45:23.512274+0000","last_clean":"2024-08-17T17:45:23.512274+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:45:23.512274+0000","last_undegraded":"2024-08-17T17:45:23.512274+0000","last_fullsized":"2024-08-17T17:45:23.512274+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":385,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.479268+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:45:13.479268+0000","last_peered":"2024-08-17T17:45:13.479268+0000","last_clean":"2024-08-17T17:45:13.479268+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:45:13.479268+0000","last_undegraded":"2024-08-17T17:45:13.479268+0000","last_fullsized":"2024-08-17T17:45:13.479268+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"405'13557","reported_seq":13749,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:39.951150+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:45:39.951150+0000","last_peered":"2024-08-17T17:45:39.951150+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:45:39.951150+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"328'6513","ondisk_log_start":"328'6513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91815936,"num_objects":1401,"num_object_clones":0,"num_object_copies":4203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13557,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-5368},{"shards":"0,3,7","objects":1401}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":567,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670703+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:45:37.670703+0000","last_peered":"2024-08-17T17:45:37.670703+0000","last_clean":"2024-08-17T17:45:37.670703+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:45:37.670703+0000","last_undegraded":"2024-08-17T17:45:37.670703+0000","last_fullsized":"2024-08-17T17:45:37.670703+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":333,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.713014+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:45:37.713014+0000","last_peered":"2024-08-17T17:45:37.713014+0000","last_clean":"2024-08-17T17:45:37.713014+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:45:37.713014+0000","last_undegraded":"2024-08-17T17:45:37.713014+0000","last_fullsized":"2024-08-17T17:45:37.713014+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"404'13117","reported_seq":13210,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.014758+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:45:37.014758+0000","last_peered":"2024-08-17T17:45:37.014758+0000","last_clean":"2024-08-17T17:45:37.014758+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:45:37.014758+0000","last_undegraded":"2024-08-17T17:45:37.014758+0000","last_fullsized":"2024-08-17T17:45:37.014758+0000","mapping_epoch":319,"log_start":"329'6574","ondisk_log_start":"329'6574","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6543,"log_dups_size":0,"ondisk_log_size":6543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96796672,"num_objects":1477,"num_object_clones":0,"num_object_copies":4431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13117,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":656,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670636+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:45:37.670636+0000","last_peered":"2024-08-17T17:45:37.670636+0000","last_clean":"2024-08-17T17:45:37.670636+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:45:37.670636+0000","last_undegraded":"2024-08-17T17:45:37.670636+0000","last_fullsized":"2024-08-17T17:45:37.670636+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":358,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475708+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:45:13.475708+0000","last_peered":"2024-08-17T17:45:13.475708+0000","last_clean":"2024-08-17T17:45:13.475708+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:45:13.475708+0000","last_undegraded":"2024-08-17T17:45:13.475708+0000","last_fullsized":"2024-08-17T17:45:13.475708+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"404'13440","reported_seq":13543,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:37.117938+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:45:37.117938+0000","last_peered":"2024-08-17T17:45:37.117938+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:45:37.117938+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"329'6835","ondisk_log_start":"329'6835","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13440,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":1492},{"shards":"1,6","objects":-5874}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":359,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.479390+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:45:13.479390+0000","last_peered":"2024-08-17T17:45:13.479390+0000","last_clean":"2024-08-17T17:45:13.479390+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:45:13.479390+0000","last_undegraded":"2024-08-17T17:45:13.479390+0000","last_fullsized":"2024-08-17T17:45:13.479390+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":385,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.476140+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:45:13.476140+0000","last_peered":"2024-08-17T17:45:13.476140+0000","last_clean":"2024-08-17T17:45:13.476140+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:45:13.476140+0000","last_undegraded":"2024-08-17T17:45:13.476140+0000","last_fullsized":"2024-08-17T17:45:13.476140+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"405'13616","reported_seq":13809,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:38.983310+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:45:38.983310+0000","last_peered":"2024-08-17T17:45:38.983310+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:45:38.983310+0000","last_undegraded":"2024-08-17T17:45:38.983310+0000","last_fullsized":"2024-08-17T17:45:38.983310+0000","mapping_epoch":337,"log_start":"335'7961","ondisk_log_start":"335'7961","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91357184,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1394,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13616,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1394}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475225+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:45:13.475225+0000","last_peered":"2024-08-17T17:45:13.475225+0000","last_clean":"2024-08-17T17:45:13.475225+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:45:13.475225+0000","last_undegraded":"2024-08-17T17:45:13.475225+0000","last_fullsized":"2024-08-17T17:45:13.475225+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":358,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475697+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:45:13.475697+0000","last_peered":"2024-08-17T17:45:13.475697+0000","last_clean":"2024-08-17T17:45:13.475697+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:45:13.475697+0000","last_undegraded":"2024-08-17T17:45:13.475697+0000","last_fullsized":"2024-08-17T17:45:13.475697+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"404'13289","reported_seq":23874,"reported_epoch":404,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:35.717706+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:45:35.717706+0000","last_peered":"2024-08-17T17:45:35.717706+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:45:35.717706+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:45:35.717706+0000","mapping_epoch":337,"log_start":"328'6441","ondisk_log_start":"328'6441","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6848,"log_dups_size":0,"ondisk_log_size":6848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92733440,"num_objects":1415,"num_object_clones":0,"num_object_copies":4245,"num_objects_missing_on_primary":2107,"num_objects_missing":2107,"num_objects_degraded":1415,"num_objects_misplaced":2107,"num_objects_unfound":0,"num_objects_dirty":1415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13289,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5231,"num_bytes_recovered":342753280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":14},{"shards":"1,2,7","objects":1401}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":626,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670426+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:45:37.670426+0000","last_peered":"2024-08-17T17:45:37.670426+0000","last_clean":"2024-08-17T17:45:37.670426+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:45:37.670426+0000","last_undegraded":"2024-08-17T17:45:37.670426+0000","last_fullsized":"2024-08-17T17:45:37.670426+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":563,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670736+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:45:37.670736+0000","last_peered":"2024-08-17T17:45:37.670736+0000","last_clean":"2024-08-17T17:45:37.670736+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:45:37.670736+0000","last_undegraded":"2024-08-17T17:45:37.670736+0000","last_fullsized":"2024-08-17T17:45:37.670736+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"405'13534","reported_seq":24388,"reported_epoch":405,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:39.010710+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:45:39.010710+0000","last_peered":"2024-08-17T17:45:39.010710+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:45:39.010710+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:45:39.010710+0000","mapping_epoch":337,"log_start":"335'7944","ondisk_log_start":"335'7944","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5590,"log_dups_size":0,"ondisk_log_size":5590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89194520,"num_objects":1362,"num_object_clones":0,"num_object_copies":4086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1362,"num_objects_misplaced":2112,"num_objects_unfound":0,"num_objects_dirty":1362,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13534,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5336,"num_bytes_recovered":349634560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1362}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":756,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:37.712535+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:45:37.712535+0000","last_peered":"2024-08-17T17:45:37.712535+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:45:37.712535+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":334,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:45:23.511885+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:45:23.511885+0000","last_peered":"2024-08-17T17:45:23.511885+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:45:23.511885+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:45:23.511885+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":605,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670564+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:45:37.670564+0000","last_peered":"2024-08-17T17:45:37.670564+0000","last_clean":"2024-08-17T17:45:37.670564+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:45:37.670564+0000","last_undegraded":"2024-08-17T17:45:37.670564+0000","last_fullsized":"2024-08-17T17:45:37.670564+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":368,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.480614+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:45:13.480614+0000","last_peered":"2024-08-17T17:45:13.480614+0000","last_clean":"2024-08-17T17:45:13.480614+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:45:13.480614+0000","last_undegraded":"2024-08-17T17:45:13.480614+0000","last_fullsized":"2024-08-17T17:45:13.480614+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"405'13361","reported_seq":13556,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:39.967620+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:45:39.967620+0000","last_peered":"2024-08-17T17:45:39.967620+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:45:39.967620+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"328'6321","ondisk_log_start":"328'6321","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90636288,"num_objects":1383,"num_object_clones":0,"num_object_copies":4149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13361,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-5264},{"shards":"0,4,7","objects":1383}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":421,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669072+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:45:37.669072+0000","last_peered":"2024-08-17T17:45:37.669072+0000","last_clean":"2024-08-17T17:45:37.669072+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:45:37.669072+0000","last_undegraded":"2024-08-17T17:45:37.669072+0000","last_fullsized":"2024-08-17T17:45:37.669072+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":340,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.474883+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:45:13.474883+0000","last_peered":"2024-08-17T17:45:13.474883+0000","last_clean":"2024-08-17T17:45:13.474883+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:45:13.474883+0000","last_undegraded":"2024-08-17T17:45:13.474883+0000","last_fullsized":"2024-08-17T17:45:13.474883+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":347,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.715058+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:45:37.715058+0000","last_peered":"2024-08-17T17:45:37.715058+0000","last_clean":"2024-08-17T17:45:37.715058+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:45:37.715058+0000","last_undegraded":"2024-08-17T17:45:37.715058+0000","last_fullsized":"2024-08-17T17:45:37.715058+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"404'13444","reported_seq":13547,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:37.193980+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:45:37.193980+0000","last_peered":"2024-08-17T17:45:37.193980+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:45:37.193980+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"329'6917","ondisk_log_start":"329'6917","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604672,"num_objects":1352,"num_object_clones":0,"num_object_copies":4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13444,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1352},{"shards":"5,6","objects":-5944}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":513,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670340+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:45:37.670340+0000","last_peered":"2024-08-17T17:45:37.670340+0000","last_clean":"2024-08-17T17:45:37.670340+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:45:37.670340+0000","last_undegraded":"2024-08-17T17:45:37.670340+0000","last_fullsized":"2024-08-17T17:45:37.670340+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.479030+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:45:13.479030+0000","last_peered":"2024-08-17T17:45:13.479030+0000","last_clean":"2024-08-17T17:45:13.479030+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:45:13.479030+0000","last_undegraded":"2024-08-17T17:45:13.479030+0000","last_fullsized":"2024-08-17T17:45:13.479030+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":698,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669744+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:45:37.669744+0000","last_peered":"2024-08-17T17:45:37.669744+0000","last_clean":"2024-08-17T17:45:37.669744+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:45:37.669744+0000","last_undegraded":"2024-08-17T17:45:37.669744+0000","last_fullsized":"2024-08-17T17:45:37.669744+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"405'13285","reported_seq":14204,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:39.815063+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:45:39.815063+0000","last_peered":"2024-08-17T17:45:39.815063+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:45:39.815063+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"328'6216","ondisk_log_start":"328'6216","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89194496,"num_objects":1361,"num_object_clones":0,"num_object_copies":4083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13285,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":1361},{"shards":"0,5","objects":-5293}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":490,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669287+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:45:37.669287+0000","last_peered":"2024-08-17T17:45:37.669287+0000","last_clean":"2024-08-17T17:45:37.669287+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:45:37.669287+0000","last_undegraded":"2024-08-17T17:45:37.669287+0000","last_fullsized":"2024-08-17T17:45:37.669287+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":379,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475911+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:45:13.475911+0000","last_peered":"2024-08-17T17:45:13.475911+0000","last_clean":"2024-08-17T17:45:13.475911+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:45:13.475911+0000","last_undegraded":"2024-08-17T17:45:13.475911+0000","last_fullsized":"2024-08-17T17:45:13.475911+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":284,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670423+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:45:37.670423+0000","last_peered":"2024-08-17T17:45:37.670423+0000","last_clean":"2024-08-17T17:45:37.670423+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:45:37.670423+0000","last_undegraded":"2024-08-17T17:45:37.670423+0000","last_fullsized":"2024-08-17T17:45:37.670423+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"405'13443","reported_seq":28481,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.861572+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:45:37.861572+0000","last_peered":"2024-08-17T17:45:37.861572+0000","last_clean":"2024-08-17T17:45:37.861572+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:45:37.861572+0000","last_undegraded":"2024-08-17T17:45:37.861572+0000","last_fullsized":"2024-08-17T17:45:37.861572+0000","mapping_epoch":355,"log_start":"330'7130","ondisk_log_start":"330'7130","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91291648,"num_objects":1393,"num_object_clones":0,"num_object_copies":4179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13443,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":319,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.511803+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:45:23.511803+0000","last_peered":"2024-08-17T17:45:23.511803+0000","last_clean":"2024-08-17T17:45:23.511803+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:45:23.511803+0000","last_undegraded":"2024-08-17T17:45:23.511803+0000","last_fullsized":"2024-08-17T17:45:23.511803+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":153,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670323+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:45:37.670323+0000","last_peered":"2024-08-17T17:45:37.670323+0000","last_clean":"2024-08-17T17:45:37.670323+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:45:37.670323+0000","last_undegraded":"2024-08-17T17:45:37.670323+0000","last_fullsized":"2024-08-17T17:45:37.670323+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":649,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669525+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:45:37.669525+0000","last_peered":"2024-08-17T17:45:37.669525+0000","last_clean":"2024-08-17T17:45:37.669525+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:45:37.669525+0000","last_undegraded":"2024-08-17T17:45:37.669525+0000","last_fullsized":"2024-08-17T17:45:37.669525+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"405'13085","reported_seq":27691,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.871002+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:45:37.871002+0000","last_peered":"2024-08-17T17:45:37.871002+0000","last_clean":"2024-08-17T17:45:37.871002+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:45:37.871002+0000","last_undegraded":"2024-08-17T17:45:37.871002+0000","last_fullsized":"2024-08-17T17:45:37.871002+0000","mapping_epoch":345,"log_start":"329'6772","ondisk_log_start":"329'6772","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86441984,"num_objects":1319,"num_object_clones":0,"num_object_copies":3957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13085,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":315,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.512011+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:45:23.512011+0000","last_peered":"2024-08-17T17:45:23.512011+0000","last_clean":"2024-08-17T17:45:23.512011+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:45:23.512011+0000","last_undegraded":"2024-08-17T17:45:23.512011+0000","last_fullsized":"2024-08-17T17:45:23.512011+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":404,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.713153+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:45:37.713153+0000","last_peered":"2024-08-17T17:45:37.713153+0000","last_clean":"2024-08-17T17:45:37.713153+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:45:37.713153+0000","last_undegraded":"2024-08-17T17:45:37.713153+0000","last_fullsized":"2024-08-17T17:45:37.713153+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":457,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.511776+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:45:23.511776+0000","last_peered":"2024-08-17T17:45:23.511776+0000","last_clean":"2024-08-17T17:45:23.511776+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:45:23.511776+0000","last_undegraded":"2024-08-17T17:45:23.511776+0000","last_fullsized":"2024-08-17T17:45:23.511776+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"405'13321","reported_seq":13535,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:37.735478+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:45:37.735478+0000","last_peered":"2024-08-17T17:45:37.735478+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:45:37.735478+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"330'6987","ondisk_log_start":"330'6987","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6334,"log_dups_size":0,"ondisk_log_size":6334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95879168,"num_objects":1463,"num_object_clones":0,"num_object_copies":4389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13321,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1463},{"shards":"0,3","objects":-5785}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":679,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670844+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:45:37.670844+0000","last_peered":"2024-08-17T17:45:37.670844+0000","last_clean":"2024-08-17T17:45:37.670844+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:45:37.670844+0000","last_undegraded":"2024-08-17T17:45:37.670844+0000","last_fullsized":"2024-08-17T17:45:37.670844+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":647,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669913+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:45:37.669913+0000","last_peered":"2024-08-17T17:45:37.669913+0000","last_clean":"2024-08-17T17:45:37.669913+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:45:37.669913+0000","last_undegraded":"2024-08-17T17:45:37.669913+0000","last_fullsized":"2024-08-17T17:45:37.669913+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":665,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670230+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:45:37.670230+0000","last_peered":"2024-08-17T17:45:37.670230+0000","last_clean":"2024-08-17T17:45:37.670230+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:45:37.670230+0000","last_undegraded":"2024-08-17T17:45:37.670230+0000","last_fullsized":"2024-08-17T17:45:37.670230+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"404'13142","reported_seq":27843,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:36.888224+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:45:36.888224+0000","last_peered":"2024-08-17T17:45:36.888224+0000","last_clean":"2024-08-17T17:45:36.888224+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:45:36.888224+0000","last_undegraded":"2024-08-17T17:45:36.888224+0000","last_fullsized":"2024-08-17T17:45:36.888224+0000","mapping_epoch":347,"log_start":"329'6572","ondisk_log_start":"329'6572","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93847552,"num_objects":1432,"num_object_clones":0,"num_object_copies":4296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13142,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":337,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.714639+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:45:37.714639+0000","last_peered":"2024-08-17T17:45:37.714639+0000","last_clean":"2024-08-17T17:45:37.714639+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:45:37.714639+0000","last_undegraded":"2024-08-17T17:45:37.714639+0000","last_fullsized":"2024-08-17T17:45:37.714639+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":408,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.480696+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:45:13.480696+0000","last_peered":"2024-08-17T17:45:13.480696+0000","last_clean":"2024-08-17T17:45:13.480696+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:45:13.480696+0000","last_undegraded":"2024-08-17T17:45:13.480696+0000","last_fullsized":"2024-08-17T17:45:13.480696+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":358,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475153+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:45:13.475153+0000","last_peered":"2024-08-17T17:45:13.475153+0000","last_clean":"2024-08-17T17:45:13.475153+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:45:13.475153+0000","last_undegraded":"2024-08-17T17:45:13.475153+0000","last_fullsized":"2024-08-17T17:45:13.475153+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":358,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475772+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:45:13.475772+0000","last_peered":"2024-08-17T17:45:13.475772+0000","last_clean":"2024-08-17T17:45:13.475772+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:45:13.475772+0000","last_undegraded":"2024-08-17T17:45:13.475772+0000","last_fullsized":"2024-08-17T17:45:13.475772+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":401,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475880+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:45:13.475880+0000","last_peered":"2024-08-17T17:45:13.475880+0000","last_clean":"2024-08-17T17:45:13.475880+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:45:13.475880+0000","last_undegraded":"2024-08-17T17:45:13.475880+0000","last_fullsized":"2024-08-17T17:45:13.475880+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":654,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669378+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:45:37.669378+0000","last_peered":"2024-08-17T17:45:37.669378+0000","last_clean":"2024-08-17T17:45:37.669378+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:45:37.669378+0000","last_undegraded":"2024-08-17T17:45:37.669378+0000","last_fullsized":"2024-08-17T17:45:37.669378+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":193,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.671014+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:45:37.671014+0000","last_peered":"2024-08-17T17:45:37.671014+0000","last_clean":"2024-08-17T17:45:37.671014+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:45:37.671014+0000","last_undegraded":"2024-08-17T17:45:37.671014+0000","last_fullsized":"2024-08-17T17:45:37.671014+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":309,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.512180+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:45:23.512180+0000","last_peered":"2024-08-17T17:45:23.512180+0000","last_clean":"2024-08-17T17:45:23.512180+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:45:23.512180+0000","last_undegraded":"2024-08-17T17:45:23.512180+0000","last_fullsized":"2024-08-17T17:45:23.512180+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":612,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670178+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:45:37.670178+0000","last_peered":"2024-08-17T17:45:37.670178+0000","last_clean":"2024-08-17T17:45:37.670178+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:45:37.670178+0000","last_undegraded":"2024-08-17T17:45:37.670178+0000","last_fullsized":"2024-08-17T17:45:37.670178+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":453,"reported_epoch":404,"state":"active+clean","last_fresh":"2024-08-17T17:45:23.514010+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:45:23.514010+0000","last_peered":"2024-08-17T17:45:23.514010+0000","last_clean":"2024-08-17T17:45:23.514010+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:45:23.514010+0000","last_undegraded":"2024-08-17T17:45:23.514010+0000","last_fullsized":"2024-08-17T17:45:23.514010+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":606,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670134+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:45:37.670134+0000","last_peered":"2024-08-17T17:45:37.670134+0000","last_clean":"2024-08-17T17:45:37.670134+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:45:37.670134+0000","last_undegraded":"2024-08-17T17:45:37.670134+0000","last_fullsized":"2024-08-17T17:45:37.670134+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":375,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.480622+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:45:13.480622+0000","last_peered":"2024-08-17T17:45:13.480622+0000","last_clean":"2024-08-17T17:45:13.480622+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:45:13.480622+0000","last_undegraded":"2024-08-17T17:45:13.480622+0000","last_fullsized":"2024-08-17T17:45:13.480622+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":637,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670314+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:45:37.670314+0000","last_peered":"2024-08-17T17:45:37.670314+0000","last_clean":"2024-08-17T17:45:37.670314+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:45:37.670314+0000","last_undegraded":"2024-08-17T17:45:37.670314+0000","last_fullsized":"2024-08-17T17:45:37.670314+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":353,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670988+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:45:37.670988+0000","last_peered":"2024-08-17T17:45:37.670988+0000","last_clean":"2024-08-17T17:45:37.670988+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:45:37.670988+0000","last_undegraded":"2024-08-17T17:45:37.670988+0000","last_fullsized":"2024-08-17T17:45:37.670988+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":645,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.669253+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:45:37.669253+0000","last_peered":"2024-08-17T17:45:37.669253+0000","last_clean":"2024-08-17T17:45:37.669253+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:45:37.669253+0000","last_undegraded":"2024-08-17T17:45:37.669253+0000","last_fullsized":"2024-08-17T17:45:37.669253+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":605,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670900+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:45:37.670900+0000","last_peered":"2024-08-17T17:45:37.670900+0000","last_clean":"2024-08-17T17:45:37.670900+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:45:37.670900+0000","last_undegraded":"2024-08-17T17:45:37.670900+0000","last_fullsized":"2024-08-17T17:45:37.670900+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":387,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475397+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:45:13.475397+0000","last_peered":"2024-08-17T17:45:13.475397+0000","last_clean":"2024-08-17T17:45:13.475397+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:45:13.475397+0000","last_undegraded":"2024-08-17T17:45:13.475397+0000","last_fullsized":"2024-08-17T17:45:13.475397+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":293,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670626+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:45:37.670626+0000","last_peered":"2024-08-17T17:45:37.670626+0000","last_clean":"2024-08-17T17:45:37.670626+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:45:37.670626+0000","last_undegraded":"2024-08-17T17:45:37.670626+0000","last_fullsized":"2024-08-17T17:45:37.670626+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":350,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.712921+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:45:37.712921+0000","last_peered":"2024-08-17T17:45:37.712921+0000","last_clean":"2024-08-17T17:45:37.712921+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:45:37.712921+0000","last_undegraded":"2024-08-17T17:45:37.712921+0000","last_fullsized":"2024-08-17T17:45:37.712921+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":395,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475702+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:45:13.475702+0000","last_peered":"2024-08-17T17:45:13.475702+0000","last_clean":"2024-08-17T17:45:13.475702+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:45:13.475702+0000","last_undegraded":"2024-08-17T17:45:13.475702+0000","last_fullsized":"2024-08-17T17:45:13.475702+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":258,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475795+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:45:13.475795+0000","last_peered":"2024-08-17T17:45:13.475795+0000","last_clean":"2024-08-17T17:45:13.475795+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:45:13.475795+0000","last_undegraded":"2024-08-17T17:45:13.475795+0000","last_fullsized":"2024-08-17T17:45:13.475795+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":395,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.475318+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:45:13.475318+0000","last_peered":"2024-08-17T17:45:13.475318+0000","last_clean":"2024-08-17T17:45:13.475318+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:45:13.475318+0000","last_undegraded":"2024-08-17T17:45:13.475318+0000","last_fullsized":"2024-08-17T17:45:13.475318+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":258,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.476075+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:45:13.476075+0000","last_peered":"2024-08-17T17:45:13.476075+0000","last_clean":"2024-08-17T17:45:13.476075+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:45:13.476075+0000","last_undegraded":"2024-08-17T17:45:13.476075+0000","last_fullsized":"2024-08-17T17:45:13.476075+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":657,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.670373+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:45:37.670373+0000","last_peered":"2024-08-17T17:45:37.670373+0000","last_clean":"2024-08-17T17:45:37.670373+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:45:37.670373+0000","last_undegraded":"2024-08-17T17:45:37.670373+0000","last_fullsized":"2024-08-17T17:45:37.670373+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":631,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.671046+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:45:37.671046+0000","last_peered":"2024-08-17T17:45:37.671046+0000","last_clean":"2024-08-17T17:45:37.671046+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:45:37.671046+0000","last_undegraded":"2024-08-17T17:45:37.671046+0000","last_fullsized":"2024-08-17T17:45:37.671046+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":358,"reported_epoch":403,"state":"active+clean","last_fresh":"2024-08-17T17:45:13.474879+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:45:13.474879+0000","last_peered":"2024-08-17T17:45:13.474879+0000","last_clean":"2024-08-17T17:45:13.474879+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:45:13.474879+0000","last_undegraded":"2024-08-17T17:45:13.474879+0000","last_fullsized":"2024-08-17T17:45:13.474879+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":353,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.713286+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:45:37.713286+0000","last_peered":"2024-08-17T17:45:37.713286+0000","last_clean":"2024-08-17T17:45:37.713286+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:45:37.713286+0000","last_undegraded":"2024-08-17T17:45:37.713286+0000","last_fullsized":"2024-08-17T17:45:37.713286+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1480261656,"num_objects":22588,"num_object_clones":0,"num_object_copies":67764,"num_objects_missing_on_primary":2107,"num_objects_missing":2107,"num_objects_degraded":47196,"num_objects_misplaced":9957,"num_objects_unfound":0,"num_objects_dirty":22588,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":213802,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37971,"num_bytes_recovered":2365915136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7819964416,"data_stored":7819952200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104721,"ondisk_log_size":104721,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170150,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463520,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830360,"kb_used_data":648272,"kb_used_omap":1,"kb_used_meta":1182078,"kb_avail":91914792,"statfs":{"total":95995035648,"available":94120747008,"internally_reserved":0,"allocated":663830528,"data_stored":661581917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1210448216},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492195,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576332,"kb_used_data":176836,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":93168820,"statfs":{"total":95995035648,"available":95404871680,"internally_reserved":0,"allocated":181080064,"data_stored":178845960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986785,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1875660,"kb_used_data":375300,"kb_used_omap":1,"kb_used_meta":1500350,"kb_avail":91869492,"statfs":{"total":95995035648,"available":94074359808,"internally_reserved":0,"allocated":384307200,"data_stored":382069529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1536358744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225614,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1719740,"kb_used_data":735988,"kb_used_omap":4,"kb_used_meta":983739,"kb_avail":92025412,"statfs":{"total":95995035648,"available":94234021888,"internally_reserved":0,"allocated":753651712,"data_stored":751425699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1007349184},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075939,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533832,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1716948,"kb_used_data":638856,"kb_used_omap":5,"kb_used_meta":1078074,"kb_avail":92028204,"statfs":{"total":95995035648,"available":94236880896,"internally_reserved":0,"allocated":654188544,"data_stored":651949416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1103948016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089809,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1563692,"kb_used_data":536416,"kb_used_omap":5,"kb_used_meta":1027258,"kb_avail":92181460,"statfs":{"total":95995035648,"available":94393815040,"internally_reserved":0,"allocated":549289984,"data_stored":547033639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1051912251},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":542441472,"data_stored":542441472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":639373312,"data_stored":639369240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":742850560,"data_stored":742850560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":376635392,"data_stored":376635392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":175308800,"data_stored":175308800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":653721600,"data_stored":653721600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1350373376,"data_stored":1350369304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:45:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.671 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:42.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:42.750 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:42.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:42.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:42 smithi152 ceph-mon[93456]: from='client.38025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:42 smithi142 ceph-mon[106729]: from='client.38025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:42 smithi142 ceph-mon[103614]: from='client.38025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:45:42.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:42.924 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:42.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:43.026 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:43.049 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:43.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:43.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:43.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:43.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:43.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:43.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:43.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:43.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:43.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:43.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:43.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:43.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:43.799 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:43 smithi142 ceph-mon[106729]: pgmap v1599: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 47102/65532 objects degraded (71.876%); 9567/65532 objects misplaced (14.599%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:45:43.800 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:43 smithi142 ceph-mon[103614]: pgmap v1599: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 47102/65532 objects degraded (71.876%); 9567/65532 objects misplaced (14.599%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:45:43.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:43 smithi152 ceph-mon[93456]: pgmap v1599: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 47102/65532 objects degraded (71.876%); 9567/65532 objects misplaced (14.599%); 3.3 MiB/s, 53 objects/s recovering 2024-08-17T17:45:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:43.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:43.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:43.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:44.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:44.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:44.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:44.207 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:44.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:44.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:44.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:44.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:44.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:44.557 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:44.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:44.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:44.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:44.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:44.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:44.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:44.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:44.895 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:44.901 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:44.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:44.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:44.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:44.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:45.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:45.094 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:45.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:45.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:45.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:45.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:45.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:45.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:45.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:45.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:45.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:45.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:45.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:45.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:45.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:45:45.705 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:45.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:45 smithi152 ceph-mon[93456]: pgmap v1600: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 46991/64023 objects degraded (73.397%); 9357/64023 objects misplaced (14.615%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:45:45.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:45 smithi142 ceph-mon[106729]: pgmap v1600: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 46991/64023 objects degraded (73.397%); 9357/64023 objects misplaced (14.615%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:45:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:45 smithi142 ceph-mon[103614]: pgmap v1600: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 46991/64023 objects degraded (73.397%); 9357/64023 objects misplaced (14.615%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:45:45.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:45.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:45.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:46.007 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:46.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:46.108 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:46.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:46.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:46.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:46.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:46.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:46.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:46.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:46.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:46.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:46.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 46991/64023 objects degraded (73.397%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:46.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 46991/64023 objects degraded (73.397%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:46.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:46.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:46.761 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 46991/64023 objects degraded (73.397%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:46.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:46.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:46.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:46.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:46.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:47.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:47.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:47.141 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:47.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:47.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:47.378 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:47.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:47.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:47.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:47.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:47.659 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:47.742 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:47 smithi142 ceph-mon[106729]: pgmap v1601: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 46889/62235 objects degraded (75.342%); 9052/62235 objects misplaced (14.545%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:45:47.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:47 smithi142 ceph-mon[103614]: pgmap v1601: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 46889/62235 objects degraded (75.342%); 9052/62235 objects misplaced (14.545%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:45:47.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:47.785 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:47.796 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:45:47.796 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:45:47.796 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:45:47.796 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:45:47.796 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:45:47.797 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:45:47.798 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:45:47.798 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:45:47.799 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:45:47.799 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:45:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:47 smithi152 ceph-mon[93456]: pgmap v1601: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.3 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 46889/62235 objects degraded (75.342%); 9052/62235 objects misplaced (14.545%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:45:47.887 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:47.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:47.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:48.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:48.036 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:48.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:48.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:48.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:48.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:48.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:48.412 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:48.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:48.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:48.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:48.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:48.750 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3120760166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:48.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3120760166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:48.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:48.790 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3120760166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:45:48.892 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:48.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:48.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:48.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:49.023 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:49.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:49.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:49.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:49.271 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:45:49.271 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:45:49.271 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e354/337 cost 1049576)", 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:14.850423+0000", 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 574.31821434300002, 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00094614500000000002, 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.272 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850423+0000", 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850423+0000", 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850424+0000", 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1960000000000001e-06 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.273 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850435+0000", 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1375e-05 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850450+0000", 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4575e-05 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.274 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850458+0000", 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3899999999999993e-06 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.850520+0000", 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1626999999999998e-05 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.275 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:14.851369+0000", 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00084898199999999997 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e356/337 cost 1049576)", 2024-08-17T17:45:49.276 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:25.354294+0000", 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 563.81434347100003, 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011246629999999999, 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354294+0000", 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.277 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354294+0000", 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354295+0000", 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1230000000000001e-06 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354307+0000", 2024-08-17T17:45:49.278 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2517999999999999e-05 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354322+0000", 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4842e-05 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354330+0000", 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.422e-06 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.279 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.354389+0000", 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8996000000000002e-05 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:25.355418+0000", 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0010287619999999999 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.280 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e363/337 cost 1049576)", 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:42.250411+0000", 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 486.91822566799999, 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00091109999999999997, 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250411+0000", 2024-08-17T17:45:49.281 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250411+0000", 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.282 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250413+0000", 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.477e-06 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250429+0000", 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6166000000000001e-05 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250451+0000", 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2192e-05 2024-08-17T17:45:49.283 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250461+0000", 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4280000000000002e-06 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.250527+0000", 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6298999999999997e-05 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.284 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:42.251322+0000", 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00079553800000000004 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e364/337 cost 1049576)", 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:37:46.750065+0000", 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 482.41857195599999, 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014278940000000001, 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.285 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750065+0000", 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750065+0000", 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.286 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750066+0000", 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1629999999999999e-06 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750080+0000", 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.366e-05 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750095+0000", 2024-08-17T17:45:49.287 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.504e-05 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750102+0000", 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8179999999999998e-06 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.750164+0000", 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2589000000000007e-05 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.288 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:37:46.751493+0000", 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001328624 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e372/337 cost 1049576)", 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:10.396609+0000", 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 398.77202826299998, 2024-08-17T17:45:49.289 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00083456000000000003, 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396609+0000", 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396609+0000", 2024-08-17T17:45:49.290 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396611+0000", 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.857e-06 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396632+0000", 2024-08-17T17:45:49.291 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1756e-05 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396661+0000", 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8580000000000001e-05 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396674+0000", 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3249e-05 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.292 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.396801+0000", 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012715200000000001 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:10.397443+0000", 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000641966 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.293 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e376/337 cost 1049576)", 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.819130+0000", 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 354.34950746700002, 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00090844500000000002, 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819130+0000", 2024-08-17T17:45:49.294 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819130+0000", 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819131+0000", 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.37e-06 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.295 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819147+0000", 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6504e-05 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819168+0000", 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0712000000000001e-05 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819178+0000", 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8870000000000004e-06 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.296 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.819267+0000", 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9237e-05 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.820038+0000", 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00077073500000000004 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.297 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e377/337 cost 1049576)", 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:13.580528+0000", 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 335.58810871499998, 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00083345299999999995, 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580528+0000", 2024-08-17T17:45:49.298 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580528+0000", 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580536+0000", 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0800000000000006e-06 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.299 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580556+0000", 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9553000000000001e-05 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580581+0000", 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4936999999999999e-05 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580593+0000", 2024-08-17T17:45:49.300 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2405999999999999e-05 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.580731+0000", 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013768400000000001 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:13.581362+0000", 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00063079300000000004 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.301 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e378/337 cost 1049576)", 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:40:18.512744+0000", 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 330.65589279900001, 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013030660000000001, 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.302 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512744+0000", 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512744+0000", 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512746+0000", 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.373e-06 2024-08-17T17:45:49.303 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512761+0000", 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5372000000000001e-05 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512785+0000", 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3770000000000001e-05 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.304 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512794+0000", 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6139999999999998e-06 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.512908+0000", 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000113936 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:40:18.514047+0000", 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011390009999999999 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.305 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e385/337 cost 1049576)", 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:41:40.907585+0000", 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 248.26105208600001, 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00088815499999999996, 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.306 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907585+0000", 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907585+0000", 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907587+0000", 2024-08-17T17:45:49.307 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.61e-06 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907606+0000", 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8983000000000001e-05 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907630+0000", 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4816000000000001e-05 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.308 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907643+0000", 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2663000000000001e-05 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.907789+0000", 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000145842 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:41:40.908473+0000", 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00068424100000000004 2024-08-17T17:45:49.309 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e387/337 cost 1049576)", 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:00.500523+0000", 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 228.668114156, 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00083437799999999999, 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.310 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500523+0000", 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500523+0000", 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.311 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500525+0000", 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8199999999999999e-06 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500549+0000", 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4275999999999999e-05 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500578+0000", 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9067000000000001e-05 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.312 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500592+0000", 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4126e-05 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.500753+0000", 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016118799999999999 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:00.501357+0000", 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000603901 2024-08-17T17:45:49.313 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e391/337 cost 1049576)", 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:45.755500+0000", 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 183.413136909, 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002456522, 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.314 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755500+0000", 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755500+0000", 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755501+0000", 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1310000000000001e-06 2024-08-17T17:45:49.315 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755514+0000", 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2458e-05 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755529+0000", 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5741999999999999e-05 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.316 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755536+0000", 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.058e-06 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755592+0000", 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5347999999999999e-05 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.757957+0000", 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023647849999999999 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.317 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e393/337 cost 1049576)", 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:24.992776+0000", 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 144.17586120300001, 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00093179799999999996, 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.318 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.992776+0000", 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.992776+0000", 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.992778+0000", 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4260000000000002e-06 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.319 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.992806+0000", 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7702999999999999e-05 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.992840+0000", 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.417e-05 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.320 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.992859+0000", 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8926000000000001e-05 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.993013+0000", 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015426700000000001 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:24.993708+0000", 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000694306 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.321 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e393/337 cost 1049576)", 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:26.415423+0000", 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 142.75321351700001, 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00084360299999999995, 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.322 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415423+0000", 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415423+0000", 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.323 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415425+0000", 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3200000000000001e-06 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415451+0000", 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5826e-05 2024-08-17T17:45:49.324 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415464+0000", 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3678e-05 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415472+0000", 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9869999999999999e-06 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.415526+0000", 2024-08-17T17:45:49.325 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4107999999999998e-05 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:26.416267+0000", 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00074068400000000003 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e395/337 cost 1049576)", 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:43:40.205987+0000", 2024-08-17T17:45:49.326 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 128.962649621, 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00089414500000000005, 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.205987+0000", 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.327 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.205987+0000", 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.205989+0000", 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1710000000000001e-06 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.206000+0000", 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1372999999999999e-05 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.328 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.206014+0000", 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3927000000000001e-05 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.206020+0000", 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4230000000000002e-06 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.329 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.206074+0000", 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4166999999999998e-05 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:43:40.206882+0000", 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00080708400000000001 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e397/337 cost 1049576)", 2024-08-17T17:45:49.330 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:04.117282+0000", 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 105.051355336, 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002925394, 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117282+0000", 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.331 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117282+0000", 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117283+0000", 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1459999999999999e-06 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117295+0000", 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1948e-05 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.332 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117309+0000", 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4175000000000001e-05 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117325+0000", 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6030999999999999e-05 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117399+0000", 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3910000000000002e-05 2024-08-17T17:45:49.333 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.120207+0000", 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002808184 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e401/337 cost 1049576)", 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:48.288008+0000", 2024-08-17T17:45:49.334 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 60.880629427000002, 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00088211400000000003, 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288008+0000", 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.335 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288008+0000", 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288010+0000", 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4889999999999998e-06 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288035+0000", 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4623000000000001e-05 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.336 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288061+0000", 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.667e-05 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288074+0000", 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3091999999999999e-05 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288155+0000", 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0845000000000003e-05 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.337 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:48.288890+0000", 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00073439499999999999 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e402/337 cost 1049576)", 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:10.224223+0000", 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 38.944413595999997, 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030112820000000001, 2024-08-17T17:45:49.338 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224223+0000", 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224223+0000", 2024-08-17T17:45:49.339 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224224+0000", 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2699999999999998e-07 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224246+0000", 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1263999999999998e-05 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.340 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224258+0000", 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2599000000000001e-05 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224263+0000", 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0819999999999998e-06 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.341 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224317+0000", 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3857000000000003e-05 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.227235+0000", 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029176530000000001 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.342 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e404/337 cost 1049576)", 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.756543+0000", 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 25.412094362000001, 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00093355800000000002, 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756543+0000", 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.343 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756543+0000", 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756545+0000", 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9999999999999999e-06 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756567+0000", 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2393000000000001e-05 2024-08-17T17:45:49.344 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756596+0000", 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9272999999999999e-05 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756610+0000", 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3655999999999999e-05 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.345 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.756727+0000", 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000116902 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.757476+0000", 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00074933400000000005 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.346 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e404/337 cost 1049576)", 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:28.278020+0000", 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 20.890616790999999, 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001295944, 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278020+0000", 2024-08-17T17:45:49.347 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278020+0000", 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278021+0000", 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.201e-06 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.348 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278035+0000", 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3254999999999999e-05 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278051+0000", 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6344000000000001e-05 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278060+0000", 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6070000000000008e-06 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.349 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.278687+0000", 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00062702000000000005 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:28.279316+0000", 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00062951700000000001 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.350 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e404/337 cost 1049576)", 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:33.407392+0000", 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 15.761244649, 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0016614139999999999, 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.351 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407392+0000", 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407392+0000", 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407393+0000", 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.091e-06 2024-08-17T17:45:49.352 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407407+0000", 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3905e-05 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407423+0000", 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5889e-05 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.353 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407431+0000", 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9750000000000007e-06 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407494+0000", 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3178999999999999e-05 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.409054+0000", 2024-08-17T17:45:49.354 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001559375 2024-08-17T17:45:49.355 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.355 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.355 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.355 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:45:49.355 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:45:49.355 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:45:49.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:49.378 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:49.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:49.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:49.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:49.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:49.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:49.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:49 smithi152 ceph-mon[93456]: pgmap v1602: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.2 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 318 op/s; 46889/59784 objects degraded (78.431%); 8952/59784 objects misplaced (14.974%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:45:49.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:49 smithi142 ceph-mon[106729]: pgmap v1602: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.2 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 318 op/s; 46889/59784 objects degraded (78.431%); 8952/59784 objects misplaced (14.974%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:45:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:49 smithi142 ceph-mon[103614]: pgmap v1602: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.2 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 318 op/s; 46889/59784 objects degraded (78.431%); 8952/59784 objects misplaced (14.974%); 3.5 MiB/s, 56 objects/s recovering 2024-08-17T17:45:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:49.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:49.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:49.997 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:50.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:50.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:50.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:50.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:50.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:50.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:50.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:50.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:50.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:50.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:50.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:45:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:45:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:50.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:50.658 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:50.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:50.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:50.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:50.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:50.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:50.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:51.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:51.042 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:51.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:51.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:51.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:51.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:51.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:51.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:51.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:51.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:51.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:51.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:51.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:51 smithi142 ceph-mon[106729]: pgmap v1603: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 46792/58245 objects degraded (80.337%); 8758/58245 objects misplaced (15.036%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:45:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 46792/58245 objects degraded (80.337%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:51 smithi142 ceph-mon[103614]: pgmap v1603: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 46792/58245 objects degraded (80.337%); 8758/58245 objects misplaced (15.036%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:45:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 46792/58245 objects degraded (80.337%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:51.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:51.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:51.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:51.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:51.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:51 smithi152 ceph-mon[93456]: pgmap v1603: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 46792/58245 objects degraded (80.337%); 8758/58245 objects misplaced (15.036%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T17:45:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 46792/58245 objects degraded (80.337%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:51.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:51.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:51.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:52.021 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:45:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:45:51] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-08-17T17:45:52.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:52.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:52.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:52.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:52.188 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:52.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:52.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:52.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:52.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:52.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:52.502 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:45:52.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:52.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:52.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/739249737' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:52.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/739249737' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:52.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:52.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:52.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:52.786 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:52.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:52.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:45:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/739249737' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:45:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:52.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:52.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:52.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:52.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:52.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:45:53.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:53.102 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:53.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:53.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:53.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:53.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:53.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:53.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:53.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:53.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:53.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[103614]: pgmap v1604: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 46685/55851 objects degraded (83.588%); 8352/55851 objects misplaced (14.954%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:45:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/739249737' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:53.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[103614]: osdmap e406: 8 total, 8 up, 6 in 2024-08-17T17:45:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[106729]: pgmap v1604: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 46685/55851 objects degraded (83.588%); 8352/55851 objects misplaced (14.954%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:45:53.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/739249737' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:53.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:53 smithi142 ceph-mon[106729]: osdmap e406: 8 total, 8 up, 6 in 2024-08-17T17:45:53.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:53.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:53.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:53.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:53 smithi152 ceph-mon[93456]: pgmap v1604: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 46685/55851 objects degraded (83.588%); 8352/55851 objects misplaced (14.954%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:45:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:53 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:45:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/739249737' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:45:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:53 smithi152 ceph-mon[93456]: osdmap e406: 8 total, 8 up, 6 in 2024-08-17T17:45:53.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:53.877 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:53.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:53.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:53.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:45:54.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:45:54.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:54.155 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:54.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:54.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:54.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:54.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:54.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:54.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:54.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:54.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:54.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:54.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:54.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:54.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:54.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:54.925 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:54.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:55.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:55.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:55.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:55.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:55.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:55.205 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:55.255 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:45:55.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:55.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:55.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:55.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:55.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:55.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:55.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:55.674 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:55.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:55.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:55 smithi152 ceph-mon[93456]: pgmap v1606: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 326 op/s; 46571/54222 objects degraded (85.889%); 8125/54222 objects misplaced (14.985%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:45:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:45:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:45:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[103614]: pgmap v1606: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 326 op/s; 46571/54222 objects degraded (85.889%); 8125/54222 objects misplaced (14.985%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:45:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:45:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:45:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[106729]: pgmap v1606: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 326 op/s; 46571/54222 objects degraded (85.889%); 8125/54222 objects misplaced (14.985%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:45:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:45:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:45:55.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:45:55.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:55.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:55.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:56.053 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:56.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:56.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:56.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:56.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:56.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:56.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:56.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:56.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:56.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:56.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:56.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:56.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:56.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 46571/54222 objects degraded (85.889%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:56.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 46571/54222 objects degraded (85.889%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 46571/54222 objects degraded (85.889%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:45:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:56.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:56.970 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:56.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:57.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:57.096 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:57.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:57.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:57.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:45:57.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:57.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:57.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:57.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:57.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:57.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:57.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:57.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:57.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:57.794 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:57 smithi142 ceph-mon[106729]: pgmap v1607: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 46466/52152 objects degraded (89.097%); 7753/52152 objects misplaced (14.866%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:45:57.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:57 smithi142 ceph-mon[103614]: pgmap v1607: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 46466/52152 objects degraded (89.097%); 7753/52152 objects misplaced (14.866%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:45:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:57 smithi152 ceph-mon[93456]: pgmap v1607: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.1 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 46466/52152 objects degraded (89.097%); 7753/52152 objects misplaced (14.866%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:45:57.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:45:57.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:45:57.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:57.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:57.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:58.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:58.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:58.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:58.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:58.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:58.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:58.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:58.480 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:58.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:58.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:58.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:58.678 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:45:58.679 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:45:58.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:58.737 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:58.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:58.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:58.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:58.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:58.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:58.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:58.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:59.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:45:59.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:45:59.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:45:59.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:45:59.344 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:45:59.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:45:59.468 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:59.496 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:45:59.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:45:59.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:45:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:45:59.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:45:59.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:59.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:45:59.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:45:59 smithi152 ceph-mon[93456]: pgmap v1608: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.0 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 46466/49242 objects degraded (94.363%); 7621/49242 objects misplaced (15.477%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:45:59.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:45:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:45:59 smithi142 ceph-mon[106729]: pgmap v1608: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.0 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 46466/49242 objects degraded (94.363%); 7621/49242 objects misplaced (15.477%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:45:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:45:59 smithi142 ceph-mon[103614]: pgmap v1608: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 1.0 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 46466/49242 objects degraded (94.363%); 7621/49242 objects misplaced (15.477%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:45:59.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:45:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:45:59.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:00.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:00.041 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:00.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:00.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:00.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:00.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:00.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:00.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:00.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:00.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:00.513 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:46:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:46:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:00.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:00.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:00.579 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:00 smithi142 ceph-mon[106729]: from='client.38043 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:00.580 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:00 smithi142 ceph-mon[103614]: from='client.38043 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:00.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:00.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:00.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:00.806 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:00.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:00 smithi152 ceph-mon[93456]: from='client.38043 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:00.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:00.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:00.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:00.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:01.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:01.072 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:01.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:01.087 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:46:01.087 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:01.107 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:46:01.108 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:01.130 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:46:01.130 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:01.151 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:46:01.151 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:01.172 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:46:01.172 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:01.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:01.189 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:46:01.190 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:01.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:01.206 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:46:01.207 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:01.226 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:46:01.226 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:01.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:01.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:01.329 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:01.329 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:01.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:01.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:01.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:01.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:01.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:01.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-mon[103614]: pgmap v1609: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 989 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 356 op/s; 46361/47541 objects degraded (97.518%); 7389/47541 objects misplaced (15.542%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:46:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 46361/47541 objects degraded (97.518%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3136061949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-mon[106729]: pgmap v1609: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 989 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 356 op/s; 46361/47541 objects degraded (97.518%); 7389/47541 objects misplaced (15.542%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:46:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 46361/47541 objects degraded (97.518%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:01.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3136061949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:01.796 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:01.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:01 smithi152 ceph-mon[93456]: pgmap v1609: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 989 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 356 op/s; 46361/47541 objects degraded (97.518%); 7389/47541 objects misplaced (15.542%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:46:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 46361/47541 objects degraded (97.518%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3136061949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:01.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:01.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:01.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:01.959 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:01.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:02.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:02.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:02.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:46:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:46:01] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-08-17T17:46:02.192 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.215 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:02.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:02.316 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:02.348 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:02.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:02.351 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1608,"stamp":"2024-08-17T17:45:58.420944+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1074534373,"num_objects":16414,"num_object_clones":0,"num_object_copies":49242,"num_objects_missing_on_primary":1696,"num_objects_missing":1696,"num_objects_degraded":46466,"num_objects_misplaced":7621,"num_objects_unfound":0,"num_objects_dirty":16414,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":220280,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39211,"num_bytes_recovered":2446860942,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104928,"ondisk_log_size":104928,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8446204,"kb_used_data":2300100,"kb_used_omap":27,"kb_used_meta":6146020,"kb_avail":554024708,"statfs":{"total":575970213888,"available":567321300992,"internally_reserved":0,"allocated":2355302400,"data_stored":2341790391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6293525224},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,4,2,2,5,0,1,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-230293504,"num_objects":-3514,"num_object_clones":0,"num_object_copies":-10542,"num_objects_missing_on_primary":-218,"num_objects_missing":-218,"num_objects_degraded":-423,"num_objects_misplaced":-1331,"num_objects_unfound":0,"num_objects_dirty":-3514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3514,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":682,"num_bytes_recovered":44695552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002868"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":657,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509093+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:45:52.509093+0000","last_peered":"2024-08-17T17:45:52.509093+0000","last_clean":"2024-08-17T17:45:52.509093+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:45:52.509093+0000","last_undegraded":"2024-08-17T17:45:52.509093+0000","last_fullsized":"2024-08-17T17:45:52.509093+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":115,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680734+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:45:37.680734+0000","last_peered":"2024-08-17T17:45:37.680734+0000","last_clean":"2024-08-17T17:45:37.680734+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:45:37.680734+0000","last_undegraded":"2024-08-17T17:45:37.680734+0000","last_fullsized":"2024-08-17T17:45:37.680734+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.695240+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:45:37.695240+0000","last_peered":"2024-08-17T17:45:37.695240+0000","last_clean":"2024-08-17T17:45:37.695240+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:45:37.695240+0000","last_undegraded":"2024-08-17T17:45:37.695240+0000","last_fullsized":"2024-08-17T17:45:37.695240+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":355,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.510115+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:45:52.510115+0000","last_peered":"2024-08-17T17:45:52.510115+0000","last_clean":"2024-08-17T17:45:52.510115+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:45:52.510115+0000","last_undegraded":"2024-08-17T17:45:52.510115+0000","last_fullsized":"2024-08-17T17:45:52.510115+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":532,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509486+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:45:52.509486+0000","last_peered":"2024-08-17T17:45:52.509486+0000","last_clean":"2024-08-17T17:45:52.509486+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:45:52.509486+0000","last_undegraded":"2024-08-17T17:45:52.509486+0000","last_fullsized":"2024-08-17T17:45:52.509486+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":258,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680215+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:45:37.680215+0000","last_peered":"2024-08-17T17:45:37.680215+0000","last_clean":"2024-08-17T17:45:37.680215+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:45:37.680215+0000","last_undegraded":"2024-08-17T17:45:37.680215+0000","last_fullsized":"2024-08-17T17:45:37.680215+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":625,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508963+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:45:52.508963+0000","last_peered":"2024-08-17T17:45:52.508963+0000","last_clean":"2024-08-17T17:45:52.508963+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:45:52.508963+0000","last_undegraded":"2024-08-17T17:45:52.508963+0000","last_fullsized":"2024-08-17T17:45:52.508963+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":426,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.712880+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:45:37.712880+0000","last_peered":"2024-08-17T17:45:37.712880+0000","last_clean":"2024-08-17T17:45:37.712880+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:45:37.712880+0000","last_undegraded":"2024-08-17T17:45:37.712880+0000","last_fullsized":"2024-08-17T17:45:37.712880+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"406'13651","reported_seq":24949,"reported_epoch":406,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:45:55.725195+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:45:55.725195+0000","last_peered":"2024-08-17T17:45:55.725195+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:45:55.725195+0000","last_undegraded":"2024-08-17T17:45:55.725195+0000","last_fullsized":"2024-08-17T17:45:55.725195+0000","mapping_epoch":337,"log_start":"329'6763","ondisk_log_start":"329'6763","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6888,"log_dups_size":0,"ondisk_log_size":6888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69009408,"num_objects":1053,"num_object_clones":0,"num_object_copies":3159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1033,"num_objects_unfound":0,"num_objects_dirty":1053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13651,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5590,"num_bytes_recovered":366346240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":1033},{"shards":"4,6,7","objects":20}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":111,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.706928+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:45:37.706928+0000","last_peered":"2024-08-17T17:45:37.706928+0000","last_clean":"2024-08-17T17:45:37.706928+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:45:37.706928+0000","last_undegraded":"2024-08-17T17:45:37.706928+0000","last_fullsized":"2024-08-17T17:45:37.706928+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":292,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.694625+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:45:37.694625+0000","last_peered":"2024-08-17T17:45:37.694625+0000","last_clean":"2024-08-17T17:45:37.694625+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:45:37.694625+0000","last_undegraded":"2024-08-17T17:45:37.694625+0000","last_fullsized":"2024-08-17T17:45:37.694625+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"406'14079","reported_seq":14183,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:57.192866+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:45:57.192866+0000","last_peered":"2024-08-17T17:45:57.192866+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:45:57.192866+0000","last_undegraded":"2024-08-17T17:45:57.192866+0000","last_fullsized":"2024-08-17T17:45:57.192866+0000","mapping_epoch":337,"log_start":"330'7482","ondisk_log_start":"330'7482","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":995,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14079,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":995}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":113,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.681088+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:45:37.681088+0000","last_peered":"2024-08-17T17:45:37.681088+0000","last_clean":"2024-08-17T17:45:37.681088+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:45:37.681088+0000","last_undegraded":"2024-08-17T17:45:37.681088+0000","last_fullsized":"2024-08-17T17:45:37.681088+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":377,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680150+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:45:37.680150+0000","last_peered":"2024-08-17T17:45:37.680150+0000","last_clean":"2024-08-17T17:45:37.680150+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:45:37.680150+0000","last_undegraded":"2024-08-17T17:45:37.680150+0000","last_fullsized":"2024-08-17T17:45:37.680150+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"406'13712","reported_seq":13819,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:55.643337+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:45:55.643337+0000","last_peered":"2024-08-17T17:45:55.643337+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:45:55.643337+0000","last_undegraded":"2024-08-17T17:45:55.643337+0000","last_fullsized":"2024-08-17T17:45:55.643337+0000","mapping_epoch":337,"log_start":"329'6853","ondisk_log_start":"329'6853","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1022,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13712,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1022}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":109,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.695180+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:45:37.695180+0000","last_peered":"2024-08-17T17:45:37.695180+0000","last_clean":"2024-08-17T17:45:37.695180+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:45:37.695180+0000","last_undegraded":"2024-08-17T17:45:37.695180+0000","last_fullsized":"2024-08-17T17:45:37.695180+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":387,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.712818+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:45:37.712818+0000","last_peered":"2024-08-17T17:45:37.712818+0000","last_clean":"2024-08-17T17:45:37.712818+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:45:37.712818+0000","last_undegraded":"2024-08-17T17:45:37.712818+0000","last_fullsized":"2024-08-17T17:45:37.712818+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"406'13877","reported_seq":14071,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:54.802108+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:45:54.802108+0000","last_peered":"2024-08-17T17:45:54.802108+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:45:54.802108+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"329'6813","ondisk_log_start":"329'6813","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13877,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-5688},{"shards":"0,3,7","objects":1081}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":569,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509429+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:45:52.509429+0000","last_peered":"2024-08-17T17:45:52.509429+0000","last_clean":"2024-08-17T17:45:52.509429+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:45:52.509429+0000","last_undegraded":"2024-08-17T17:45:52.509429+0000","last_fullsized":"2024-08-17T17:45:52.509429+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":335,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.513566+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:45:52.513566+0000","last_peered":"2024-08-17T17:45:52.513566+0000","last_clean":"2024-08-17T17:45:52.513566+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:45:52.513566+0000","last_undegraded":"2024-08-17T17:45:52.513566+0000","last_fullsized":"2024-08-17T17:45:52.513566+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"406'13561","reported_seq":13656,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:57.189752+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:45:57.189752+0000","last_peered":"2024-08-17T17:45:57.189752+0000","last_clean":"2024-08-17T17:45:57.189752+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:45:57.189752+0000","last_undegraded":"2024-08-17T17:45:57.189752+0000","last_fullsized":"2024-08-17T17:45:57.189752+0000","mapping_epoch":319,"log_start":"330'6974","ondisk_log_start":"330'6974","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67698688,"num_objects":1033,"num_object_clones":0,"num_object_copies":3099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13561,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":658,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511568+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:45:52.511568+0000","last_peered":"2024-08-17T17:45:52.511568+0000","last_clean":"2024-08-17T17:45:52.511568+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:45:52.511568+0000","last_undegraded":"2024-08-17T17:45:52.511568+0000","last_fullsized":"2024-08-17T17:45:52.511568+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680684+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:45:37.680684+0000","last_peered":"2024-08-17T17:45:37.680684+0000","last_clean":"2024-08-17T17:45:37.680684+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:45:37.680684+0000","last_undegraded":"2024-08-17T17:45:37.680684+0000","last_fullsized":"2024-08-17T17:45:37.680684+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"406'13872","reported_seq":13977,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:57.190130+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:45:57.190130+0000","last_peered":"2024-08-17T17:45:57.190130+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:45:57.190130+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"330'7335","ondisk_log_start":"330'7335","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69468160,"num_objects":1060,"num_object_clones":0,"num_object_copies":3180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13872,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":1060},{"shards":"1,6","objects":-6306}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":361,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.712946+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:45:37.712946+0000","last_peered":"2024-08-17T17:45:37.712946+0000","last_clean":"2024-08-17T17:45:37.712946+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:45:37.712946+0000","last_undegraded":"2024-08-17T17:45:37.712946+0000","last_fullsized":"2024-08-17T17:45:37.712946+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":387,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.681768+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:45:37.681768+0000","last_peered":"2024-08-17T17:45:37.681768+0000","last_clean":"2024-08-17T17:45:37.681768+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:45:37.681768+0000","last_undegraded":"2024-08-17T17:45:37.681768+0000","last_fullsized":"2024-08-17T17:45:37.681768+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"406'13925","reported_seq":14120,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:45:54.063661+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:45:54.063661+0000","last_peered":"2024-08-17T17:45:54.063661+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:45:54.063661+0000","last_undegraded":"2024-08-17T17:45:54.063661+0000","last_fullsized":"2024-08-17T17:45:54.063661+0000","mapping_epoch":337,"log_start":"348'8361","ondisk_log_start":"348'8361","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5564,"log_dups_size":0,"ondisk_log_size":5564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71106560,"num_objects":1085,"num_object_clones":0,"num_object_copies":3255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1085,"num_objects_unfound":0,"num_objects_dirty":1085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13925,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":1085}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":398,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680362+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:45:37.680362+0000","last_peered":"2024-08-17T17:45:37.680362+0000","last_clean":"2024-08-17T17:45:37.680362+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:45:37.680362+0000","last_undegraded":"2024-08-17T17:45:37.680362+0000","last_fullsized":"2024-08-17T17:45:37.680362+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.681158+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:45:37.681158+0000","last_peered":"2024-08-17T17:45:37.681158+0000","last_clean":"2024-08-17T17:45:37.681158+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:45:37.681158+0000","last_undegraded":"2024-08-17T17:45:37.681158+0000","last_fullsized":"2024-08-17T17:45:37.681158+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"406'13700","reported_seq":25111,"reported_epoch":406,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:55.784353+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:45:55.784353+0000","last_peered":"2024-08-17T17:45:55.784353+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:45:55.784353+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:45:55.784353+0000","mapping_epoch":337,"log_start":"329'6841","ondisk_log_start":"329'6841","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":1696,"num_objects_missing":1696,"num_objects_degraded":1004,"num_objects_misplaced":1696,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13700,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5643,"num_bytes_recovered":369754112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":13},{"shards":"1,2,7","objects":990},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":628,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511867+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:45:52.511867+0000","last_peered":"2024-08-17T17:45:52.511867+0000","last_clean":"2024-08-17T17:45:52.511867+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:45:52.511867+0000","last_undegraded":"2024-08-17T17:45:52.511867+0000","last_fullsized":"2024-08-17T17:45:52.511867+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":565,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509094+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:45:52.509094+0000","last_peered":"2024-08-17T17:45:52.509094+0000","last_clean":"2024-08-17T17:45:52.509094+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:45:52.509094+0000","last_undegraded":"2024-08-17T17:45:52.509094+0000","last_fullsized":"2024-08-17T17:45:52.509094+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"406'13856","reported_seq":25356,"reported_epoch":406,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:45:54.036306+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:45:54.036306+0000","last_peered":"2024-08-17T17:45:54.036306+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:45:54.036306+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:45:54.036306+0000","mapping_epoch":337,"log_start":"342'8244","ondisk_log_start":"342'8244","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091928,"num_objects":1040,"num_object_clones":0,"num_object_copies":3120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1040,"num_objects_misplaced":1790,"num_objects_unfound":0,"num_objects_dirty":1040,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13856,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5658,"num_bytes_recovered":370737152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1040}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":759,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:52.511110+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:45:52.511110+0000","last_peered":"2024-08-17T17:45:52.511110+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:45:52.511110+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":335,"reported_epoch":405,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:45:37.694939+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:45:37.694939+0000","last_peered":"2024-08-17T17:45:37.694939+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:45:37.694939+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:45:37.694939+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":608,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.510072+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:45:52.510072+0000","last_peered":"2024-08-17T17:45:52.510072+0000","last_clean":"2024-08-17T17:45:52.510072+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:45:52.510072+0000","last_undegraded":"2024-08-17T17:45:52.510072+0000","last_fullsized":"2024-08-17T17:45:52.510072+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":370,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.713807+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:45:37.713807+0000","last_peered":"2024-08-17T17:45:37.713807+0000","last_clean":"2024-08-17T17:45:37.713807+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:45:37.713807+0000","last_undegraded":"2024-08-17T17:45:37.713807+0000","last_fullsized":"2024-08-17T17:45:37.713807+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"406'13672","reported_seq":13869,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:54.974186+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:45:54.974186+0000","last_peered":"2024-08-17T17:45:54.974186+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:45:54.974186+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"329'6621","ondisk_log_start":"329'6621","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70254592,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13672,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-5575},{"shards":"0,4,7","objects":1072}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":424,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507194+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:45:52.507194+0000","last_peered":"2024-08-17T17:45:52.507194+0000","last_clean":"2024-08-17T17:45:52.507194+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:45:52.507194+0000","last_undegraded":"2024-08-17T17:45:52.507194+0000","last_fullsized":"2024-08-17T17:45:52.507194+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":342,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680029+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:45:37.680029+0000","last_peered":"2024-08-17T17:45:37.680029+0000","last_clean":"2024-08-17T17:45:37.680029+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:45:37.680029+0000","last_undegraded":"2024-08-17T17:45:37.680029+0000","last_fullsized":"2024-08-17T17:45:37.680029+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":349,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.512025+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:45:52.512025+0000","last_peered":"2024-08-17T17:45:52.512025+0000","last_clean":"2024-08-17T17:45:52.512025+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:45:52.512025+0000","last_undegraded":"2024-08-17T17:45:52.512025+0000","last_fullsized":"2024-08-17T17:45:52.512025+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"406'13824","reported_seq":13929,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:57.179655+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:45:57.179655+0000","last_peered":"2024-08-17T17:45:57.179655+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:45:57.179655+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"330'7217","ondisk_log_start":"330'7217","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13824,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":972},{"shards":"5,6","objects":-6324}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":516,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509925+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:45:52.509925+0000","last_peered":"2024-08-17T17:45:52.509925+0000","last_clean":"2024-08-17T17:45:52.509925+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:45:52.509925+0000","last_undegraded":"2024-08-17T17:45:52.509925+0000","last_fullsized":"2024-08-17T17:45:52.509925+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":142,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.712658+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:45:37.712658+0000","last_peered":"2024-08-17T17:45:37.712658+0000","last_clean":"2024-08-17T17:45:37.712658+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:45:37.712658+0000","last_undegraded":"2024-08-17T17:45:37.712658+0000","last_fullsized":"2024-08-17T17:45:37.712658+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":700,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507896+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:45:52.507896+0000","last_peered":"2024-08-17T17:45:52.507896+0000","last_clean":"2024-08-17T17:45:52.507896+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:45:52.507896+0000","last_undegraded":"2024-08-17T17:45:52.507896+0000","last_fullsized":"2024-08-17T17:45:52.507896+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"406'13600","reported_seq":14521,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:54.933107+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:45:54.933107+0000","last_peered":"2024-08-17T17:45:54.933107+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:45:54.933107+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"329'6616","ondisk_log_start":"329'6616","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68550656,"num_objects":1046,"num_object_clones":0,"num_object_copies":3138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13600,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":1046},{"shards":"0,5","objects":-5608}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":492,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507390+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:45:52.507390+0000","last_peered":"2024-08-17T17:45:52.507390+0000","last_clean":"2024-08-17T17:45:52.507390+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:45:52.507390+0000","last_undegraded":"2024-08-17T17:45:52.507390+0000","last_fullsized":"2024-08-17T17:45:52.507390+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":381,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680937+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:45:37.680937+0000","last_peered":"2024-08-17T17:45:37.680937+0000","last_clean":"2024-08-17T17:45:37.680937+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:45:37.680937+0000","last_undegraded":"2024-08-17T17:45:37.680937+0000","last_fullsized":"2024-08-17T17:45:37.680937+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":287,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509053+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:45:52.509053+0000","last_peered":"2024-08-17T17:45:52.509053+0000","last_clean":"2024-08-17T17:45:52.509053+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:45:52.509053+0000","last_undegraded":"2024-08-17T17:45:52.509053+0000","last_fullsized":"2024-08-17T17:45:52.509053+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"406'13848","reported_seq":28888,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:57.834765+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:45:57.834765+0000","last_peered":"2024-08-17T17:45:57.834765+0000","last_clean":"2024-08-17T17:45:57.834765+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:45:57.834765+0000","last_undegraded":"2024-08-17T17:45:57.834765+0000","last_fullsized":"2024-08-17T17:45:57.834765+0000","mapping_epoch":355,"log_start":"330'7530","ondisk_log_start":"330'7530","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6318,"log_dups_size":0,"ondisk_log_size":6318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64749568,"num_objects":988,"num_object_clones":0,"num_object_copies":2964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13848,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.694673+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:45:37.694673+0000","last_peered":"2024-08-17T17:45:37.694673+0000","last_clean":"2024-08-17T17:45:37.694673+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:45:37.694673+0000","last_undegraded":"2024-08-17T17:45:37.694673+0000","last_fullsized":"2024-08-17T17:45:37.694673+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":155,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508933+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:45:52.508933+0000","last_peered":"2024-08-17T17:45:52.508933+0000","last_clean":"2024-08-17T17:45:52.508933+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:45:52.508933+0000","last_undegraded":"2024-08-17T17:45:52.508933+0000","last_fullsized":"2024-08-17T17:45:52.508933+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":651,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508172+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:45:52.508172+0000","last_peered":"2024-08-17T17:45:52.508172+0000","last_clean":"2024-08-17T17:45:52.508172+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:45:52.508172+0000","last_undegraded":"2024-08-17T17:45:52.508172+0000","last_fullsized":"2024-08-17T17:45:52.508172+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"406'13497","reported_seq":28105,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:57.873239+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:45:57.873239+0000","last_peered":"2024-08-17T17:45:57.873239+0000","last_clean":"2024-08-17T17:45:57.873239+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:45:57.873239+0000","last_undegraded":"2024-08-17T17:45:57.873239+0000","last_fullsized":"2024-08-17T17:45:57.873239+0000","mapping_epoch":345,"log_start":"330'7172","ondisk_log_start":"330'7172","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6325,"log_dups_size":0,"ondisk_log_size":6325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59441152,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13497,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":316,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.695005+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:45:37.695005+0000","last_peered":"2024-08-17T17:45:37.695005+0000","last_clean":"2024-08-17T17:45:37.695005+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:45:37.695005+0000","last_undegraded":"2024-08-17T17:45:37.695005+0000","last_fullsized":"2024-08-17T17:45:37.695005+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":406,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511192+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:45:52.511192+0000","last_peered":"2024-08-17T17:45:52.511192+0000","last_clean":"2024-08-17T17:45:52.511192+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:45:52.511192+0000","last_undegraded":"2024-08-17T17:45:52.511192+0000","last_fullsized":"2024-08-17T17:45:52.511192+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":458,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.694827+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:45:37.694827+0000","last_peered":"2024-08-17T17:45:37.694827+0000","last_clean":"2024-08-17T17:45:37.694827+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:45:37.694827+0000","last_undegraded":"2024-08-17T17:45:37.694827+0000","last_fullsized":"2024-08-17T17:45:37.694827+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"406'13760","reported_seq":13976,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:45:57.906648+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:45:57.906648+0000","last_peered":"2024-08-17T17:45:57.906648+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:45:57.906648+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"330'7487","ondisk_log_start":"330'7487","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67108864,"num_objects":1024,"num_object_clones":0,"num_object_copies":3072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13760,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1024},{"shards":"0,3","objects":-6224}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":681,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509175+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:45:52.509175+0000","last_peered":"2024-08-17T17:45:52.509175+0000","last_clean":"2024-08-17T17:45:52.509175+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:45:52.509175+0000","last_undegraded":"2024-08-17T17:45:52.509175+0000","last_fullsized":"2024-08-17T17:45:52.509175+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":649,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508052+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:45:52.508052+0000","last_peered":"2024-08-17T17:45:52.508052+0000","last_clean":"2024-08-17T17:45:52.508052+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:45:52.508052+0000","last_undegraded":"2024-08-17T17:45:52.508052+0000","last_fullsized":"2024-08-17T17:45:52.508052+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":667,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511686+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:45:52.511686+0000","last_peered":"2024-08-17T17:45:52.511686+0000","last_clean":"2024-08-17T17:45:52.511686+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:45:52.511686+0000","last_undegraded":"2024-08-17T17:45:52.511686+0000","last_fullsized":"2024-08-17T17:45:52.511686+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"406'13568","reported_seq":28271,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:57.217710+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:45:57.217710+0000","last_peered":"2024-08-17T17:45:57.217710+0000","last_clean":"2024-08-17T17:45:57.217710+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:45:57.217710+0000","last_undegraded":"2024-08-17T17:45:57.217710+0000","last_fullsized":"2024-08-17T17:45:57.217710+0000","mapping_epoch":347,"log_start":"330'6972","ondisk_log_start":"330'6972","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6596,"log_dups_size":0,"ondisk_log_size":6596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65929216,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13568,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":339,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511942+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:45:52.511942+0000","last_peered":"2024-08-17T17:45:52.511942+0000","last_clean":"2024-08-17T17:45:52.511942+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:45:52.511942+0000","last_undegraded":"2024-08-17T17:45:52.511942+0000","last_fullsized":"2024-08-17T17:45:52.511942+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":410,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.713548+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:45:37.713548+0000","last_peered":"2024-08-17T17:45:37.713548+0000","last_clean":"2024-08-17T17:45:37.713548+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:45:37.713548+0000","last_undegraded":"2024-08-17T17:45:37.713548+0000","last_fullsized":"2024-08-17T17:45:37.713548+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680274+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:45:37.680274+0000","last_peered":"2024-08-17T17:45:37.680274+0000","last_clean":"2024-08-17T17:45:37.680274+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:45:37.680274+0000","last_undegraded":"2024-08-17T17:45:37.680274+0000","last_fullsized":"2024-08-17T17:45:37.680274+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680842+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:45:37.680842+0000","last_peered":"2024-08-17T17:45:37.680842+0000","last_clean":"2024-08-17T17:45:37.680842+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:45:37.680842+0000","last_undegraded":"2024-08-17T17:45:37.680842+0000","last_fullsized":"2024-08-17T17:45:37.680842+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":403,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680980+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:45:37.680980+0000","last_peered":"2024-08-17T17:45:37.680980+0000","last_clean":"2024-08-17T17:45:37.680980+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:45:37.680980+0000","last_undegraded":"2024-08-17T17:45:37.680980+0000","last_fullsized":"2024-08-17T17:45:37.680980+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":656,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507489+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:45:52.507489+0000","last_peered":"2024-08-17T17:45:52.507489+0000","last_clean":"2024-08-17T17:45:52.507489+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:45:52.507489+0000","last_undegraded":"2024-08-17T17:45:52.507489+0000","last_fullsized":"2024-08-17T17:45:52.507489+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":196,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509320+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:45:52.509320+0000","last_peered":"2024-08-17T17:45:52.509320+0000","last_clean":"2024-08-17T17:45:52.509320+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:45:52.509320+0000","last_undegraded":"2024-08-17T17:45:52.509320+0000","last_fullsized":"2024-08-17T17:45:52.509320+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":310,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.695241+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:45:37.695241+0000","last_peered":"2024-08-17T17:45:37.695241+0000","last_clean":"2024-08-17T17:45:37.695241+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:45:37.695241+0000","last_undegraded":"2024-08-17T17:45:37.695241+0000","last_fullsized":"2024-08-17T17:45:37.695241+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":614,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508674+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:45:52.508674+0000","last_peered":"2024-08-17T17:45:52.508674+0000","last_clean":"2024-08-17T17:45:52.508674+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:45:52.508674+0000","last_undegraded":"2024-08-17T17:45:52.508674+0000","last_fullsized":"2024-08-17T17:45:52.508674+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":454,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.696274+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:45:37.696274+0000","last_peered":"2024-08-17T17:45:37.696274+0000","last_clean":"2024-08-17T17:45:37.696274+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:45:37.696274+0000","last_undegraded":"2024-08-17T17:45:37.696274+0000","last_fullsized":"2024-08-17T17:45:37.696274+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":608,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508599+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:45:52.508599+0000","last_peered":"2024-08-17T17:45:52.508599+0000","last_clean":"2024-08-17T17:45:52.508599+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:45:52.508599+0000","last_undegraded":"2024-08-17T17:45:52.508599+0000","last_fullsized":"2024-08-17T17:45:52.508599+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.713454+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:45:37.713454+0000","last_peered":"2024-08-17T17:45:37.713454+0000","last_clean":"2024-08-17T17:45:37.713454+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:45:37.713454+0000","last_undegraded":"2024-08-17T17:45:37.713454+0000","last_fullsized":"2024-08-17T17:45:37.713454+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":639,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511763+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:45:52.511763+0000","last_peered":"2024-08-17T17:45:52.511763+0000","last_clean":"2024-08-17T17:45:52.511763+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:45:52.511763+0000","last_undegraded":"2024-08-17T17:45:52.511763+0000","last_fullsized":"2024-08-17T17:45:52.511763+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":356,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509355+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:45:52.509355+0000","last_peered":"2024-08-17T17:45:52.509355+0000","last_clean":"2024-08-17T17:45:52.509355+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:45:52.509355+0000","last_undegraded":"2024-08-17T17:45:52.509355+0000","last_fullsized":"2024-08-17T17:45:52.509355+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":647,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507771+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:45:52.507771+0000","last_peered":"2024-08-17T17:45:52.507771+0000","last_clean":"2024-08-17T17:45:52.507771+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:45:52.507771+0000","last_undegraded":"2024-08-17T17:45:52.507771+0000","last_fullsized":"2024-08-17T17:45:52.507771+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":607,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509087+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:45:52.509087+0000","last_peered":"2024-08-17T17:45:52.509087+0000","last_clean":"2024-08-17T17:45:52.509087+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:45:52.509087+0000","last_undegraded":"2024-08-17T17:45:52.509087+0000","last_fullsized":"2024-08-17T17:45:52.509087+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":389,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680577+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:45:37.680577+0000","last_peered":"2024-08-17T17:45:37.680577+0000","last_clean":"2024-08-17T17:45:37.680577+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:45:37.680577+0000","last_undegraded":"2024-08-17T17:45:37.680577+0000","last_fullsized":"2024-08-17T17:45:37.680577+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":296,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509194+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:45:52.509194+0000","last_peered":"2024-08-17T17:45:52.509194+0000","last_clean":"2024-08-17T17:45:52.509194+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:45:52.509194+0000","last_undegraded":"2024-08-17T17:45:52.509194+0000","last_fullsized":"2024-08-17T17:45:52.509194+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":352,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511896+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:45:52.511896+0000","last_peered":"2024-08-17T17:45:52.511896+0000","last_clean":"2024-08-17T17:45:52.511896+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:45:52.511896+0000","last_undegraded":"2024-08-17T17:45:52.511896+0000","last_fullsized":"2024-08-17T17:45:52.511896+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":397,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680804+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:45:37.680804+0000","last_peered":"2024-08-17T17:45:37.680804+0000","last_clean":"2024-08-17T17:45:37.680804+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:45:37.680804+0000","last_undegraded":"2024-08-17T17:45:37.680804+0000","last_fullsized":"2024-08-17T17:45:37.680804+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":260,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680761+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:45:37.680761+0000","last_peered":"2024-08-17T17:45:37.680761+0000","last_clean":"2024-08-17T17:45:37.680761+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:45:37.680761+0000","last_undegraded":"2024-08-17T17:45:37.680761+0000","last_fullsized":"2024-08-17T17:45:37.680761+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":397,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680404+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:45:37.680404+0000","last_peered":"2024-08-17T17:45:37.680404+0000","last_clean":"2024-08-17T17:45:37.680404+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:45:37.680404+0000","last_undegraded":"2024-08-17T17:45:37.680404+0000","last_fullsized":"2024-08-17T17:45:37.680404+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":260,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.681682+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:45:37.681682+0000","last_peered":"2024-08-17T17:45:37.681682+0000","last_clean":"2024-08-17T17:45:37.681682+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:45:37.681682+0000","last_undegraded":"2024-08-17T17:45:37.681682+0000","last_fullsized":"2024-08-17T17:45:37.681682+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":659,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509953+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:45:52.509953+0000","last_peered":"2024-08-17T17:45:52.509953+0000","last_clean":"2024-08-17T17:45:52.509953+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:45:52.509953+0000","last_undegraded":"2024-08-17T17:45:52.509953+0000","last_fullsized":"2024-08-17T17:45:52.509953+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":633,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.509561+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:45:52.509561+0000","last_peered":"2024-08-17T17:45:52.509561+0000","last_clean":"2024-08-17T17:45:52.509561+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:45:52.509561+0000","last_undegraded":"2024-08-17T17:45:52.509561+0000","last_fullsized":"2024-08-17T17:45:52.509561+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2024-08-17T17:45:37.680557+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:45:37.680557+0000","last_peered":"2024-08-17T17:45:37.680557+0000","last_clean":"2024-08-17T17:45:37.680557+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:45:37.680557+0000","last_undegraded":"2024-08-17T17:45:37.680557+0000","last_fullsized":"2024-08-17T17:45:37.680557+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":355,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.511499+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:45:52.511499+0000","last_peered":"2024-08-17T17:45:52.511499+0000","last_clean":"2024-08-17T17:45:52.511499+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:45:52.511499+0000","last_undegraded":"2024-08-17T17:45:52.511499+0000","last_fullsized":"2024-08-17T17:45:52.511499+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1073938456,"num_objects":16388,"num_object_clones":0,"num_object_copies":49164,"num_objects_missing_on_primary":1696,"num_objects_missing":1696,"num_objects_degraded":46463,"num_objects_misplaced":7621,"num_objects_unfound":0,"num_objects_dirty":16388,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":220002,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39170,"num_bytes_recovered":2444492800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6848851968,"data_stored":6848839752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104721,"ondisk_log_size":104721,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170154,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463524,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1641016,"kb_used_data":458928,"kb_used_omap":1,"kb_used_meta":1182078,"kb_avail":92104136,"statfs":{"total":95995035648,"available":94314635264,"internally_reserved":0,"allocated":469942272,"data_stored":467678075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1210448216},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492198,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525532,"kb_used_data":138516,"kb_used_omap":7,"kb_used_meta":387000,"kb_avail":93219620,"statfs":{"total":95995035648,"available":95456890880,"internally_reserved":0,"allocated":141840384,"data_stored":139598487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":396288333},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986789,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1771500,"kb_used_data":271140,"kb_used_omap":1,"kb_used_meta":1500350,"kb_avail":91973652,"statfs":{"total":95995035648,"available":94181019648,"internally_reserved":0,"allocated":277647360,"data_stored":275394103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1536358744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1,0,2,0,1,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225618,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507292,"kb_used_data":519380,"kb_used_omap":4,"kb_used_meta":987899,"kb_avail":92237860,"statfs":{"total":95995035648,"available":94451568640,"internally_reserved":0,"allocated":531845120,"data_stored":529603521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1011609024},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075942,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533835,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1567332,"kb_used_data":497560,"kb_used_omap":5,"kb_used_meta":1069754,"kb_avail":92177820,"statfs":{"total":95995035648,"available":94390087680,"internally_reserved":0,"allocated":509501440,"data_stored":507254519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1095428336},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089812,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433532,"kb_used_data":414576,"kb_used_omap":5,"kb_used_meta":1018938,"kb_avail":92311620,"statfs":{"total":95995035648,"available":94527098880,"internally_reserved":0,"allocated":424525824,"data_stored":422261686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1043392571},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":417923072,"data_stored":417923072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":495718400,"data_stored":495714328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":518520832,"data_stored":518520832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":268500992,"data_stored":268500992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":135266304,"data_stored":135266304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":459079680,"data_stored":459079680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1214582784,"data_stored":1214578712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:46:02.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:46:02.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:02.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:02.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:02.609 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:02.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:02.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:02.816 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:02 smithi142 ceph-mon[103614]: pgmap v1610: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 937 MiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 360 op/s; 46235/45021 objects degraded (102.697%); 6981/45021 objects misplaced (15.506%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:46:02.817 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:02 smithi142 ceph-mon[106729]: pgmap v1610: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 937 MiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 360 op/s; 46235/45021 objects degraded (102.697%); 6981/45021 objects misplaced (15.506%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:46:02.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:46:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:02 smithi152 ceph-mon[93456]: pgmap v1610: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 937 MiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 360 op/s; 46235/45021 objects degraded (102.697%); 6981/45021 objects misplaced (15.506%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:46:02.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:02.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:02.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:02.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:02.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:03.028 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:03.130 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:03.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:03.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:03.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:03.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:03.417 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:03.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:03.544 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:03.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:03.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:03.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:03.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:03.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:03.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:03.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:03.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:04.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:04.057 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:04.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:04.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:04.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:04.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:04.247 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:04.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:04.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:04.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:04.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:04.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:04.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:04.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:04.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:04.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:04.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:04.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:04.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:04.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:05.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:05.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:05.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:05.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:05.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:05.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:05.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:05.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:05.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:46:05.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:05.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:05.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:05.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:05 smithi142 ceph-mon[106729]: pgmap v1611: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 900 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 46103/43266 objects degraded (106.557%); 6722/43266 objects misplaced (15.536%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:46:05.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:05 smithi142 ceph-mon[103614]: pgmap v1611: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 900 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 46103/43266 objects degraded (106.557%); 6722/43266 objects misplaced (15.536%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:46:05.638 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:05.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:05.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:05 smithi152 ceph-mon[93456]: pgmap v1611: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 900 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 46103/43266 objects degraded (106.557%); 6722/43266 objects misplaced (15.536%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:46:05.869 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:05.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:05.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:05.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:06.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:06.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:06.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:06.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:06.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:06.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:06.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:06.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:06.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:06.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:06.486 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:46:06.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:06.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:06.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:06.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:06.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:06.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:06.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:06.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:06.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 46103/43266 objects degraded (106.557%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:06 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3244493613' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:06 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 46103/43266 objects degraded (106.557%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:06 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3244493613' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:06 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 46103/43266 objects degraded (106.557%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:06 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3244493613' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:06 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:46:06.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:06.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:06.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:07.022 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:07.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:07.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:07.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:07.179 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:07.281 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:07.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:07.410 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:07.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:07.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:07.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:07.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:07.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:07.578 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[106729]: pgmap v1612: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 851 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 45984/40881 objects degraded (112.483%); 6258/40881 objects misplaced (15.308%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T17:46:07.578 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:07.579 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:46:07.579 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[106729]: osdmap e407: 8 total, 8 up, 6 in 2024-08-17T17:46:07.579 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[103614]: pgmap v1612: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 851 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 45984/40881 objects degraded (112.483%); 6258/40881 objects misplaced (15.308%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T17:46:07.579 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:07.579 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:46:07.579 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:07 smithi142 ceph-mon[103614]: osdmap e407: 8 total, 8 up, 6 in 2024-08-17T17:46:07.599 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:07.599 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:07.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:07.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:07.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:07 smithi152 ceph-mon[93456]: pgmap v1612: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 851 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 45984/40881 objects degraded (112.483%); 6258/40881 objects misplaced (15.308%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T17:46:07.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:07 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:07 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:46:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:07 smithi152 ceph-mon[93456]: osdmap e407: 8 total, 8 up, 6 in 2024-08-17T17:46:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:07.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:07.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:07.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:07.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:08.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:08.076 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:08.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:08.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.229 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:08.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:08.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:08.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:08.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:08.488 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:08.590 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:08.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:08.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:46:08.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:08.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:08.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:08.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:08.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:08.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:08.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:09.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:09.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:09.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:09.233 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:09.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:09.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:09.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:09.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:09.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:09.587 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:09 smithi142 ceph-mon[103614]: pgmap v1614: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 780 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 45984/37470 objects degraded (122.722%); 6109/37470 objects misplaced (16.304%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:46:09.587 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:09 smithi142 ceph-mon[106729]: pgmap v1614: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 780 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 45984/37470 objects degraded (122.722%); 6109/37470 objects misplaced (16.304%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:46:09.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:09.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:09.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:09.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:09.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:09 smithi152 ceph-mon[93456]: pgmap v1614: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 780 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 45984/37470 objects degraded (122.722%); 6109/37470 objects misplaced (16.304%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:46:09.824 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:09.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:09.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:09.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:10.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:10.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:10.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:10.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:10.274 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:10.301 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:10.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:10.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:10.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:10.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:46:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:46:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:10.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:10.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:10.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:10.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:10.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:10.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:10.950 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:10.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:11.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:11.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:11.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:11.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:11.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:11.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:11.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:11.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:11.339 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:11.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:46:11.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:11.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:11.576 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:11.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:11.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-mon[103614]: pgmap v1615: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 736 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 45855/35382 objects degraded (129.600%); 5823/35382 objects misplaced (16.458%); 4.9 MiB/s, 79 objects/s recovering 2024-08-17T17:46:11.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 45855/35382 objects degraded (129.600%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:11.633 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:11.635 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-mon[106729]: pgmap v1615: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 736 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 45855/35382 objects degraded (129.600%); 5823/35382 objects misplaced (16.458%); 4.9 MiB/s, 79 objects/s recovering 2024-08-17T17:46:11.635 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 45855/35382 objects degraded (129.600%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:11.635 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:11.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:11.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:11 smithi152 ceph-mon[93456]: pgmap v1615: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 736 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 45855/35382 objects degraded (129.600%); 5823/35382 objects misplaced (16.458%); 4.9 MiB/s, 79 objects/s recovering 2024-08-17T17:46:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 45855/35382 objects degraded (129.600%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:11.834 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:11.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:11.904 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:46:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:46:11] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-08-17T17:46:11.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:11.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:11.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:12.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:12.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:12.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:12.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:12.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:12.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:12.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:12.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:12.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:12.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:12.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:12.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:12.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:12.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:12.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:13.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:13.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:13.085 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:46:13.085 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:13.116 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:46:13.116 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:13.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:13.142 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:46:13.143 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:13.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:13.165 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:46:13.165 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:13.185 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:46:13.186 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:13.209 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:46:13.209 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:13.230 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:46:13.230 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:13.250 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:46:13.251 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:13.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:13.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:13.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:13.408 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:13.510 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:13.537 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:13.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:13.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:13.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:13.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:13.781 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:13 smithi142 ceph-mon[106729]: pgmap v1616: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 664 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 45722/31932 objects degraded (143.186%); 5254/31932 objects misplaced (16.454%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:46:13.781 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3668847715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:13.781 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:13 smithi142 ceph-mon[103614]: pgmap v1616: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 664 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 45722/31932 objects degraded (143.186%); 5254/31932 objects misplaced (16.454%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:46:13.782 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3668847715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:13.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:13.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:13 smithi152 ceph-mon[93456]: pgmap v1616: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 664 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 45722/31932 objects degraded (143.186%); 5254/31932 objects misplaced (16.454%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:46:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3668847715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:13.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:13.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:13.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:13.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:14.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:14.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:14.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:14.191 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:14.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:14.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:14.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:14.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:14.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:14.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:14.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:14.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:14.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:14.830 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:14.868 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:14.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:14.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:137199 12.a 12:52e19bf6:::benchmark_data_smithi142_261410_object19002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2024-08-17T17:46:14.885 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 599.40292707699996, 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.056001591, 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 137199 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.886 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478709+0000", 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.887 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478710+0000", 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4600000000000003e-07 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478714+0000", 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7330000000000003e-06 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478724+0000", 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.888 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478731+0000", 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1169999999999998e-06 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478795+0000", 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4641999999999995e-05 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:15.478804+0000", 2024-08-17T17:46:14.889 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531212+0000", 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.052407855 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531320+0000", 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000107692 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.890 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.531454+0000", 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000134055 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534306+0000", 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028519069999999999 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534367+0000", 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1266000000000006e-05 2024-08-17T17:46:14.891 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534598+0000", 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000231373 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534651+0000", 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2369e-05 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:36:16.534710+0000", 2024-08-17T17:46:14.892 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9517e-05 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:158269 12.e e376/337 12:7a28666e:::benchmark_data_smithi142_261410_object40072:head v 376'9843, mlcod=324'725)", 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 380.05064091100002, 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.037591274000000001, 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.893 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830995+0000", 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.830996+0000", 2024-08-17T17:46:14.894 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9500000000000001e-07 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.831003+0000", 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9709999999999998e-06 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861303+0000", 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.030299947000000001 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.895 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861329+0000", 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6420000000000001e-05 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861402+0000", 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2539999999999993e-05 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.861430+0000", 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8707000000000001e-05 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.896 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.868547+0000", 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0071166140000000003 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:39:54.868586+0000", 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9180000000000001e-05 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.897 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176488 12.8 12:1bdea9b6:::benchmark_data_smithi142_261410_object58291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.125797+0000", 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 226.75583821800001, 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.20733109, 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176488 2024-08-17T17:46:14.898 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125797+0000", 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125797+0000", 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.899 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125799+0000", 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.111e-06 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.900 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125804+0000", 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9200000000000003e-06 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125814+0000", 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0101e-05 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125820+0000", 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3679999999999998e-06 2024-08-17T17:46:14.901 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.125918+0000", 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7891000000000002e-05 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.126103+0000", 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000185012 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.902 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.126274+0000", 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000171158 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.129440+0000", 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003165541 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.129466+0000", 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6562000000000001e-05 2024-08-17T17:46:14.903 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137780+0000", 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0083137720000000005 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137812+0000", 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2169000000000003e-05 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.904 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333129+0000", 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19531648500000001 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:176488 12.8 e389/332 ondisk, result = 0)", 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 226.743917106, 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.195464622, 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.905 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.906 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137719+0000", 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7100000000000001e-07 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137724+0000", 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9019999999999998e-06 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137732+0000", 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7889999999999995e-06 2024-08-17T17:46:14.907 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137737+0000", 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7679999999999999e-06 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137771+0000", 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4144999999999999e-05 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.137776+0000", 2024-08-17T17:46:14.908 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3750000000000002e-06 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333183+0000", 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19540697200000001 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:176491 12.5 e389/332 12:a00ed9f7:::benchmark_data_smithi142_261410_object58294:head v 389'11091, mlcod=322'100)", 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.142636+0000", 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 226.73899973600001, 2024-08-17T17:46:14.909 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.19829270600000001, 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142636+0000", 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142636+0000", 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.910 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142637+0000", 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0359999999999999e-06 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.142644+0000", 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7850000000000003e-06 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333074+0000", 2024-08-17T17:46:14.911 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.190430558 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333085+0000", 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0677e-05 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333168+0000", 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2996000000000003e-05 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.912 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333193+0000", 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4790000000000002e-05 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.340871+0000", 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0076780340000000002 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.340929+0000", 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.783e-05 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.913 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:176493 12.3 e389/337 12:c78d0f5f:::benchmark_data_smithi142_261410_object58296:head v 389'11204, mlcod=324'710)", 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.172427+0000", 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 226.709209136, 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.16378883399999999, 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172427+0000", 2024-08-17T17:46:14.914 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172427+0000", 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172428+0000", 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1090000000000001e-06 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.915 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172437+0000", 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6719999999999999e-06 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333053+0000", 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.160615863 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333072+0000", 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8984999999999999e-05 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.916 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333136+0000", 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3948000000000005e-05 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.333166+0000", 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9390000000000002e-05 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.336181+0000", 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003015551 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.917 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.336215+0000", 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4316000000000001e-05 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:195254 12.a 12:513f1928:::benchmark_data_smithi142_261410_object77057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:21.044159+0000", 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 113.837476485, 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10656647700000001, 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.918 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 195254 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044159+0000", 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.919 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044159+0000", 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044161+0000", 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4920000000000001e-06 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.920 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044166+0000", 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1220000000000003e-06 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044175+0000", 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9299999999999992e-06 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044181+0000", 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8200000000000002e-06 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.921 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.044226+0000", 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5875999999999999e-05 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145525+0000", 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.101298254 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145660+0000", 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135127 2024-08-17T17:46:14.922 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.150503+0000", 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048431489999999997 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.150533+0000", 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9570999999999999e-05 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.923 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.150632+0000", 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9467000000000002e-05 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.150682+0000", 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9608999999999999e-05 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.150726+0000", 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4060000000000002e-05 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.924 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:195257 12.4 e398/319 12:2c7de3bc:::benchmark_data_smithi142_261410_object77060:head v 398'12010, mlcod=398'12008)", 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:21.067009+0000", 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 113.814626734, 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.082271769999999994, 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.925 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.067009+0000", 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.067009+0000", 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.926 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.067010+0000", 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4000000000000001e-07 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.067014+0000", 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1880000000000004e-06 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145492+0000", 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.078478131000000007 2024-08-17T17:46:14.927 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145502+0000", 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0088000000000001e-05 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145566+0000", 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3955999999999995e-05 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.928 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145587+0000", 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0615000000000002e-05 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.149234+0000", 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003647843 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.149281+0000", 2024-08-17T17:46:14.929 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6309e-05 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPush(12.d 398/337 [PushOp(12:b733b66d:::benchmark_data_smithi142_261410_object77061:head, version: 327'4900, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:b733b66d:::benchmark_data_smithi142_261410_object77061:head@327'4900, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:21.077973+0000", 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 113.803662711, 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.070847195000000002, 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.930 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077973+0000", 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077973+0000", 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.931 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077974+0000", 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0299999999999998e-07 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.078001+0000", 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6859000000000002e-05 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145501+0000", 2024-08-17T17:46:14.932 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.067500235000000006 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145526+0000", 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5117999999999999e-05 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145706+0000", 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017985899999999999 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.933 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.145717+0000", 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1306e-05 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.148820+0000", 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003103115 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.934 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:195288 12.5 e398/332 12:ae9db006:::benchmark_data_smithi142_261410_object77091:head v 398'12273, mlcod=322'100)", 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:21.220338+0000", 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 113.661297839, 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.040446120000000002, 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.220338+0000", 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.935 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.220338+0000", 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.220339+0000", 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.071e-06 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.936 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.220346+0000", 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.216e-06 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.220358+0000", 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1841999999999999e-05 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.220365+0000", 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1629999999999999e-06 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.937 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.224550+0000", 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041847179999999996 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.224624+0000", 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4257000000000004e-05 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.260697+0000", 2024-08-17T17:46:14.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.036072699999999999 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.260784+0000", 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7152999999999997e-05 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:205518 12.8 12:1ab94e76:::benchmark_data_smithi142_261410_object87321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:07.482323+0000", 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 67.399313096, 2024-08-17T17:46:14.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.035765646999999998, 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 205518 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.482323+0000", 2024-08-17T17:46:14.940 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.482323+0000", 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.482323+0000", 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3900000000000004e-07 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.482327+0000", 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3069999999999998e-06 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.482336+0000", 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0429999999999996e-06 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.482343+0000", 2024-08-17T17:46:14.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9419999999999998e-06 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.485624+0000", 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032810109999999999 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.485731+0000", 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000107598 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.485835+0000", 2024-08-17T17:46:14.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010357300000000001 2024-08-17T17:46:14.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.488822+0000", 2024-08-17T17:46:14.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029869440000000001 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.488893+0000", 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1478999999999998e-05 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.514046+0000", 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025152906999999999 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.514108+0000", 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2193999999999994e-05 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.518088+0000", 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00398001 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:205638 12.8 12:16e29903:::benchmark_data_smithi142_261410_object87441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:07.921950+0000", 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.959685286999999, 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.039758995999999998, 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 205638 2024-08-17T17:46:14.947 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.921950+0000", 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.921950+0000", 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.921951+0000", 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3300000000000002e-07 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.921954+0000", 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.954e-06 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.921961+0000", 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6200000000000001e-06 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.921965+0000", 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7029999999999997e-06 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.926179+0000", 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042137060000000002 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.926364+0000", 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000184983 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.926546+0000", 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000182375 2024-08-17T17:46:14.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.929658+0000", 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003111319 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.930482+0000", 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00082422300000000001 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.958622+0000", 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.028140321999999999 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.958685+0000", 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2914000000000001e-05 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:07.961709+0000", 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003024344 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.953 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:209637 12.8 12:1a1c756e:::benchmark_data_smithi142_261410_object91440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.102557+0000", 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.779079046, 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.051043553999999998, 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.954 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209637 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.102557+0000", 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.102557+0000", 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.955 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.102557+0000", 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8799999999999998e-07 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.102559+0000", 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0190000000000001e-06 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.102565+0000", 2024-08-17T17:46:14.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3759999999999996e-06 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.102570+0000", 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6360000000000002e-06 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.106763+0000", 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041928879999999997 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.106907+0000", 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014407499999999999 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.107045+0000", 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013798900000000001 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.111583+0000", 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045384620000000001 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.114456+0000", 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028723920000000001 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.150673+0000", 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.036216918000000001 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.150713+0000", 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0624000000000003e-05 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.153600+0000", 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002886787 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.960 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:209641 12.8 12:1fc1321b:::benchmark_data_smithi142_261410_object91444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.110307+0000", 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.771328631999999, 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.045862459000000001, 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209641 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.110307+0000", 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.110307+0000", 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.110308+0000", 2024-08-17T17:46:14.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.33e-06 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.110314+0000", 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.305e-06 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.110325+0000", 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.136e-05 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.110332+0000", 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0849999999999999e-06 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.111610+0000", 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012774889999999999 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.964 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.111778+0000", 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016866 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.111929+0000", 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015027000000000001 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.114601+0000", 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002672574 2024-08-17T17:46:14.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.115356+0000", 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00075500299999999997 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.153674+0000", 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.038317999999999998 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.153740+0000", 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6248999999999995e-05 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.156170+0000", 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024291339999999999 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.967 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:209644 12.8 12:1ae9054b:::benchmark_data_smithi142_261410_object91447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.116555+0000", 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.765080724000001, 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.042393293999999998, 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209644 2024-08-17T17:46:14.968 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116555+0000", 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116555+0000", 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.969 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116556+0000", 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.42e-06 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116562+0000", 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5040000000000002e-06 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116575+0000", 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3484e-05 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116583+0000", 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.943e-06 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116634+0000", 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0436999999999998e-05 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.116796+0000", 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016222900000000001 2024-08-17T17:46:14.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.117001+0000", 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020538899999999999 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.120282+0000", 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032802819999999998 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.120870+0000", 2024-08-17T17:46:14.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00058844300000000002 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.156230+0000", 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.035360317000000002 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.156291+0000", 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0949000000000002e-05 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.158948+0000", 2024-08-17T17:46:14.973 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026568970000000001 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:209646 12.8 12:1bafa1ac:::benchmark_data_smithi142_261410_object91449:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.121188+0000", 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.760447407999997, 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.040548541, 2024-08-17T17:46:14.974 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209646 2024-08-17T17:46:14.975 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121188+0000", 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121188+0000", 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121190+0000", 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2580000000000001e-06 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121195+0000", 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1399999999999999e-06 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121205+0000", 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0491e-05 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.977 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121210+0000", 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0159999999999999e-06 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121263+0000", 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2933999999999998e-05 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.978 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121413+0000", 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149512 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.121537+0000", 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000124077 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.126790+0000", 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052528549999999999 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.126834+0000", 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4561999999999998e-05 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.159003+0000", 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032168836999999999 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.159053+0000", 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0169e-05 2024-08-17T17:46:14.980 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.161737+0000", 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026836899999999999 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:209648 12.8 12:1a9ef934:::benchmark_data_smithi142_261410_object91451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.124803+0000", 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.756832412000001, 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.040953205999999999, 2024-08-17T17:46:14.981 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209648 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.124803+0000", 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.124803+0000", 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.124804+0000", 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1999999999999998e-07 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.124808+0000", 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3409999999999999e-06 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.124815+0000", 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8830000000000006e-06 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.124820+0000", 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5390000000000001e-06 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.126865+0000", 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002045505 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.127050+0000", 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018402300000000001 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.127233+0000", 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018380100000000001 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.130020+0000", 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027869520000000001 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.130395+0000", 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00037440500000000002 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.162697+0000", 2024-08-17T17:46:14.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032302499999999998 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.162759+0000", 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2182999999999997e-05 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.165756+0000", 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029971540000000001 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.987 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:209699 12.e e403/337 12:72b90757:::benchmark_data_smithi142_261410_object91502:head v 403'13063, mlcod=324'725)", 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.227232+0000", 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.654403299000002, 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.041475692000000002, 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.227232+0000", 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.227232+0000", 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.227233+0000", 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7999999999999995e-07 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.227239+0000", 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3609999999999997e-06 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.227250+0000", 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0362e-05 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.227258+0000", 2024-08-17T17:46:14.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2109999999999998e-06 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231884+0000", 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046263340000000002 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231911+0000", 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6506e-05 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.268669+0000", 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.036758117999999999 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.268708+0000", 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9119999999999998e-05 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:209701 12.e e403/337 12:7fa1edc8:::benchmark_data_smithi142_261410_object91504:head v 403'13064, mlcod=324'725)", 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.231858+0000", 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.649777520999997, 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.036876040999999998, 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231858+0000", 2024-08-17T17:46:14.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231858+0000", 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231859+0000", 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.004e-06 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231866+0000", 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8140000000000003e-06 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231876+0000", 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6120000000000005e-06 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.231894+0000", 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8615e-05 2024-08-17T17:46:14.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.240909+0000", 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0090150530000000003 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.240931+0000", 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1759999999999998e-05 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:14.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.268711+0000", 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.027779939999999999 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.268734+0000", 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3243000000000001e-05 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.997 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:209721 12.8 12:1de74163:::benchmark_data_smithi142_261410_object91524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:23.266818+0000", 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 51.614817797999997, 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.035491398, 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 209721 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:46:14.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.266818+0000", 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.266818+0000", 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.266819+0000", 2024-08-17T17:46:14.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3999999999999999e-06 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.266825+0000", 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6609999999999997e-06 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.266839+0000", 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4008e-05 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.266846+0000", 2024-08-17T17:46:15.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2420000000000003e-06 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.270260+0000", 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034135440000000001 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.270434+0000", 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000174372 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:46:15.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.270611+0000", 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017670199999999999 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.274051+0000", 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034400170000000001 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.274647+0000", 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059661399999999998 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.299123+0000", 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024475828000000002 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.299165+0000", 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1652e-05 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:46:15.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:23.302309+0000", 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031443579999999999 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:46:15.004 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:46:15.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:15.034 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:15.138 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:15.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.172 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:46:15.172 INFO:teuthology.orchestra.run.smithi142.stdout:{"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 45583/29826 objects degraded (152.830%), 10 pgs degraded, 7 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2455,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":407,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915591,"num_in_osds":6,"osd_in_since":1723916020,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":9942,"data_bytes":650385381,"bytes_used":7775866880,"bytes_avail":568194347008,"bytes_total":575970213888,"degraded_objects":45583,"degraded_total":29826,"degraded_ratio":1.5282974585931737,"misplaced_objects":4982,"misplaced_total":29826,"misplaced_ratio":0.16703547240662509,"recovering_objects_per_sec":88,"recovering_bytes_per_sec":5817948,"recovering_keys_per_sec":0,"num_objects_recovered":888,"num_bytes_recovered":58195968,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":447},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi142.ndmoid","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.142:8443/","prometheus":"http://172.21.15.142:9283/"}},"servicemap":{"epoch":35,"modified":"2024-08-17T17:43:38.379260+0000","services":{}},"progress_events":{"3d3bda10-bab7-4c41-9814-cc3fb3adf59f":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 36s)","progress":0.83146065473556519,"add_to_ceph_s":true},"802c1a7f-0a81-4579-89da-0888757bb400":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (9m)\n [============................] (remaining: 11m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:46:15.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:15.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:15.304 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:15.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:15.436 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:15.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:15.577 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:15.679 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:15.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:15.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:15 smithi152 ceph-mon[93456]: pgmap v1617: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 620 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 447 op/s; 45583/29826 objects degraded (152.830%); 4982/29826 objects misplaced (16.704%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:46:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3159028101' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:46:15.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:15 smithi142 ceph-mon[103614]: pgmap v1617: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 620 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 447 op/s; 45583/29826 objects degraded (152.830%); 4982/29826 objects misplaced (16.704%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:46:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3159028101' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:46:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:15 smithi142 ceph-mon[106729]: pgmap v1617: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 620 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 447 op/s; 45583/29826 objects degraded (152.830%); 4982/29826 objects misplaced (16.704%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:46:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3159028101' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:46:15.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:15.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:15.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:15.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:16.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:16.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:16.220 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:16.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:16.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:16.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:16.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:16.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:16.613 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:16.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:16.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:16.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:16.775 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 45583/29826 objects degraded (152.830%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:16.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 45583/29826 objects degraded (152.830%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 45583/29826 objects degraded (152.830%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:16.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:16.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:16.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:17.010 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:17.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:17.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:17.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:17.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:17.296 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:17.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:46:17.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:17.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:17.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:17.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:17.538 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:17 smithi142 ceph-mon[106729]: pgmap v1618: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 565 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 45435/27189 objects degraded (167.108%); 4427/27189 objects misplaced (16.282%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:46:17.538 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:17 smithi142 ceph-mon[103614]: pgmap v1618: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 565 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 45435/27189 objects degraded (167.108%); 4427/27189 objects misplaced (16.282%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:46:17.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:17.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:17.682 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:17.718 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:17.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:17.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:17.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:17 smithi152 ceph-mon[93456]: pgmap v1618: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 565 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 45435/27189 objects degraded (167.108%); 4427/27189 objects misplaced (16.282%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:46:17.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:17.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:17.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:18.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:18.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:18.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:18.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:18.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:18.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:18.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:18.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:18.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:18.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:18.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:18.589 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:18.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:18.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:18.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:18.855 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:18.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:18.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:18.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:19.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:19.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.134 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:19.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:19.155 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:46:19.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:46:19.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:19.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:19.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:19.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:19.427 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:19.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:19.560 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:19.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:19.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:19.703 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:19.782 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:19 smithi142 ceph-mon[106729]: pgmap v1619: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 483 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 45435/23259 objects degraded (195.344%); 4242/23259 objects misplaced (18.238%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:46:19.783 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:19 smithi142 ceph-mon[103614]: pgmap v1619: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 483 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 45435/23259 objects degraded (195.344%); 4242/23259 objects misplaced (18.238%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:46:19.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:19 smithi152 ceph-mon[93456]: pgmap v1619: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 483 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 45435/23259 objects degraded (195.344%); 4242/23259 objects misplaced (18.238%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:46:19.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:19.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:19.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:20.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:20.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:20.085 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:230618 12.0 12:0635ba88:::benchmark_data_smithi142_261410_object112421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.080775+0000", 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.003578234, 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036095469999999998, 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:46:20.086 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 230618 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.080775+0000", 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.080775+0000", 2024-08-17T17:46:20.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.080774+0000", 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.080782+0000", 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8949999999999998e-06 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.080794+0000", 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2561000000000001e-05 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.080801+0000", 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.691e-06 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.081138+0000", 2024-08-17T17:46:20.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00033692200000000002 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.081273+0000", 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134976 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6,7", 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.081404+0000", 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000131373 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.090 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:230602 12.f e407/332 12:f09460fc:::benchmark_data_smithi142_261410_object112405:head v 407'14427, mlcod=322'102)", 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.054950+0000", 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.029403417000000001, 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.029506496, 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.054950+0000", 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.054950+0000", 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.054950+0000", 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.054955+0000", 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6559999999999997e-06 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.055053+0000", 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7751999999999995e-05 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.055066+0000", 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.29e-05 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.066280+0000", 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01121369 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.066303+0000", 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3295999999999999e-05 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.094 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:230606 12.7 12:e5b130a7:::benchmark_data_smithi142_261410_object112409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.062850+0000", 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.021503616999999999, 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021663064999999999, 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for missing object", 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 230606 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.062850+0000", 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.062850+0000", 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.062848+0000", 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.062875+0000", 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6254e-05 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.062891+0000", 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6747000000000001e-05 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.062899+0000", 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2690000000000002e-06 2024-08-17T17:46:20.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065203+0000", 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023039810000000001 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.067894+0000", 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002691531 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:230609 12.3 e407/337 12:c39a7471:::benchmark_data_smithi142_261410_object112412:head v 407'14588, mlcod=324'710)", 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.065880+0000", 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.018473558000000001, 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018695031000000001, 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065880+0000", 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065880+0000", 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065879+0000", 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:46:20.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065891+0000", 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2173e-05 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065903+0000", 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1769e-05 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.065909+0000", 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.49e-06 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.067794+0000", 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018849139999999999 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.067816+0000", 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1937e-05 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.102 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(12.7 407/337 [PushOp(12:e5b130a7:::benchmark_data_smithi142_261410_object112409:head, version: 330'7008, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:e5b130a7:::benchmark_data_smithi142_261410_object112409:head@330'7008, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.068693+0000", 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.015660697000000001, 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015961271999999999, 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.068693+0000", 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.068693+0000", 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.068692+0000", 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.068732+0000", 2024-08-17T17:46:20.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0098999999999997e-05 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.068754+0000", 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1776999999999998e-05 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.068764+0000", 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7219999999999994e-06 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.077936+0000", 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091720469999999991 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.078042+0000", 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106189 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:230605 12.f e407/332 12:faf9d916:::benchmark_data_smithi142_261410_object112408:head v 407'14428, mlcod=322'102)", 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.069260+0000", 2024-08-17T17:46:20.106 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.015093591999999999, 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015452712, 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.069260+0000", 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.069260+0000", 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.069259+0000", 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.069267+0000", 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1419999999999993e-06 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.069285+0000", 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7702000000000001e-05 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.069292+0000", 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6479999999999997e-06 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.078191+0000", 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088996800000000001 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.078216+0000", 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4349e-05 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:230612 12.3 e407/337 12:c19a6bfb:::benchmark_data_smithi142_261410_object112415:head v 407'14589, mlcod=324'710)", 2024-08-17T17:46:20.110 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.075122+0000", 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0092320230000000007, 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096405060000000001, 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.075122+0000", 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.075122+0000", 2024-08-17T17:46:20.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.075121+0000", 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.075129+0000", 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6640000000000004e-06 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.075141+0000", 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1610999999999999e-05 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.075147+0000", 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0639999999999999e-06 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.079977+0000", 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048303280000000001 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.079999+0000", 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2082e-05 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46715.0:230613 12.3 e407/337)", 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.076870+0000", 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0074835869999999999, 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079387409999999992, 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.114 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076870+0000", 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076870+0000", 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076869+0000", 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076877+0000", 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1130000000000001e-06 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076894+0000", 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6359999999999999e-05 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076899+0000", 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5369999999999997e-06 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:230614 12.7 12.73b09b07 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:46:20.076927+0000", 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0074271220000000004, 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079215659999999997, 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:46:20.117 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 230614 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076927+0000", 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076927+0000", 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:46:20.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076925+0000", 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076932+0000", 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.985e-06 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076944+0000", 2024-08-17T17:46:20.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2322e-05 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:46:20.076950+0000", 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6370000000000004e-06 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 9 2024-08-17T17:46:20.120 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:46:20.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:20.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:20.224 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:20.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:20.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:20.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:20.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:20.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:20.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:46:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:46:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:20.583 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:20.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:20.715 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:20.816 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:20.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:20.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:20.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:20.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:20.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:21.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:21.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:21.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:21.219 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:21.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:21.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:46:21.353 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:21.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:21.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:21.505 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:21.512 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:46:21.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:21.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:21.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:21.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:21.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-mon[103614]: pgmap v1620: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 433 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 461 op/s; 45274/20844 objects degraded (217.204%); 3942/20844 objects misplaced (18.912%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:46:21.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 45274/20844 objects degraded (217.204%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:21.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3263653243' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:21.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-mon[106729]: pgmap v1620: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 433 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 461 op/s; 45274/20844 objects degraded (217.204%); 3942/20844 objects misplaced (18.912%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:46:21.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 45274/20844 objects degraded (217.204%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:21.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3263653243' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:21 smithi152 ceph-mon[93456]: pgmap v1620: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 433 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 461 op/s; 45274/20844 objects degraded (217.204%); 3942/20844 objects misplaced (18.912%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:46:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 45274/20844 objects degraded (217.204%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:21.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3263653243' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:46:21.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:21.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:21.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:21.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.023 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:22.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:22.077 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:46:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:46:21] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-08-17T17:46:22.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:22.153 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:22.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:22.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:22.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:22.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:22.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:22.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:22.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:22.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:22.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:22.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:22.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:22.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:22 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:46:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3263653243' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:22 smithi152 ceph-mon[93456]: osdmap e408: 8 total, 8 up, 6 in 2024-08-17T17:46:22.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:22 smithi152 ceph-mon[93456]: pgmap v1622: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 354 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 45121/17061 objects degraded (264.469%); 3158/17061 objects misplaced (18.510%); 7.4 MiB/s, 123 objects/s recovering 2024-08-17T17:46:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:46:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3263653243' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[106729]: osdmap e408: 8 total, 8 up, 6 in 2024-08-17T17:46:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[106729]: pgmap v1622: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 354 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 45121/17061 objects degraded (264.469%); 3158/17061 objects misplaced (18.510%); 7.4 MiB/s, 123 objects/s recovering 2024-08-17T17:46:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:46:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3263653243' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:46:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[103614]: osdmap e408: 8 total, 8 up, 6 in 2024-08-17T17:46:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:22 smithi142 ceph-mon[103614]: pgmap v1622: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 354 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 45121/17061 objects degraded (264.469%); 3158/17061 objects misplaced (18.510%); 7.4 MiB/s, 123 objects/s recovering 2024-08-17T17:46:22.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:22.942 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:22.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:23.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:23.087 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:23.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:23.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:23.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:23.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:23.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:23.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:23.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:23.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:23.716 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:23.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:23.854 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:23.891 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:23.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:23.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:24.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:24.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:24.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:24.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:24.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:24.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:24.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:24.423 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:24.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:46:24.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:24.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:24.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:24.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:24.837 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:24.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:24.874 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:24.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:24.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:24.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:24.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:25.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:25.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:25.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:25.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.272 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:25.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:25.373 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:25.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:25.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:25.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:25.587 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:46:25.588 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:25 smithi142 ceph-mon[106729]: pgmap v1623: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 305 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 44973/14688 objects degraded (306.189%); 2832/14688 objects misplaced (19.281%); 7.5 MiB/s, 124 objects/s recovering 2024-08-17T17:46:25.590 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:46:25.592 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:25 smithi142 ceph-mon[103614]: pgmap v1623: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 305 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 44973/14688 objects degraded (306.189%); 2832/14688 objects misplaced (19.281%); 7.5 MiB/s, 124 objects/s recovering 2024-08-17T17:46:25.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:25.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:25.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:25.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:25 smithi152 ceph-mon[93456]: pgmap v1623: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 305 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 44973/14688 objects degraded (306.189%); 2832/14688 objects misplaced (19.281%); 7.5 MiB/s, 124 objects/s recovering 2024-08-17T17:46:25.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:25.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:25.950 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:25.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:26.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:26.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:26.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:26.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:26.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:26.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:26.372 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:46:26.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:26.480 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:26.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:26.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:26.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[103614]: from='client.47942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:26.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 44973/14688 objects degraded (306.189%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:26.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:46:26.669 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:26.670 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[106729]: from='client.47942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:26.670 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 44973/14688 objects degraded (306.189%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:26.670 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:46:26.670 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:26.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:26.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:26.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:26.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:26 smithi152 ceph-mon[93456]: from='client.47942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:46:26.778 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 44973/14688 objects degraded (306.189%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:26.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:46:26.779 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:26.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:26.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:26.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:27.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:27.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:27.112 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:46:27.112 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:46:27.112 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:46:27.112 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:46:27.113 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:46:27.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:27.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:27.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:27.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:27.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:27.410 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:27.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:27.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:27.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:27.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:27.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:27.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:27 smithi152 ceph-mon[93456]: pgmap v1624: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 242 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 44801/11649 objects degraded (384.591%); 2475/11649 objects misplaced (21.246%); 5.9 MiB/s, 155 objects/s recovering 2024-08-17T17:46:27.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:27.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:27.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:27 smithi142 ceph-mon[103614]: pgmap v1624: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 242 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 44801/11649 objects degraded (384.591%); 2475/11649 objects misplaced (21.246%); 5.9 MiB/s, 155 objects/s recovering 2024-08-17T17:46:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:27 smithi142 ceph-mon[106729]: pgmap v1624: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 242 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 44801/11649 objects degraded (384.591%); 2475/11649 objects misplaced (21.246%); 5.9 MiB/s, 155 objects/s recovering 2024-08-17T17:46:27.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:27.990 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1623,"stamp":"2024-08-17T17:46:24.427956+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":319690725,"num_objects":4896,"num_object_clones":0,"num_object_copies":14688,"num_objects_missing_on_primary":1026,"num_objects_missing":1026,"num_objects_degraded":44973,"num_objects_misplaced":2832,"num_objects_unfound":0,"num_objects_dirty":4896,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":231798,"num_write_kb":7566864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41783,"num_bytes_recovered":2612404878,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104946,"ondisk_log_size":104946,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6939272,"kb_used_data":697300,"kb_used_omap":27,"kb_used_meta":6241892,"kb_avail":555531640,"statfs":{"total":575970213888,"available":568864399360,"internally_reserved":0,"allocated":714035200,"data_stored":700453054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6391698152},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,4,3,4,3,2,2,4],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-330694656,"num_objects":-5046,"num_object_clones":0,"num_object_copies":-15138,"num_objects_missing_on_primary":-296,"num_objects_missing":-296,"num_objects_degraded":-610,"num_objects_misplaced":-2150,"num_objects_unfound":0,"num_objects_dirty":-5046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5046,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1246,"num_bytes_recovered":78643200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002654"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":661,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519097+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:46:21.519097+0000","last_peered":"2024-08-17T17:46:21.519097+0000","last_clean":"2024-08-17T17:46:21.519097+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:46:21.519097+0000","last_undegraded":"2024-08-17T17:46:21.519097+0000","last_fullsized":"2024-08-17T17:46:21.519097+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":116,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508306+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:45:52.508306+0000","last_peered":"2024-08-17T17:45:52.508306+0000","last_clean":"2024-08-17T17:45:52.508306+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:45:52.508306+0000","last_undegraded":"2024-08-17T17:45:52.508306+0000","last_fullsized":"2024-08-17T17:45:52.508306+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":310,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.521262+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:45:52.521262+0000","last_peered":"2024-08-17T17:45:52.521262+0000","last_clean":"2024-08-17T17:45:52.521262+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:45:52.521262+0000","last_undegraded":"2024-08-17T17:45:52.521262+0000","last_fullsized":"2024-08-17T17:45:52.521262+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":359,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519136+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:46:21.519136+0000","last_peered":"2024-08-17T17:46:21.519136+0000","last_clean":"2024-08-17T17:46:21.519136+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:46:21.519136+0000","last_undegraded":"2024-08-17T17:46:21.519136+0000","last_fullsized":"2024-08-17T17:46:21.519136+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":536,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519453+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:46:21.519453+0000","last_peered":"2024-08-17T17:46:21.519453+0000","last_clean":"2024-08-17T17:46:21.519453+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:46:21.519453+0000","last_undegraded":"2024-08-17T17:46:21.519453+0000","last_fullsized":"2024-08-17T17:46:21.519453+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":259,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.506975+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:45:52.506975+0000","last_peered":"2024-08-17T17:45:52.506975+0000","last_clean":"2024-08-17T17:45:52.506975+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:45:52.506975+0000","last_undegraded":"2024-08-17T17:45:52.506975+0000","last_fullsized":"2024-08-17T17:45:52.506975+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":629,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519176+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:46:21.519176+0000","last_peered":"2024-08-17T17:46:21.519176+0000","last_clean":"2024-08-17T17:46:21.519176+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:46:21.519176+0000","last_undegraded":"2024-08-17T17:46:21.519176+0000","last_fullsized":"2024-08-17T17:46:21.519176+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":428,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.498110+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:46:06.498110+0000","last_peered":"2024-08-17T17:46:06.498110+0000","last_clean":"2024-08-17T17:46:06.498110+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:46:06.498110+0000","last_undegraded":"2024-08-17T17:46:06.498110+0000","last_fullsized":"2024-08-17T17:46:06.498110+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"407'14366","reported_seq":27823,"reported_epoch":407,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:46:20.929209+0000","last_change":"2024-08-17T17:34:59.191982+0000","last_active":"2024-08-17T17:46:20.929209+0000","last_peered":"2024-08-17T17:46:20.929209+0000","last_clean":"2024-08-17T17:33:19.241164+0000","last_became_active":"2024-08-17T17:33:42.096040+0000","last_became_peered":"2024-08-17T17:33:42.096040+0000","last_unstale":"2024-08-17T17:46:20.929209+0000","last_undegraded":"2024-08-17T17:46:20.929209+0000","last_fullsized":"2024-08-17T17:46:20.929209+0000","mapping_epoch":337,"log_start":"330'7463","ondisk_log_start":"330'7463","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:53.040353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22151168,"num_objects":338,"num_object_clones":0,"num_object_copies":1014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14366,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6669,"num_bytes_recovered":434044928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,6,7","objects":338}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":112,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.521878+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:45:52.521878+0000","last_peered":"2024-08-17T17:45:52.521878+0000","last_clean":"2024-08-17T17:45:52.521878+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:45:52.521878+0000","last_undegraded":"2024-08-17T17:45:52.521878+0000","last_fullsized":"2024-08-17T17:45:52.521878+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":293,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.522081+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:45:52.522081+0000","last_peered":"2024-08-17T17:45:52.522081+0000","last_clean":"2024-08-17T17:45:52.522081+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:45:52.522081+0000","last_undegraded":"2024-08-17T17:45:52.522081+0000","last_fullsized":"2024-08-17T17:45:52.522081+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"408'14812","reported_seq":14918,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:46:22.248758+0000","last_change":"2024-08-17T17:33:41.788820+0000","last_active":"2024-08-17T17:46:22.248758+0000","last_peered":"2024-08-17T17:46:22.248758+0000","last_clean":"2024-08-17T17:33:40.165964+0000","last_became_active":"2024-08-17T17:33:41.766813+0000","last_became_peered":"2024-08-17T17:33:41.766813+0000","last_unstale":"2024-08-17T17:46:22.248758+0000","last_undegraded":"2024-08-17T17:46:22.248758+0000","last_fullsized":"2024-08-17T17:46:22.248758+0000","mapping_epoch":337,"log_start":"335'8282","ondisk_log_start":"335'8282","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:14.232382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17170432,"num_objects":262,"num_object_clones":0,"num_object_copies":786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":262,"num_objects_unfound":0,"num_objects_dirty":262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14812,"num_write_kb":482368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":262}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":114,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507666+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:45:52.507666+0000","last_peered":"2024-08-17T17:45:52.507666+0000","last_clean":"2024-08-17T17:45:52.507666+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:45:52.507666+0000","last_undegraded":"2024-08-17T17:45:52.507666+0000","last_fullsized":"2024-08-17T17:45:52.507666+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":378,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.506905+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:45:52.506905+0000","last_peered":"2024-08-17T17:45:52.506905+0000","last_clean":"2024-08-17T17:45:52.506905+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:45:52.506905+0000","last_undegraded":"2024-08-17T17:45:52.506905+0000","last_fullsized":"2024-08-17T17:45:52.506905+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"407'14390","reported_seq":14498,"reported_epoch":407,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:46:20.858363+0000","last_change":"2024-08-17T17:33:41.860890+0000","last_active":"2024-08-17T17:46:20.858363+0000","last_peered":"2024-08-17T17:46:20.858363+0000","last_clean":"2024-08-17T17:33:40.265405+0000","last_became_active":"2024-08-17T17:33:41.847263+0000","last_became_peered":"2024-08-17T17:33:41.847263+0000","last_unstale":"2024-08-17T17:46:20.858363+0000","last_undegraded":"2024-08-17T17:46:20.858363+0000","last_fullsized":"2024-08-17T17:46:20.858363+0000","mapping_epoch":337,"log_start":"334'7553","ondisk_log_start":"334'7553","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:47.169039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22544384,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":344,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14390,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":344}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":110,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.521052+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:45:52.521052+0000","last_peered":"2024-08-17T17:45:52.521052+0000","last_clean":"2024-08-17T17:45:52.521052+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:45:52.521052+0000","last_undegraded":"2024-08-17T17:45:52.521052+0000","last_fullsized":"2024-08-17T17:45:52.521052+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":389,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.498048+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:46:06.498048+0000","last_peered":"2024-08-17T17:46:06.498048+0000","last_clean":"2024-08-17T17:46:06.498048+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:46:06.498048+0000","last_undegraded":"2024-08-17T17:46:06.498048+0000","last_fullsized":"2024-08-17T17:46:06.498048+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"407'14587","reported_seq":14784,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:19.980901+0000","last_change":"2024-08-17T17:33:41.982942+0000","last_active":"2024-08-17T17:46:19.980901+0000","last_peered":"2024-08-17T17:46:19.980901+0000","last_clean":"2024-08-17T17:33:40.256678+0000","last_became_active":"2024-08-17T17:33:41.948711+0000","last_became_peered":"2024-08-17T17:33:41.948711+0000","last_unstale":"2024-08-17T17:46:19.980901+0000","last_undegraded":"2024-08-17T17:33:41.696752+0000","last_fullsized":"2024-08-17T17:33:41.632023+0000","mapping_epoch":337,"log_start":"330'7513","ondisk_log_start":"330'7513","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24313856,"num_objects":371,"num_object_clones":0,"num_object_copies":1113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14587,"num_write_kb":478656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":-6398},{"shards":"0,3,7","objects":371}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":573,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519292+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:46:21.519292+0000","last_peered":"2024-08-17T17:46:21.519292+0000","last_clean":"2024-08-17T17:46:21.519292+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:46:21.519292+0000","last_undegraded":"2024-08-17T17:46:21.519292+0000","last_fullsized":"2024-08-17T17:46:21.519292+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":337,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.493076+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:46:06.493076+0000","last_peered":"2024-08-17T17:46:06.493076+0000","last_clean":"2024-08-17T17:46:06.493076+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:46:06.493076+0000","last_undegraded":"2024-08-17T17:46:06.493076+0000","last_fullsized":"2024-08-17T17:46:06.493076+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"408'14302","reported_seq":14399,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:22.268807+0000","last_change":"2024-08-17T17:31:36.267392+0000","last_active":"2024-08-17T17:46:22.268807+0000","last_peered":"2024-08-17T17:46:22.268807+0000","last_clean":"2024-08-17T17:46:22.268807+0000","last_became_active":"2024-08-17T17:31:36.267080+0000","last_became_peered":"2024-08-17T17:31:36.267080+0000","last_unstale":"2024-08-17T17:46:22.268807+0000","last_undegraded":"2024-08-17T17:46:22.268807+0000","last_fullsized":"2024-08-17T17:46:22.268807+0000","mapping_epoch":319,"log_start":"335'7774","ondisk_log_start":"335'7774","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:29.034106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19136512,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14302,"num_write_kb":467008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":663,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519368+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:46:21.519368+0000","last_peered":"2024-08-17T17:46:21.519368+0000","last_clean":"2024-08-17T17:46:21.519368+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:46:21.519368+0000","last_undegraded":"2024-08-17T17:46:21.519368+0000","last_fullsized":"2024-08-17T17:46:21.519368+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":361,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507647+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:45:52.507647+0000","last_peered":"2024-08-17T17:45:52.507647+0000","last_clean":"2024-08-17T17:45:52.507647+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:45:52.507647+0000","last_undegraded":"2024-08-17T17:45:52.507647+0000","last_fullsized":"2024-08-17T17:45:52.507647+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"408'14610","reported_seq":14717,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:22.244023+0000","last_change":"2024-08-17T17:33:21.776048+0000","last_active":"2024-08-17T17:46:22.244023+0000","last_peered":"2024-08-17T17:46:22.244023+0000","last_clean":"2024-08-17T17:33:19.199778+0000","last_became_active":"2024-08-17T17:33:21.749797+0000","last_became_peered":"2024-08-17T17:33:21.749797+0000","last_unstale":"2024-08-17T17:46:22.244023+0000","last_undegraded":"2024-08-17T17:33:21.340796+0000","last_fullsized":"2024-08-17T17:33:21.270280+0000","mapping_epoch":332,"log_start":"335'8035","ondisk_log_start":"335'8035","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6575,"log_dups_size":0,"ondisk_log_size":6575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:28.488360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21102592,"num_objects":322,"num_object_clones":0,"num_object_copies":966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7466,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14610,"num_write_kb":477824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":322},{"shards":"1,6","objects":-7044}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":363,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.498179+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:46:06.498179+0000","last_peered":"2024-08-17T17:46:06.498179+0000","last_clean":"2024-08-17T17:46:06.498179+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:46:06.498179+0000","last_undegraded":"2024-08-17T17:46:06.498179+0000","last_fullsized":"2024-08-17T17:46:06.498179+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":388,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507517+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:45:52.507517+0000","last_peered":"2024-08-17T17:45:52.507517+0000","last_clean":"2024-08-17T17:45:52.507517+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:45:52.507517+0000","last_undegraded":"2024-08-17T17:45:52.507517+0000","last_fullsized":"2024-08-17T17:45:52.507517+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"408'14786","reported_seq":14985,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:46:23.945199+0000","last_change":"2024-08-17T17:33:41.901974+0000","last_active":"2024-08-17T17:46:23.945199+0000","last_peered":"2024-08-17T17:46:23.945199+0000","last_clean":"2024-08-17T17:33:40.278341+0000","last_became_active":"2024-08-17T17:33:41.888930+0000","last_became_peered":"2024-08-17T17:33:41.888930+0000","last_unstale":"2024-08-17T17:46:23.945199+0000","last_undegraded":"2024-08-17T17:46:23.945199+0000","last_fullsized":"2024-08-17T17:46:23.945199+0000","mapping_epoch":337,"log_start":"363'9161","ondisk_log_start":"363'9161","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14680064,"num_objects":224,"num_object_clones":0,"num_object_copies":672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":224,"num_objects_unfound":0,"num_objects_dirty":224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14786,"num_write_kb":480320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,7],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":224}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":399,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507103+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:45:52.507103+0000","last_peered":"2024-08-17T17:45:52.507103+0000","last_clean":"2024-08-17T17:45:52.507103+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:45:52.507103+0000","last_undegraded":"2024-08-17T17:45:52.507103+0000","last_fullsized":"2024-08-17T17:45:52.507103+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":361,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507784+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:45:52.507784+0000","last_peered":"2024-08-17T17:45:52.507784+0000","last_clean":"2024-08-17T17:45:52.507784+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:45:52.507784+0000","last_undegraded":"2024-08-17T17:45:52.507784+0000","last_fullsized":"2024-08-17T17:45:52.507784+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"407'14379","reported_seq":27140,"reported_epoch":407,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:46:20.897331+0000","last_change":"2024-08-17T17:34:24.477872+0000","last_active":"2024-08-17T17:46:20.897331+0000","last_peered":"2024-08-17T17:46:20.897331+0000","last_clean":"2024-08-17T17:33:19.264288+0000","last_became_active":"2024-08-17T17:34:24.300565+0000","last_became_peered":"2024-08-17T17:34:24.300565+0000","last_unstale":"2024-08-17T17:46:20.897331+0000","last_undegraded":"2024-08-17T17:34:12.732068+0000","last_fullsized":"2024-08-17T17:46:20.897331+0000","mapping_epoch":337,"log_start":"334'7541","ondisk_log_start":"334'7541","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6838,"log_dups_size":0,"ondisk_log_size":6838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:05.585754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299200,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":1026,"num_objects_missing":1026,"num_objects_degraded":325,"num_objects_misplaced":1026,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14379,"num_write_kb":470528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6322,"num_bytes_recovered":414253056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,4,7","objects":4},{"shards":"1,2,7","objects":320},{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":632,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.518774+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:46:21.518774+0000","last_peered":"2024-08-17T17:46:21.518774+0000","last_clean":"2024-08-17T17:46:21.518774+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:46:21.518774+0000","last_undegraded":"2024-08-17T17:46:21.518774+0000","last_fullsized":"2024-08-17T17:46:21.518774+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":569,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519452+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:46:21.519452+0000","last_peered":"2024-08-17T17:46:21.519452+0000","last_clean":"2024-08-17T17:46:21.519452+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:46:21.519452+0000","last_undegraded":"2024-08-17T17:46:21.519452+0000","last_fullsized":"2024-08-17T17:46:21.519452+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.d","version":"408'14670","reported_seq":27802,"reported_epoch":408,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-08-17T17:46:24.078136+0000","last_change":"2024-08-17T17:33:42.004429+0000","last_active":"2024-08-17T17:46:24.078136+0000","last_peered":"2024-08-17T17:46:24.078136+0000","last_clean":"2024-08-17T17:33:19.231931+0000","last_became_active":"2024-08-17T17:33:41.904913+0000","last_became_peered":"2024-08-17T17:33:41.904913+0000","last_unstale":"2024-08-17T17:46:24.078136+0000","last_undegraded":"2024-08-17T17:33:41.881049+0000","last_fullsized":"2024-08-17T17:46:24.078136+0000","mapping_epoch":337,"log_start":"362'9044","ondisk_log_start":"362'9044","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":5626,"log_dups_size":0,"ondisk_log_size":5626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14745624,"num_objects":226,"num_object_clones":0,"num_object_copies":678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":226,"num_objects_misplaced":976,"num_objects_unfound":0,"num_objects_dirty":226,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14670,"num_write_kb":476609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6472,"num_bytes_recovered":424083456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":226}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":761,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:06.491538+0000","last_change":"2024-08-17T17:33:41.651679+0000","last_active":"2024-08-17T17:46:06.491538+0000","last_peered":"2024-08-17T17:46:06.491538+0000","last_clean":"2024-08-17T17:26:30.205823+0000","last_became_active":"2024-08-17T17:33:41.647680+0000","last_became_peered":"2024-08-17T17:33:41.647680+0000","last_unstale":"2024-08-17T17:46:06.491538+0000","last_undegraded":"2024-08-17T17:33:41.632562+0000","last_fullsized":"2024-08-17T17:33:41.632017+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":336,"reported_epoch":406,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:45:52.520497+0000","last_change":"2024-08-17T17:33:40.660129+0000","last_active":"2024-08-17T17:45:52.520497+0000","last_peered":"2024-08-17T17:45:52.520497+0000","last_clean":"2024-08-17T17:33:29.417577+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:45:52.520497+0000","last_undegraded":"2024-08-17T17:33:40.633427+0000","last_fullsized":"2024-08-17T17:45:52.520497+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":612,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519026+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:46:21.519026+0000","last_peered":"2024-08-17T17:46:21.519026+0000","last_clean":"2024-08-17T17:46:21.519026+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:46:21.519026+0000","last_undegraded":"2024-08-17T17:46:21.519026+0000","last_fullsized":"2024-08-17T17:46:21.519026+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":372,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.500255+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:46:06.500255+0000","last_peered":"2024-08-17T17:46:06.500255+0000","last_clean":"2024-08-17T17:46:06.500255+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:46:06.500255+0000","last_undegraded":"2024-08-17T17:46:06.500255+0000","last_fullsized":"2024-08-17T17:46:06.500255+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"407'14363","reported_seq":14562,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:20.031036+0000","last_change":"2024-08-17T17:33:41.984992+0000","last_active":"2024-08-17T17:46:20.031036+0000","last_peered":"2024-08-17T17:46:20.031036+0000","last_clean":"2024-08-17T17:33:40.248775+0000","last_became_active":"2024-08-17T17:33:41.901789+0000","last_became_peered":"2024-08-17T17:33:41.901789+0000","last_unstale":"2024-08-17T17:46:20.031036+0000","last_undegraded":"2024-08-17T17:33:41.880529+0000","last_fullsized":"2024-08-17T17:33:41.633438+0000","mapping_epoch":337,"log_start":"330'7321","ondisk_log_start":"330'7321","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7042,"log_dups_size":0,"ondisk_log_size":7042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14363,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":-6266},{"shards":"0,4,7","objects":381}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":428,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.515452+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:46:21.515452+0000","last_peered":"2024-08-17T17:46:21.515452+0000","last_clean":"2024-08-17T17:46:21.515452+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:46:21.515452+0000","last_undegraded":"2024-08-17T17:46:21.515452+0000","last_fullsized":"2024-08-17T17:46:21.515452+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":343,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.506776+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:45:52.506776+0000","last_peered":"2024-08-17T17:45:52.506776+0000","last_clean":"2024-08-17T17:45:52.506776+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:45:52.506776+0000","last_undegraded":"2024-08-17T17:45:52.506776+0000","last_fullsized":"2024-08-17T17:45:52.506776+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":352,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.495870+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:46:06.495870+0000","last_peered":"2024-08-17T17:46:06.495870+0000","last_clean":"2024-08-17T17:46:06.495870+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:46:06.495870+0000","last_undegraded":"2024-08-17T17:46:06.495870+0000","last_fullsized":"2024-08-17T17:46:06.495870+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"408'14487","reported_seq":14594,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:22.258139+0000","last_change":"2024-08-17T17:33:21.820245+0000","last_active":"2024-08-17T17:46:22.258139+0000","last_peered":"2024-08-17T17:46:22.258139+0000","last_clean":"2024-08-17T17:33:19.264087+0000","last_became_active":"2024-08-17T17:33:21.793060+0000","last_became_peered":"2024-08-17T17:33:21.793060+0000","last_unstale":"2024-08-17T17:46:22.258139+0000","last_undegraded":"2024-08-17T17:33:21.360483+0000","last_fullsized":"2024-08-17T17:33:21.270511+0000","mapping_epoch":332,"log_start":"335'7917","ondisk_log_start":"335'7917","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:41:48.197293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20250624,"num_objects":309,"num_object_clones":0,"num_object_copies":927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7398,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14487,"num_write_kb":473472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":309},{"shards":"5,6","objects":-6987}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":520,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519548+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:46:21.519548+0000","last_peered":"2024-08-17T17:46:21.519548+0000","last_clean":"2024-08-17T17:46:21.519548+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:46:21.519548+0000","last_undegraded":"2024-08-17T17:46:21.519548+0000","last_fullsized":"2024-08-17T17:46:21.519548+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.497670+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:46:06.497670+0000","last_peered":"2024-08-17T17:46:06.497670+0000","last_clean":"2024-08-17T17:46:06.497670+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:46:06.497670+0000","last_undegraded":"2024-08-17T17:46:06.497670+0000","last_fullsized":"2024-08-17T17:46:06.497670+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":704,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.516740+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:46:21.516740+0000","last_peered":"2024-08-17T17:46:21.516740+0000","last_clean":"2024-08-17T17:46:21.516740+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:46:21.516740+0000","last_undegraded":"2024-08-17T17:46:21.516740+0000","last_fullsized":"2024-08-17T17:46:21.516740+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"407'14286","reported_seq":15209,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:20.005286+0000","last_change":"2024-08-17T17:33:40.905555+0000","last_active":"2024-08-17T17:46:20.005286+0000","last_peered":"2024-08-17T17:46:20.005286+0000","last_clean":"2024-08-17T17:33:19.272829+0000","last_became_active":"2024-08-17T17:33:40.881065+0000","last_became_peered":"2024-08-17T17:33:40.881065+0000","last_unstale":"2024-08-17T17:46:20.005286+0000","last_undegraded":"2024-08-17T17:33:40.688884+0000","last_fullsized":"2024-08-17T17:33:40.630020+0000","mapping_epoch":336,"log_start":"330'7216","ondisk_log_start":"330'7216","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23592960,"num_objects":360,"num_object_clones":0,"num_object_copies":1080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14286,"num_write_kb":468672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":23658496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":360},{"shards":"0,5","objects":-6294}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":496,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.515719+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:46:21.515719+0000","last_peered":"2024-08-17T17:46:21.515719+0000","last_clean":"2024-08-17T17:46:21.515719+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:46:21.515719+0000","last_undegraded":"2024-08-17T17:46:21.515719+0000","last_fullsized":"2024-08-17T17:46:21.515719+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":382,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508453+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:45:52.508453+0000","last_peered":"2024-08-17T17:45:52.508453+0000","last_clean":"2024-08-17T17:45:52.508453+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:45:52.508453+0000","last_undegraded":"2024-08-17T17:45:52.508453+0000","last_fullsized":"2024-08-17T17:45:52.508453+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519008+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:46:21.519008+0000","last_peered":"2024-08-17T17:46:21.519008+0000","last_clean":"2024-08-17T17:46:21.519008+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:46:21.519008+0000","last_undegraded":"2024-08-17T17:46:21.519008+0000","last_fullsized":"2024-08-17T17:46:21.519008+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"408'14563","reported_seq":29608,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:22.960746+0000","last_change":"2024-08-17T17:36:16.534158+0000","last_active":"2024-08-17T17:46:22.960746+0000","last_peered":"2024-08-17T17:46:22.960746+0000","last_clean":"2024-08-17T17:46:22.960746+0000","last_became_active":"2024-08-17T17:36:16.517494+0000","last_became_peered":"2024-08-17T17:36:16.517494+0000","last_unstale":"2024-08-17T17:46:22.960746+0000","last_undegraded":"2024-08-17T17:46:22.960746+0000","last_fullsized":"2024-08-17T17:46:22.960746+0000","mapping_epoch":355,"log_start":"342'8230","ondisk_log_start":"342'8230","created":319,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:01.985177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17891328,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14563,"num_write_kb":474752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7418,"num_bytes_recovered":457179136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":321,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.520954+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:45:52.520954+0000","last_peered":"2024-08-17T17:45:52.520954+0000","last_clean":"2024-08-17T17:45:52.520954+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:45:52.520954+0000","last_undegraded":"2024-08-17T17:45:52.520954+0000","last_fullsized":"2024-08-17T17:45:52.520954+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":160,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.518865+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:46:21.518865+0000","last_peered":"2024-08-17T17:46:21.518865+0000","last_clean":"2024-08-17T17:46:21.518865+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:46:21.518865+0000","last_undegraded":"2024-08-17T17:46:21.518865+0000","last_fullsized":"2024-08-17T17:46:21.518865+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":655,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.516413+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:46:21.516413+0000","last_peered":"2024-08-17T17:46:21.516413+0000","last_clean":"2024-08-17T17:46:21.516413+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:46:21.516413+0000","last_undegraded":"2024-08-17T17:46:21.516413+0000","last_fullsized":"2024-08-17T17:46:21.516413+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"408'14152","reported_seq":28764,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:22.797812+0000","last_change":"2024-08-17T17:35:00.441076+0000","last_active":"2024-08-17T17:46:22.797812+0000","last_peered":"2024-08-17T17:46:22.797812+0000","last_clean":"2024-08-17T17:46:22.797812+0000","last_became_active":"2024-08-17T17:35:00.426711+0000","last_became_peered":"2024-08-17T17:35:00.426711+0000","last_unstale":"2024-08-17T17:46:22.797812+0000","last_undegraded":"2024-08-17T17:46:22.797812+0000","last_fullsized":"2024-08-17T17:46:22.797812+0000","mapping_epoch":345,"log_start":"335'7872","ondisk_log_start":"335'7872","created":319,"last_epoch_clean":346,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6280,"log_dups_size":0,"ondisk_log_size":6280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:22.652043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16515072,"num_objects":252,"num_object_clones":0,"num_object_copies":756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14152,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7202,"num_bytes_recovered":424869888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":317,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.521299+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:45:52.521299+0000","last_peered":"2024-08-17T17:45:52.521299+0000","last_clean":"2024-08-17T17:45:52.521299+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:45:52.521299+0000","last_undegraded":"2024-08-17T17:45:52.521299+0000","last_fullsized":"2024-08-17T17:45:52.521299+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":409,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.493299+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:46:06.493299+0000","last_peered":"2024-08-17T17:46:06.493299+0000","last_clean":"2024-08-17T17:46:06.493299+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:46:06.493299+0000","last_undegraded":"2024-08-17T17:46:06.493299+0000","last_fullsized":"2024-08-17T17:46:06.493299+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":459,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.520398+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:45:52.520398+0000","last_peered":"2024-08-17T17:45:52.520398+0000","last_clean":"2024-08-17T17:45:52.520398+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:45:52.520398+0000","last_undegraded":"2024-08-17T17:45:52.520398+0000","last_fullsized":"2024-08-17T17:45:52.520398+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"408'14492","reported_seq":14712,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:46:22.941820+0000","last_change":"2024-08-17T17:33:40.874393+0000","last_active":"2024-08-17T17:46:22.941820+0000","last_peered":"2024-08-17T17:46:22.941820+0000","last_clean":"2024-08-17T17:33:19.275969+0000","last_became_active":"2024-08-17T17:33:21.671691+0000","last_became_peered":"2024-08-17T17:33:21.671691+0000","last_unstale":"2024-08-17T17:46:22.941820+0000","last_undegraded":"2024-08-17T17:33:21.322662+0000","last_fullsized":"2024-08-17T17:33:21.271932+0000","mapping_epoch":332,"log_start":"342'8187","ondisk_log_start":"342'8187","created":319,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19136512,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14492,"num_write_kb":473088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":292},{"shards":"0,3","objects":-6956}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":685,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519498+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:46:21.519498+0000","last_peered":"2024-08-17T17:46:21.519498+0000","last_clean":"2024-08-17T17:46:21.519498+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:46:21.519498+0000","last_undegraded":"2024-08-17T17:46:21.519498+0000","last_fullsized":"2024-08-17T17:46:21.519498+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":653,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.517473+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:46:21.517473+0000","last_peered":"2024-08-17T17:46:21.517473+0000","last_clean":"2024-08-17T17:46:21.517473+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:46:21.517473+0000","last_undegraded":"2024-08-17T17:46:21.517473+0000","last_fullsized":"2024-08-17T17:46:21.517473+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":671,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.518590+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:46:21.518590+0000","last_peered":"2024-08-17T17:46:21.518590+0000","last_clean":"2024-08-17T17:46:21.518590+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:46:21.518590+0000","last_undegraded":"2024-08-17T17:46:21.518590+0000","last_fullsized":"2024-08-17T17:46:21.518590+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"408'14275","reported_seq":28980,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:22.299843+0000","last_change":"2024-08-17T17:35:06.465299+0000","last_active":"2024-08-17T17:46:22.299843+0000","last_peered":"2024-08-17T17:46:22.299843+0000","last_clean":"2024-08-17T17:46:22.299843+0000","last_became_active":"2024-08-17T17:35:06.449794+0000","last_became_peered":"2024-08-17T17:35:06.449794+0000","last_unstale":"2024-08-17T17:46:22.299843+0000","last_undegraded":"2024-08-17T17:46:22.299843+0000","last_fullsized":"2024-08-17T17:46:22.299843+0000","mapping_epoch":347,"log_start":"335'7672","ondisk_log_start":"335'7672","created":319,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:31:35.240284+0000","last_clean_scrub_stamp":"2024-08-17T17:31:35.240284+0000","objects_scrubbed":0,"log_size":6603,"log_dups_size":0,"ondisk_log_size":6603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:09.465591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19595264,"num_objects":299,"num_object_clones":0,"num_object_copies":897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14275,"num_write_kb":466368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7287,"num_bytes_recovered":431357952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":342,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.493601+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:46:06.493601+0000","last_peered":"2024-08-17T17:46:06.493601+0000","last_clean":"2024-08-17T17:46:06.493601+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:46:06.493601+0000","last_undegraded":"2024-08-17T17:46:06.493601+0000","last_fullsized":"2024-08-17T17:46:06.493601+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":412,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.500451+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:46:06.500451+0000","last_peered":"2024-08-17T17:46:06.500451+0000","last_clean":"2024-08-17T17:46:06.500451+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:46:06.500451+0000","last_undegraded":"2024-08-17T17:46:06.500451+0000","last_fullsized":"2024-08-17T17:46:06.500451+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":361,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507036+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:45:52.507036+0000","last_peered":"2024-08-17T17:45:52.507036+0000","last_clean":"2024-08-17T17:45:52.507036+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:45:52.507036+0000","last_undegraded":"2024-08-17T17:45:52.507036+0000","last_fullsized":"2024-08-17T17:45:52.507036+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":361,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508387+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:45:52.508387+0000","last_peered":"2024-08-17T17:45:52.508387+0000","last_clean":"2024-08-17T17:45:52.508387+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:45:52.508387+0000","last_undegraded":"2024-08-17T17:45:52.508387+0000","last_fullsized":"2024-08-17T17:45:52.508387+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":404,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507793+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:45:52.507793+0000","last_peered":"2024-08-17T17:45:52.507793+0000","last_clean":"2024-08-17T17:45:52.507793+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:45:52.507793+0000","last_undegraded":"2024-08-17T17:45:52.507793+0000","last_fullsized":"2024-08-17T17:45:52.507793+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":660,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.516200+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:46:21.516200+0000","last_peered":"2024-08-17T17:46:21.516200+0000","last_clean":"2024-08-17T17:46:21.516200+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:46:21.516200+0000","last_undegraded":"2024-08-17T17:46:21.516200+0000","last_fullsized":"2024-08-17T17:46:21.516200+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":200,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.527460+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:46:21.527460+0000","last_peered":"2024-08-17T17:46:21.527460+0000","last_clean":"2024-08-17T17:46:21.527460+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:46:21.527460+0000","last_undegraded":"2024-08-17T17:46:21.527460+0000","last_fullsized":"2024-08-17T17:46:21.527460+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":311,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.521172+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:45:52.521172+0000","last_peered":"2024-08-17T17:45:52.521172+0000","last_clean":"2024-08-17T17:45:52.521172+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:45:52.521172+0000","last_undegraded":"2024-08-17T17:45:52.521172+0000","last_fullsized":"2024-08-17T17:45:52.521172+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":618,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.518372+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:46:21.518372+0000","last_peered":"2024-08-17T17:46:21.518372+0000","last_clean":"2024-08-17T17:46:21.518372+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:46:21.518372+0000","last_undegraded":"2024-08-17T17:46:21.518372+0000","last_fullsized":"2024-08-17T17:46:21.518372+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":455,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.521616+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:45:52.521616+0000","last_peered":"2024-08-17T17:45:52.521616+0000","last_clean":"2024-08-17T17:45:52.521616+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:45:52.521616+0000","last_undegraded":"2024-08-17T17:45:52.521616+0000","last_fullsized":"2024-08-17T17:45:52.521616+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":612,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.517933+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:46:21.517933+0000","last_peered":"2024-08-17T17:46:21.517933+0000","last_clean":"2024-08-17T17:46:21.517933+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:46:21.517933+0000","last_undegraded":"2024-08-17T17:46:21.517933+0000","last_fullsized":"2024-08-17T17:46:21.517933+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":379,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.500374+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:46:06.500374+0000","last_peered":"2024-08-17T17:46:06.500374+0000","last_clean":"2024-08-17T17:46:06.500374+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:46:06.500374+0000","last_undegraded":"2024-08-17T17:46:06.500374+0000","last_fullsized":"2024-08-17T17:46:06.500374+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":643,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.518688+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:46:21.518688+0000","last_peered":"2024-08-17T17:46:21.518688+0000","last_clean":"2024-08-17T17:46:21.518688+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:46:21.518688+0000","last_undegraded":"2024-08-17T17:46:21.518688+0000","last_fullsized":"2024-08-17T17:46:21.518688+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":360,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519198+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:46:21.519198+0000","last_peered":"2024-08-17T17:46:21.519198+0000","last_clean":"2024-08-17T17:46:21.519198+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:46:21.519198+0000","last_undegraded":"2024-08-17T17:46:21.519198+0000","last_fullsized":"2024-08-17T17:46:21.519198+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":651,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.518323+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:46:21.518323+0000","last_peered":"2024-08-17T17:46:21.518323+0000","last_clean":"2024-08-17T17:46:21.518323+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:46:21.518323+0000","last_undegraded":"2024-08-17T17:46:21.518323+0000","last_fullsized":"2024-08-17T17:46:21.518323+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":611,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519894+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:46:21.519894+0000","last_peered":"2024-08-17T17:46:21.519894+0000","last_clean":"2024-08-17T17:46:21.519894+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:46:21.519894+0000","last_undegraded":"2024-08-17T17:46:21.519894+0000","last_fullsized":"2024-08-17T17:46:21.519894+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":390,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507305+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:45:52.507305+0000","last_peered":"2024-08-17T17:45:52.507305+0000","last_clean":"2024-08-17T17:45:52.507305+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:45:52.507305+0000","last_undegraded":"2024-08-17T17:45:52.507305+0000","last_fullsized":"2024-08-17T17:45:52.507305+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":300,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519484+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:46:21.519484+0000","last_peered":"2024-08-17T17:46:21.519484+0000","last_clean":"2024-08-17T17:46:21.519484+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:46:21.519484+0000","last_undegraded":"2024-08-17T17:46:21.519484+0000","last_fullsized":"2024-08-17T17:46:21.519484+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":354,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.492572+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:46:06.492572+0000","last_peered":"2024-08-17T17:46:06.492572+0000","last_clean":"2024-08-17T17:46:06.492572+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:46:06.492572+0000","last_undegraded":"2024-08-17T17:46:06.492572+0000","last_fullsized":"2024-08-17T17:46:06.492572+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":398,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.508371+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:45:52.508371+0000","last_peered":"2024-08-17T17:45:52.508371+0000","last_clean":"2024-08-17T17:45:52.508371+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:45:52.508371+0000","last_undegraded":"2024-08-17T17:45:52.508371+0000","last_fullsized":"2024-08-17T17:45:52.508371+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":261,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507718+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:45:52.507718+0000","last_peered":"2024-08-17T17:45:52.507718+0000","last_clean":"2024-08-17T17:45:52.507718+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:45:52.507718+0000","last_undegraded":"2024-08-17T17:45:52.507718+0000","last_fullsized":"2024-08-17T17:45:52.507718+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":398,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507184+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:45:52.507184+0000","last_peered":"2024-08-17T17:45:52.507184+0000","last_clean":"2024-08-17T17:45:52.507184+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:45:52.507184+0000","last_undegraded":"2024-08-17T17:45:52.507184+0000","last_fullsized":"2024-08-17T17:45:52.507184+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":261,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507458+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:45:52.507458+0000","last_peered":"2024-08-17T17:45:52.507458+0000","last_clean":"2024-08-17T17:45:52.507458+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:45:52.507458+0000","last_undegraded":"2024-08-17T17:45:52.507458+0000","last_fullsized":"2024-08-17T17:45:52.507458+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":663,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519627+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:46:21.519627+0000","last_peered":"2024-08-17T17:46:21.519627+0000","last_clean":"2024-08-17T17:46:21.519627+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:46:21.519627+0000","last_undegraded":"2024-08-17T17:46:21.519627+0000","last_fullsized":"2024-08-17T17:46:21.519627+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":637,"reported_epoch":408,"state":"active+clean","last_fresh":"2024-08-17T17:46:21.519297+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:46:21.519297+0000","last_peered":"2024-08-17T17:46:21.519297+0000","last_clean":"2024-08-17T17:46:21.519297+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:46:21.519297+0000","last_undegraded":"2024-08-17T17:46:21.519297+0000","last_fullsized":"2024-08-17T17:46:21.519297+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":361,"reported_epoch":406,"state":"active+clean","last_fresh":"2024-08-17T17:45:52.507035+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:45:52.507035+0000","last_peered":"2024-08-17T17:45:52.507035+0000","last_clean":"2024-08-17T17:45:52.507035+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:45:52.507035+0000","last_undegraded":"2024-08-17T17:45:52.507035+0000","last_fullsized":"2024-08-17T17:45:52.507035+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":357,"reported_epoch":407,"state":"active+clean","last_fresh":"2024-08-17T17:46:06.492921+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:46:06.492921+0000","last_peered":"2024-08-17T17:46:06.492921+0000","last_clean":"2024-08-17T17:46:06.492921+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:46:06.492921+0000","last_undegraded":"2024-08-17T17:46:06.492921+0000","last_fullsized":"2024-08-17T17:46:06.492921+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":319094808,"num_objects":4870,"num_object_clones":0,"num_object_copies":14610,"num_objects_missing_on_primary":1026,"num_objects_missing":1026,"num_objects_degraded":44970,"num_objects_misplaced":2832,"num_objects_unfound":0,"num_objects_dirty":4870,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":231520,"num_write_kb":7564417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41742,"num_bytes_recovered":2610036736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5014499328,"data_stored":5014487112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104739,"ondisk_log_size":104739,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170159,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463529,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1337628,"kb_used_data":143056,"kb_used_omap":1,"kb_used_meta":1194558,"kb_avail":92407524,"statfs":{"total":95995035648,"available":94625304576,"internally_reserved":0,"allocated":146489344,"data_stored":144209561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1223227736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492203,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":452524,"kb_used_data":53028,"kb_used_omap":7,"kb_used_meta":399480,"kb_avail":93292628,"statfs":{"total":95995035648,"available":95531651072,"internally_reserved":0,"allocated":54300672,"data_stored":52050982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":409067853},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986794,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1604348,"kb_used_data":95668,"kb_used_omap":1,"kb_used_meta":1508670,"kb_avail":92140804,"statfs":{"total":95995035648,"available":94352183296,"internally_reserved":0,"allocated":97964032,"data_stored":95702982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1544878424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,3,2,2,4],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225623,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152380,"kb_used_data":146420,"kb_used_omap":4,"kb_used_meta":1005947,"kb_avail":92592772,"statfs":{"total":95995035648,"available":94814998528,"internally_reserved":0,"allocated":149934080,"data_stored":147676895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1030090176},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075947,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533841,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212480,"kb_used_data":112184,"kb_used_omap":5,"kb_used_meta":1100282,"kb_avail":92532672,"statfs":{"total":95995035648,"available":94753456128,"internally_reserved":0,"allocated":114876416,"data_stored":112613909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1126689008},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089817,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179912,"kb_used_data":146944,"kb_used_omap":5,"kb_used_meta":1032954,"kb_avail":92565240,"statfs":{"total":95995035648,"available":94786805760,"internally_reserved":0,"allocated":150470656,"data_stored":148198725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1057744955},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":144179200,"data_stored":144179200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":108728320,"data_stored":108724248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3339259904,"data_stored":3339255832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":142082048,"data_stored":142082048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":87556096,"data_stored":87556096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":48168960,"data_stored":48168960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":133431296,"data_stored":133431296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1011093504,"data_stored":1011089432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:46:27.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:27.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:28.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:46:28.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:28.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:28.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:28.276 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:28.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:28.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:28.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:28.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:28.501 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:28.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:28.582 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:28.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:28.612 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:46:28.613 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:46:28.613 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:28.651 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:46:28.651 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:46:28.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:28.686 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:46:28.687 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:46:28.719 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:46:28.719 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:46:28.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:28.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:28.753 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:46:28.754 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:46:28.778 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:46:28.778 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:46:28.807 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:46:28.807 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:46:28.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:28.839 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:46:28.839 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:46:28.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:28.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:28.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:28.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:29.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:29.132 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:29.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:29.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:29.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:29.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:29.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:29.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:29.642 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:29.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:29.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:29.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:29 smithi152 ceph-mon[93456]: pgmap v1625: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 156 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 44531/7545 objects degraded (590.205%); 2420/7545 objects misplaced (32.074%); 6.1 MiB/s, 158 objects/s recovering 2024-08-17T17:46:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3391271278' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:29.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:29 smithi142 ceph-mon[103614]: pgmap v1625: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 156 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 44531/7545 objects degraded (590.205%); 2420/7545 objects misplaced (32.074%); 6.1 MiB/s, 158 objects/s recovering 2024-08-17T17:46:29.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3391271278' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:29.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:29 smithi142 ceph-mon[106729]: pgmap v1625: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 156 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 44531/7545 objects degraded (590.205%); 2420/7545 objects misplaced (32.074%); 6.1 MiB/s, 158 objects/s recovering 2024-08-17T17:46:29.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3391271278' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:46:29.916 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:29.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:29.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:29.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:30.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:30.135 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:30.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:30.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:30.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:30.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:30.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:30.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:30.468 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:46:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:46:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:30.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:30.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:30.640 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 118194 objects 2024-08-17T17:46:30.640 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :793.636 2024-08-17T17:46:30.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:30.666 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:30.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:30.790 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:30.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:30.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:30.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:30.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:30.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:46:31.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:31.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:31.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:31.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:31.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:31.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:31.309 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:31.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:46:31.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:31.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:31.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:31.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:31.560 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:31 smithi142 ceph-mon[106729]: pgmap v1626: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 99 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 533 op/s; 43598/4821 objects degraded (904.335%); 2100/4821 objects misplaced (43.559%); 6.6 MiB/s, 166 objects/s recovering 2024-08-17T17:46:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 43598/4821 objects degraded (904.335%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:31 smithi142 ceph-mon[103614]: pgmap v1626: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 99 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 533 op/s; 43598/4821 objects degraded (904.335%); 2100/4821 objects misplaced (43.559%); 6.6 MiB/s, 166 objects/s recovering 2024-08-17T17:46:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 43598/4821 objects degraded (904.335%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:31.625 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2024-08-17T17:46:31.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2024-08-17T17:46:31.642 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:31.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:31.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:31.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:31 smithi152 ceph-mon[93456]: pgmap v1626: 89 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 99 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 533 op/s; 43598/4821 objects degraded (904.335%); 2100/4821 objects misplaced (43.559%); 6.6 MiB/s, 166 objects/s recovering 2024-08-17T17:46:31.766 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 43598/4821 objects degraded (904.335%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:31.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:31.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:31.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:31.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:31.942 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:31.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:31.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:32.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:32.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:32.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:32.084 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:46:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:46:31] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.51.0" 2024-08-17T17:46:32.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:32.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:32.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:32.312 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:32.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:32.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:32.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:32.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:32.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:32.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:32.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:32.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:32.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:32.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:32.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:32.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:32.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:33.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:33.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:33.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:33.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:33.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:33.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:33.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:33.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:33.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:33.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:33.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:33.602 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:33.602 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:33.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:33.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:33.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:33.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:33 smithi152 ceph-mon[93456]: pgmap v1627: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 31 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 43445/1518 objects degraded (2861.989%); 1699/1518 objects misplaced (111.924%); 4.2 MiB/s, 130 objects/s recovering 2024-08-17T17:46:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:33 smithi142 ceph-mon[106729]: pgmap v1627: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 31 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 43445/1518 objects degraded (2861.989%); 1699/1518 objects misplaced (111.924%); 4.2 MiB/s, 130 objects/s recovering 2024-08-17T17:46:33.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:33 smithi142 ceph-mon[103614]: pgmap v1627: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 31 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 43445/1518 objects degraded (2861.989%); 1699/1518 objects misplaced (111.924%); 4.2 MiB/s, 130 objects/s recovering 2024-08-17T17:46:33.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:33.925 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:33.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:34.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:34.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:34.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:34.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:34.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:34.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:34.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:34.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:34.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:46:34.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:34.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:34.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:34.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:34.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:34.624 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:34.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:34.665 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:34.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:34.791 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:34.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:34.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:34.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:34.994 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:35.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:35.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:35.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:35.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:35.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:35.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:35.290 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:35.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:35.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:35.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:35.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:35.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:35 smithi142 ceph-mon[103614]: pgmap v1628: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.3 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 466 op/s; 43415/255 objects degraded (17025.490%); 1478/255 objects misplaced (579.608%); 4.4 MiB/s, 141 objects/s recovering 2024-08-17T17:46:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:35 smithi142 ceph-mon[106729]: pgmap v1628: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.3 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 466 op/s; 43415/255 objects degraded (17025.490%); 1478/255 objects misplaced (579.608%); 4.4 MiB/s, 141 objects/s recovering 2024-08-17T17:46:35.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:35.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:35.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.785 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:35.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:35 smithi152 ceph-mon[93456]: pgmap v1628: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.3 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 466 op/s; 43415/255 objects degraded (17025.490%); 1478/255 objects misplaced (579.608%); 4.4 MiB/s, 141 objects/s recovering 2024-08-17T17:46:35.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:35.907 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:35.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:36.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.034 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:36.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:36.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:36.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:36.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:36.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:36.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:36.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:36.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:36.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:36.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:36.565 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:36.565 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:36.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:36 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 43415/255 objects degraded (17025.490%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:36.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:36.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:36.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:36.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:36.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:36 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 43415/255 objects degraded (17025.490%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:36 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 43415/255 objects degraded (17025.490%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:36.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:36.897 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:36.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:36.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:37.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:37.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:37.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:37.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:37.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:37.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:37.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:46:37.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:37.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:37.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:37.516 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:37.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:37.631 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:37.639 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:37.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:37.754 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:37 smithi152 ceph-mon[93456]: pgmap v1629: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 43425/78 objects degraded (55673.077%); 1456/78 objects misplaced (1866.667%); 3.7 MiB/s, 154 objects/s recovering 2024-08-17T17:46:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:37.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:37.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:37.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:37 smithi142 ceph-mon[106729]: pgmap v1629: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 43425/78 objects degraded (55673.077%); 1456/78 objects misplaced (1866.667%); 3.7 MiB/s, 154 objects/s recovering 2024-08-17T17:46:37.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:37 smithi142 ceph-mon[103614]: pgmap v1629: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 43425/78 objects degraded (55673.077%); 1456/78 objects misplaced (1866.667%); 3.7 MiB/s, 154 objects/s recovering 2024-08-17T17:46:37.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:37.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:37.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:38.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:38.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:38.133 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:38.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:38.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:38.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:38.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:38.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:38.481 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:38.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:38.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:38.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:38.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:38.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:38.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:38.874 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:38.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:38.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:38.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:38.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:38.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:39.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:39.120 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:39.222 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:39.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:39.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:39.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:39.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:39.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:39.486 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:39.486 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:39.496 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:39.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:39.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:39.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:39.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:39.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:39.791 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:39 smithi152 ceph-mon[93456]: pgmap v1630: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 2.8 MiB/s, 124 objects/s recovering 2024-08-17T17:46:39.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:39.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:39.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:39 smithi142 ceph-mon[106729]: pgmap v1630: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 2.8 MiB/s, 124 objects/s recovering 2024-08-17T17:46:39.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:39 smithi142 ceph-mon[103614]: pgmap v1630: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 2.8 MiB/s, 124 objects/s recovering 2024-08-17T17:46:39.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:39.990 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:39.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:40.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:40.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:40.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:46:40.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:40.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:40.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:40.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:40.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:40.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:40.482 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:40.485 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:46:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:46:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:40.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:40.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:40.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:40.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:40.837 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:40.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:40.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:40.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:40.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:40.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:40.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:41.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:41.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:41.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:41.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:41.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:41.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:41.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:41.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:41.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:41.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:41.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:41.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.728 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:41.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:41.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-mon[103614]: pgmap v1631: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 2.7 MiB/s, 151 objects/s recovering 2024-08-17T17:46:41.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 43053/78 objects degraded (55196.154%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-mon[106729]: pgmap v1631: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 2.7 MiB/s, 151 objects/s recovering 2024-08-17T17:46:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 43053/78 objects degraded (55196.154%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:41.802 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:41 smithi152 ceph-mon[93456]: pgmap v1631: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 2.7 MiB/s, 151 objects/s recovering 2024-08-17T17:46:41.802 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 43053/78 objects degraded (55196.154%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:41.802 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:41.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:41.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:41.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:41.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:41.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:42.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:42.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:46:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:46:41] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-08-17T17:46:42.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:42.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:42.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:42.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:42.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:42.432 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:42.432 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:42.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:42.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:42.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.595 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:42.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:42.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:42.717 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:42.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:42.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:42.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:42.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:42.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:42.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:43.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:43.088 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:43.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:43.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:43.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:46:43.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:43.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:43.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:43.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:43.497 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:43.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:43.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:43.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:43.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:43.771 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:43 smithi152 ceph-mon[93456]: pgmap v1632: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 1.5 MiB/s, 131 objects/s recovering 2024-08-17T17:46:43.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:43.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:43 smithi142 ceph-mon[106729]: pgmap v1632: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 1.5 MiB/s, 131 objects/s recovering 2024-08-17T17:46:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:43 smithi142 ceph-mon[103614]: pgmap v1632: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 43053/78 objects degraded (55196.154%); 1456/78 objects misplaced (1866.667%); 1.5 MiB/s, 131 objects/s recovering 2024-08-17T17:46:43.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:43.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:43.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:44.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:44.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:44.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:44.207 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:44.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:44.309 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:44.331 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:44.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:44.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:44.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:44.577 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:44.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:44.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:44.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:44.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:44.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:44.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:44.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:44.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:45.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:45.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:45.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:45.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:45.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:45.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:45.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:45.363 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:45.363 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:45.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:45.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:45.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:45.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:45.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:45.667 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:45.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:45.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:45.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:45 smithi152 ceph-mon[93456]: pgmap v1633: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 39 op/s; 42669/78 objects degraded (54703.846%); 1456/78 objects misplaced (1866.667%); 672 KiB/s, 140 objects/s recovering 2024-08-17T17:46:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:45 smithi142 ceph-mon[106729]: pgmap v1633: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 39 op/s; 42669/78 objects degraded (54703.846%); 1456/78 objects misplaced (1866.667%); 672 KiB/s, 140 objects/s recovering 2024-08-17T17:46:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:45 smithi142 ceph-mon[103614]: pgmap v1633: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 39 op/s; 42669/78 objects degraded (54703.846%); 1456/78 objects misplaced (1866.667%); 672 KiB/s, 140 objects/s recovering 2024-08-17T17:46:45.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:45.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:45.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:46.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:46.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:46.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:46:46.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:46.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:46.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:46.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:46.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:46.421 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:46.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:46.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:46.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:46.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:46.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:46.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:46.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:46.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 42669/78 objects degraded (54703.846%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.838 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:46.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 42669/78 objects degraded (54703.846%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 42669/78 objects degraded (54703.846%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:46.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:46.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:46.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:46.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:47.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:47.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:47.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:47.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:47.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:47.311 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:47.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:47.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:47.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:47.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:47.583 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:47.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:47 smithi152 ceph-mon[93456]: pgmap v1634: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 42342/78 objects degraded (54284.615%); 1456/78 objects misplaced (1866.667%); 64 KiB/s, 146 objects/s recovering 2024-08-17T17:46:47.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:47.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:47.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:47.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:47.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:47 smithi142 ceph-mon[106729]: pgmap v1634: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 42342/78 objects degraded (54284.615%); 1456/78 objects misplaced (1866.667%); 64 KiB/s, 146 objects/s recovering 2024-08-17T17:46:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:47 smithi142 ceph-mon[103614]: pgmap v1634: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 42342/78 objects degraded (54284.615%); 1456/78 objects misplaced (1866.667%); 64 KiB/s, 146 objects/s recovering 2024-08-17T17:46:47.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:47.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:47.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:48.056 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:48.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:48.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:48.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:48.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:48.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:48.426 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:48.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:48.508 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:48.509 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:48.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:48.572 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:48.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:48.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:48.819 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:48.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:48.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:48 smithi152 ceph-mon[93456]: pgmap v1635: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 41934/78 objects degraded (53761.538%); 1456/78 objects misplaced (1866.667%); 0 B/s, 155 objects/s recovering 2024-08-17T17:46:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:48 smithi142 ceph-mon[106729]: pgmap v1635: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 41934/78 objects degraded (53761.538%); 1456/78 objects misplaced (1866.667%); 0 B/s, 155 objects/s recovering 2024-08-17T17:46:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:48 smithi142 ceph-mon[103614]: pgmap v1635: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 41934/78 objects degraded (53761.538%); 1456/78 objects misplaced (1866.667%); 0 B/s, 155 objects/s recovering 2024-08-17T17:46:48.922 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:48.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:48.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:49.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:49.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:49.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:49.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:49.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:49.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:46:49.292 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:49.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:49.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:49.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:49.509 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:49.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:49.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:49.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:49.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:49.686 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:49.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:49.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:49.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:49.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:49.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:50.039 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:50.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:50.100 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:50.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:50.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:50.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:50.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:50.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:50.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:50.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:50.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:50.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:50.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:46:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:46:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:50.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:50.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:50.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:50.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:50.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:50.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:50.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:50.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:51.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:51.053 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:51.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:51.177 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:51.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:51.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:51.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:51.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:51.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:51.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:51.531 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:51.548 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:51.548 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:51.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:51 smithi152 ceph-mon[93456]: pgmap v1636: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 41559/78 objects degraded (53280.769%); 1456/78 objects misplaced (1866.667%); 0 B/s, 155 objects/s recovering 2024-08-17T17:46:51.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 41559/78 objects degraded (53280.769%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:51.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:51.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:51.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:51 smithi142 ceph-mon[106729]: pgmap v1636: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 41559/78 objects degraded (53280.769%); 1456/78 objects misplaced (1866.667%); 0 B/s, 155 objects/s recovering 2024-08-17T17:46:51.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 41559/78 objects degraded (53280.769%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:51.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:51 smithi142 ceph-mon[103614]: pgmap v1636: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 41559/78 objects degraded (53280.769%); 1456/78 objects misplaced (1866.667%); 0 B/s, 155 objects/s recovering 2024-08-17T17:46:51.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 41559/78 objects degraded (53280.769%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:51.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:51.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:51.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:51.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:51.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:51.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:52.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:52.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:52.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:46:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:46:51] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-08-17T17:46:52.137 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:46:52.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:52.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:52.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:46:52.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:52.301 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:52.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:52.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:52.516 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:52.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:52.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:52.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:52.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:52.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:52.801 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:52.903 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:52.924 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:52.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:53.026 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:53.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:53.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:53.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:53.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:53.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:53.399 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:53.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:53.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:53.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:53.644 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:53.669 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:53.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:53.792 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:53 smithi152 ceph-mon[93456]: pgmap v1637: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 41559/78 objects degraded (53280.769%); 1456/78 objects misplaced (1866.667%); 0 B/s, 126 objects/s recovering 2024-08-17T17:46:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:53 smithi142 ceph-mon[106729]: pgmap v1637: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 41559/78 objects degraded (53280.769%); 1456/78 objects misplaced (1866.667%); 0 B/s, 126 objects/s recovering 2024-08-17T17:46:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:53 smithi142 ceph-mon[103614]: pgmap v1637: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 41559/78 objects degraded (53280.769%); 1456/78 objects misplaced (1866.667%); 0 B/s, 126 objects/s recovering 2024-08-17T17:46:53.894 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:53.916 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:53.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:53.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:54.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:54.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:54.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:54.164 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:54.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:54.288 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:54.390 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:54.411 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:54.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:54.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:54.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:54.659 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:54.676 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:54.677 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:54.761 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:54.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:54.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:54.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:54.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:54.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:55.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:55.032 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:55.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:55.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:55.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:55.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:55.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:55.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:46:55.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:55.481 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:55 smithi152 ceph-mon[93456]: pgmap v1638: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 41150/78 objects degraded (52756.410%); 1456/78 objects misplaced (1866.667%); 0 B/s, 160 objects/s recovering 2024-08-17T17:46:55.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:55 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:46:55.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:55.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:55.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:55.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:55.729 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:55.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:55.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:55 smithi142 ceph-mon[106729]: pgmap v1638: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 41150/78 objects degraded (52756.410%); 1456/78 objects misplaced (1866.667%); 0 B/s, 160 objects/s recovering 2024-08-17T17:46:55.758 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:55 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:46:55.758 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:55 smithi142 ceph-mon[103614]: pgmap v1638: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 41150/78 objects degraded (52756.410%); 1456/78 objects misplaced (1866.667%); 0 B/s, 160 objects/s recovering 2024-08-17T17:46:55.758 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:55 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:46:55.758 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:46:55.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:55.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:55.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:55.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:55.978 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:55.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:56.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:56.102 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:56.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:56.225 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:56.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.350 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:56.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:56.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:56.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:56.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 41150/78 objects degraded (52756.410%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:56.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:56.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:56.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:56.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:56.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:56.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:56.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:56.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 41150/78 objects degraded (52756.410%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 41150/78 objects degraded (52756.410%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:46:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:56.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:56.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:56.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:57.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:57.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:57.317 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:57.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:57.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:57.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:57.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:57.667 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:46:57.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:57.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:57.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:57 smithi152 ceph-mon[93456]: pgmap v1639: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 40761/78 objects degraded (52257.692%); 1456/78 objects misplaced (1866.667%); 0 B/s, 161 objects/s recovering 2024-08-17T17:46:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2401699585' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:57 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:57.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:57 smithi142 ceph-mon[106729]: pgmap v1639: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 40761/78 objects degraded (52257.692%); 1456/78 objects misplaced (1866.667%); 0 B/s, 161 objects/s recovering 2024-08-17T17:46:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2401699585' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:57 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:57 smithi142 ceph-mon[103614]: pgmap v1639: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 40761/78 objects degraded (52257.692%); 1456/78 objects misplaced (1866.667%); 0 B/s, 161 objects/s recovering 2024-08-17T17:46:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2401699585' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:57 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:46:57.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:57.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:57.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:58.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:58.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:58.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:58.189 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:58.211 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:58.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:58.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:58.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:58.436 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:58.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:58.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:46:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.583 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:58.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:46:58.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:46:58.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:46:58.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:46:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:58 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:58 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:46:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:58 smithi152 ceph-mon[93456]: osdmap e409: 8 total, 8 up, 6 in 2024-08-17T17:46:58.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:46:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:58 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:58 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:46:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:58 smithi142 ceph-mon[103614]: osdmap e409: 8 total, 8 up, 6 in 2024-08-17T17:46:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:58 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:46:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:58 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:46:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:58 smithi142 ceph-mon[106729]: osdmap e409: 8 total, 8 up, 6 in 2024-08-17T17:46:58.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:58.955 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:58.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:59.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:46:59.078 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:46:59.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:59.202 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:59.304 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:46:59.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:46:59.413 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:46:59.413 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:46:59.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:59.450 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:59.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:46:59.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:46:59 smithi152 ceph-mon[93456]: pgmap v1641: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 40314/78 objects degraded (51684.615%); 1456/78 objects misplaced (1866.667%); 0 B/s, 161 objects/s recovering 2024-08-17T17:46:59.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:46:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:46:59.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:46:59.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:46:59.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:59.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:46:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:46:59 smithi142 ceph-mon[106729]: pgmap v1641: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 40314/78 objects degraded (51684.615%); 1456/78 objects misplaced (1866.667%); 0 B/s, 161 objects/s recovering 2024-08-17T17:46:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:46:59 smithi142 ceph-mon[103614]: pgmap v1641: 89 pgs: 2 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 40314/78 objects degraded (51684.615%); 1456/78 objects misplaced (1866.667%); 0 B/s, 161 objects/s recovering 2024-08-17T17:46:59.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:46:59.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:46:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:46:59.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:00.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:00.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:00.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:47:00.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:00.193 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:00.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:00.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:00.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:00.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:00.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:47:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:47:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:00.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:00.569 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:00.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:00.692 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:00.714 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:00 smithi152 ceph-mon[93456]: pgmap v1642: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39892/78 objects degraded (51143.590%); 1456/78 objects misplaced (1866.667%); 0 B/s, 166 objects/s recovering 2024-08-17T17:47:00.794 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:00.808 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:00 smithi142 ceph-mon[106729]: pgmap v1642: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39892/78 objects degraded (51143.590%); 1456/78 objects misplaced (1866.667%); 0 B/s, 166 objects/s recovering 2024-08-17T17:47:00.808 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:00 smithi142 ceph-mon[103614]: pgmap v1642: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39892/78 objects degraded (51143.590%); 1456/78 objects misplaced (1866.667%); 0 B/s, 166 objects/s recovering 2024-08-17T17:47:00.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:00.920 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:00.942 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:00.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:00.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:01.043 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:01.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:01.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:01.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:01.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:01.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:01.419 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:01.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:01.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:01.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:01.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:01.667 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:01.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:01.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:01.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 39892/78 objects degraded (51143.590%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 39892/78 objects degraded (51143.590%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:01.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:01.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 39892/78 objects degraded (51143.590%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:01.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:01.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:01.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:02.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:02.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:47:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:47:01] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:47:02.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:02.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:02.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:02.311 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:02.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:02.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:02.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:02.559 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:02.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:02.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:02.784 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:02.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:02.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:02 smithi152 ceph-mon[93456]: pgmap v1643: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39892/78 objects degraded (51143.590%); 1456/78 objects misplaced (1866.667%); 0 B/s, 166 objects/s recovering 2024-08-17T17:47:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:02 smithi142 ceph-mon[106729]: pgmap v1643: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39892/78 objects degraded (51143.590%); 1456/78 objects misplaced (1866.667%); 0 B/s, 166 objects/s recovering 2024-08-17T17:47:02.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:02 smithi142 ceph-mon[103614]: pgmap v1643: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39892/78 objects degraded (51143.590%); 1456/78 objects misplaced (1866.667%); 0 B/s, 166 objects/s recovering 2024-08-17T17:47:02.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:02.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:02.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:03.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:03.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:03.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:03.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:03.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:03.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:03.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:03.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:03.434 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:03.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:03.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:03.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:03.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:03.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:03.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:03.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:03.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:03.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:03.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:03.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:03.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:04.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:04.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:04.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:03 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:03 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:04.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:03 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:04.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:03 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:03 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:03 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:04.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:04.175 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:04.277 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:04.298 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:04.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:04.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:04.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:04.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:04.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:04.670 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:04.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:04.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:04.793 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:04 smithi142 ceph-mon[106729]: pgmap v1644: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39416/78 objects degraded (50533.333%); 1456/78 objects misplaced (1866.667%); 0 B/s, 173 objects/s recovering 2024-08-17T17:47:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:04 smithi142 ceph-mon[103614]: pgmap v1644: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39416/78 objects degraded (50533.333%); 1456/78 objects misplaced (1866.667%); 0 B/s, 173 objects/s recovering 2024-08-17T17:47:04.896 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:04.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:04.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:05.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:05.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:04 smithi152 ceph-mon[93456]: pgmap v1644: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 39416/78 objects degraded (50533.333%); 1456/78 objects misplaced (1866.667%); 0 B/s, 173 objects/s recovering 2024-08-17T17:47:05.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:05.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:05.267 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:05.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:05.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:05.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:05.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:05.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:05.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:05.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:05.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:05.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:05.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:05.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:05.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:05.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:06.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:06.029 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 39416/78 objects degraded (50533.333%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:06.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 39416/78 objects degraded (50533.333%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:06.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 39416/78 objects degraded (50533.333%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:06.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:06.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:06.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:06.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:06.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:06.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:06.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:06.528 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:06.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:06.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:06.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:06.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:06.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:06.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:06.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:07.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:07.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:07.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:06 smithi142 ceph-mon[106729]: pgmap v1645: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 38965/78 objects degraded (49955.128%); 1456/78 objects misplaced (1866.667%); 0 B/s, 179 objects/s recovering 2024-08-17T17:47:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:06 smithi142 ceph-mon[103614]: pgmap v1645: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 38965/78 objects degraded (49955.128%); 1456/78 objects misplaced (1866.667%); 0 B/s, 179 objects/s recovering 2024-08-17T17:47:07.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:07.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:07.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:07.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:07.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:07.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:06 smithi152 ceph-mon[93456]: pgmap v1645: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 38965/78 objects degraded (49955.128%); 1456/78 objects misplaced (1866.667%); 0 B/s, 179 objects/s recovering 2024-08-17T17:47:07.375 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:07.398 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:07.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:07.522 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:07.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:07.624 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:07.645 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:07.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:07.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:07.769 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:07.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:07.891 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:07.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:07.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:08.016 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:08.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:08.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:08.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:08.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:08.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:08.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:08.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:08.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:08.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:08.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:08.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:08.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:08.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:08.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:08.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:08.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:08.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:08.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:08.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:08.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:09.010 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:09.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:09.132 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:09.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:09.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:09.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:09.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:09.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:09.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:09.606 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:09.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:09.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:09.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:09 smithi152 ceph-mon[93456]: pgmap v1646: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 38425/78 objects degraded (49262.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 213 objects/s recovering 2024-08-17T17:47:09.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:09 smithi142 ceph-mon[106729]: pgmap v1646: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 38425/78 objects degraded (49262.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 213 objects/s recovering 2024-08-17T17:47:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:09 smithi142 ceph-mon[103614]: pgmap v1646: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 38425/78 objects degraded (49262.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 213 objects/s recovering 2024-08-17T17:47:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:09.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:09.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:09.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:10.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:10.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:10.125 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:10.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:10.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:10.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:10.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:10.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:10.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:10.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:10.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:47:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:47:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:10.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:10.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:10.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:10.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:10.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:10.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:10.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:10.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:10.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:10.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:10.997 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:10.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:11.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:11.120 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:11.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:11.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:11.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:11.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:11.368 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:11.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:11.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:11.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:11.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:11.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:11.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:11.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:11.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:11.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[103614]: pgmap v1647: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 37918/78 objects degraded (48612.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 199 objects/s recovering 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 37918/78 objects degraded (48612.821%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[106729]: pgmap v1647: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 37918/78 objects degraded (48612.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 199 objects/s recovering 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 37918/78 objects degraded (48612.821%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:11 smithi152 ceph-mon[93456]: pgmap v1647: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 37918/78 objects degraded (48612.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 199 objects/s recovering 2024-08-17T17:47:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 37918/78 objects degraded (48612.821%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:47:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:11.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:11.861 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:11.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:11.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:11.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:12.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:12.109 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:47:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:47:11] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:47:12.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:12.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:12.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:12.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:12.459 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:12.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:12.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:12.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:12.705 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_7' removed 2024-08-17T17:47:12.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.730 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:12.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3558391544' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:47:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:12 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:47:12.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:12.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:12.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3558391544' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:47:12.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:12 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:47:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3558391544' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:47:12.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:12 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:47:12.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:12.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:12.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:13.078 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:13.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:13.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:13.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:13.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:13.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:13.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:13.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:13.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:13.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:13.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:13.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:13.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:13 smithi152 ceph-mon[93456]: pgmap v1648: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 37918/78 objects degraded (48612.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 164 objects/s recovering 2024-08-17T17:47:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:13 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:47:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:13 smithi152 ceph-mon[93456]: osdmap e410: 8 total, 8 up, 6 in 2024-08-17T17:47:13.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:13.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:13 smithi142 ceph-mon[106729]: pgmap v1648: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 37918/78 objects degraded (48612.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 164 objects/s recovering 2024-08-17T17:47:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:13 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:47:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:13 smithi142 ceph-mon[106729]: osdmap e410: 8 total, 8 up, 6 in 2024-08-17T17:47:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:13 smithi142 ceph-mon[103614]: pgmap v1648: 89 pgs: 2 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 37918/78 objects degraded (48612.821%); 1456/78 objects misplaced (1866.667%); 0 B/s, 164 objects/s recovering 2024-08-17T17:47:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:13 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:47:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:13 smithi142 ceph-mon[103614]: osdmap e410: 8 total, 8 up, 6 in 2024-08-17T17:47:13.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:13.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:13.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:14.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:14.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:14.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:14.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:14.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:14.342 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:14.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:14.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:14.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:14.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:14.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.592 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:14.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:14.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:14.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:14.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:14.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:14 smithi152 ceph-mon[93456]: osdmap e411: 8 total, 8 up, 6 in 2024-08-17T17:47:14.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:14 smithi142 ceph-mon[106729]: osdmap e411: 8 total, 8 up, 6 in 2024-08-17T17:47:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:14 smithi142 ceph-mon[103614]: osdmap e411: 8 total, 8 up, 6 in 2024-08-17T17:47:14.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:14.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:14.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:15.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:15.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:15.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:15.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:15.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:15.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:15.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:15.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:15.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:15.607 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:15.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:15.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:15.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:15.785 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:15 smithi142 ceph-mon[106729]: pgmap v1651: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 2/78 objects degraded (2.564%) 2024-08-17T17:47:15.785 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:15 smithi142 ceph-mon[106729]: osdmap e412: 8 total, 8 up, 6 in 2024-08-17T17:47:15.785 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:15 smithi142 ceph-mon[103614]: pgmap v1651: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 2/78 objects degraded (2.564%) 2024-08-17T17:47:15.785 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:15 smithi142 ceph-mon[103614]: osdmap e412: 8 total, 8 up, 6 in 2024-08-17T17:47:15.802 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:15 smithi152 ceph-mon[93456]: pgmap v1651: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 2/78 objects degraded (2.564%) 2024-08-17T17:47:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:15 smithi152 ceph-mon[93456]: osdmap e412: 8 total, 8 up, 6 in 2024-08-17T17:47:15.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:15.859 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:15.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:15.986 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:15.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:15.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:16.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:16.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:16.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:16.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:16.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:16.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:16.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:16.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:16.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:16.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:16.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:16.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:16.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:47:16.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:16.861 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:16.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:16.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:47:16.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:47:16.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:16.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:16.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:17.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:17.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:17.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:17.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:17.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:17.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:17.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:17.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:17.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:17.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:17.587 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:17.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:17.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:17.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:17.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:17 smithi152 ceph-mon[93456]: pgmap v1653: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:17.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:17 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded, 1 pg undersized) 2024-08-17T17:47:17.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:17.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:17 smithi142 ceph-mon[106729]: pgmap v1653: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:17 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded, 1 pg undersized) 2024-08-17T17:47:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:17 smithi142 ceph-mon[103614]: pgmap v1653: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:17 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded, 1 pg undersized) 2024-08-17T17:47:17.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:17.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:17.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:18.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:18.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:18.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:18.237 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:18.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:18.364 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:18.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:18.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:18.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:18.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:18.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:18.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:18.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:18.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:18.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:18.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:18.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:18.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:18.998 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:19.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:19.126 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:19.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:19.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:19.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:19.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:19.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:19.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:19.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:19.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:19.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:19.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:19.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:19.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:19 smithi152 ceph-mon[93456]: pgmap v1654: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:19.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:19 smithi142 ceph-mon[106729]: pgmap v1654: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:19 smithi142 ceph-mon[103614]: pgmap v1654: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:19.892 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:19.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:19.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:20.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:20.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:20.147 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:20.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:20.249 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:20.275 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:20.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:20.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:20.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:20.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:47:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:47:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:20.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:20.662 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:20.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:20.790 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:20.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:20.917 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:20.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:21.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:21.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:21.142 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:47:21.143 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:47:21.143 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2024-08-17T17:47:21.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_8 16 2024-08-17T17:47:21.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:21.165 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:21.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:21.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:21.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:21.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:21.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:21.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:21.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:21.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:21.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:21.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:21.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:21.794 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:21.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:21.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:21 smithi152 ceph-mon[93456]: pgmap v1655: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T17:47:21.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:21 smithi142 ceph-mon[106729]: pgmap v1655: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T17:47:21.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:21 smithi142 ceph-mon[103614]: pgmap v1655: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T17:47:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:47:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:47:21] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:47:21.896 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:21.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:21.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:22.023 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:22.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:22.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:22.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:22.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:22.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:22.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:22.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:22.431 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:22.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:22.559 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:22.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:22.688 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:22.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:22.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:22.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:22.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:22.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:23.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:23.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:22 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:22 smithi142 ceph-mon[106729]: pgmap v1656: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-17T17:47:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:22 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:23.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:22 smithi142 ceph-mon[103614]: pgmap v1656: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-17T17:47:23.174 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:23.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:23.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:23.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:22 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:22 smithi152 ceph-mon[93456]: pgmap v1656: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-17T17:47:23.327 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:23.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:23.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:23.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:23.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:23.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:23.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:23.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:23.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:23.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:23.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:24.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:24.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:24.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:24.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:24.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:24.273 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:24.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:24.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:24.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:24.355 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:47:24.356 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:47:24.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:24.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:24.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:24.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:24.583 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:24.685 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:24.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:24.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:24.833 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:24.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:24.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:24.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:24.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:25.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:25.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:25.157 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1656,"stamp":"2024-08-17T17:47:22.446403+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2958510,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6561588,"kb_used_data":33940,"kb_used_omap":27,"kb_used_meta":6527524,"kb_avail":555909324,"statfs":{"total":575970213888,"available":569251147776,"internally_reserved":0,"allocated":34754560,"data_stored":21002033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":6684185320},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.937384"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":669,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594917+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:47:14.594917+0000","last_peered":"2024-08-17T17:47:14.594917+0000","last_clean":"2024-08-17T17:47:14.594917+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:47:14.594917+0000","last_undegraded":"2024-08-17T17:47:14.594917+0000","last_fullsized":"2024-08-17T17:47:14.594917+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":121,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675011+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:47:13.675011+0000","last_peered":"2024-08-17T17:47:13.675011+0000","last_clean":"2024-08-17T17:47:13.675011+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:47:13.675011+0000","last_undegraded":"2024-08-17T17:47:13.675011+0000","last_fullsized":"2024-08-17T17:47:13.675011+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":315,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675126+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:47:13.675126+0000","last_peered":"2024-08-17T17:47:13.675126+0000","last_clean":"2024-08-17T17:47:13.675126+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:47:13.675126+0000","last_undegraded":"2024-08-17T17:47:13.675126+0000","last_fullsized":"2024-08-17T17:47:13.675126+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":367,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594630+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:47:14.594630+0000","last_peered":"2024-08-17T17:47:14.594630+0000","last_clean":"2024-08-17T17:47:14.594630+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:47:14.594630+0000","last_undegraded":"2024-08-17T17:47:14.594630+0000","last_fullsized":"2024-08-17T17:47:14.594630+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":544,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595038+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:47:14.595038+0000","last_peered":"2024-08-17T17:47:14.595038+0000","last_clean":"2024-08-17T17:47:14.595038+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:47:14.595038+0000","last_undegraded":"2024-08-17T17:47:14.595038+0000","last_fullsized":"2024-08-17T17:47:14.595038+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":264,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674948+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:47:13.674948+0000","last_peered":"2024-08-17T17:47:13.674948+0000","last_clean":"2024-08-17T17:47:13.674948+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:47:13.674948+0000","last_undegraded":"2024-08-17T17:47:13.674948+0000","last_fullsized":"2024-08-17T17:47:13.674948+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":638,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595061+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:47:14.595061+0000","last_peered":"2024-08-17T17:47:14.595061+0000","last_clean":"2024-08-17T17:47:14.595061+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:47:14.595061+0000","last_undegraded":"2024-08-17T17:47:14.595061+0000","last_fullsized":"2024-08-17T17:47:14.595061+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":432,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516270+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:47:13.516270+0000","last_peered":"2024-08-17T17:47:13.516270+0000","last_clean":"2024-08-17T17:47:13.516270+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:47:13.516270+0000","last_undegraded":"2024-08-17T17:47:13.516270+0000","last_fullsized":"2024-08-17T17:47:13.516270+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":117,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675456+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:47:13.675456+0000","last_peered":"2024-08-17T17:47:13.675456+0000","last_clean":"2024-08-17T17:47:13.675456+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:47:13.675456+0000","last_undegraded":"2024-08-17T17:47:13.675456+0000","last_fullsized":"2024-08-17T17:47:13.675456+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":298,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674679+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:47:13.674679+0000","last_peered":"2024-08-17T17:47:13.674679+0000","last_clean":"2024-08-17T17:47:13.674679+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:47:13.674679+0000","last_undegraded":"2024-08-17T17:47:13.674679+0000","last_fullsized":"2024-08-17T17:47:13.674679+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":119,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675259+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:47:13.675259+0000","last_peered":"2024-08-17T17:47:13.675259+0000","last_clean":"2024-08-17T17:47:13.675259+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:47:13.675259+0000","last_undegraded":"2024-08-17T17:47:13.675259+0000","last_fullsized":"2024-08-17T17:47:13.675259+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":383,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674869+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:47:13.674869+0000","last_peered":"2024-08-17T17:47:13.674869+0000","last_clean":"2024-08-17T17:47:13.674869+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:47:13.674869+0000","last_undegraded":"2024-08-17T17:47:13.674869+0000","last_fullsized":"2024-08-17T17:47:13.674869+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":115,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675045+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:47:13.675045+0000","last_peered":"2024-08-17T17:47:13.675045+0000","last_clean":"2024-08-17T17:47:13.675045+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:47:13.675045+0000","last_undegraded":"2024-08-17T17:47:13.675045+0000","last_fullsized":"2024-08-17T17:47:13.675045+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":393,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516080+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:47:13.516080+0000","last_peered":"2024-08-17T17:47:13.516080+0000","last_clean":"2024-08-17T17:47:13.516080+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:47:13.516080+0000","last_undegraded":"2024-08-17T17:47:13.516080+0000","last_fullsized":"2024-08-17T17:47:13.516080+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":581,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595033+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:47:14.595033+0000","last_peered":"2024-08-17T17:47:14.595033+0000","last_clean":"2024-08-17T17:47:14.595033+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:47:14.595033+0000","last_undegraded":"2024-08-17T17:47:14.595033+0000","last_fullsized":"2024-08-17T17:47:14.595033+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":348,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.517253+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:47:14.517253+0000","last_peered":"2024-08-17T17:47:14.517253+0000","last_clean":"2024-08-17T17:47:14.517253+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:47:14.517253+0000","last_undegraded":"2024-08-17T17:47:14.517253+0000","last_fullsized":"2024-08-17T17:47:14.517253+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":671,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594845+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:47:14.594845+0000","last_peered":"2024-08-17T17:47:14.594845+0000","last_clean":"2024-08-17T17:47:14.594845+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:47:14.594845+0000","last_undegraded":"2024-08-17T17:47:14.594845+0000","last_fullsized":"2024-08-17T17:47:14.594845+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":366,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674823+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:47:13.674823+0000","last_peered":"2024-08-17T17:47:13.674823+0000","last_clean":"2024-08-17T17:47:13.674823+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:47:13.674823+0000","last_undegraded":"2024-08-17T17:47:13.674823+0000","last_fullsized":"2024-08-17T17:47:13.674823+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":367,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516182+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:47:13.516182+0000","last_peered":"2024-08-17T17:47:13.516182+0000","last_clean":"2024-08-17T17:47:13.516182+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:47:13.516182+0000","last_undegraded":"2024-08-17T17:47:13.516182+0000","last_fullsized":"2024-08-17T17:47:13.516182+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":393,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675524+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:47:13.675524+0000","last_peered":"2024-08-17T17:47:13.675524+0000","last_clean":"2024-08-17T17:47:13.675524+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:47:13.675524+0000","last_undegraded":"2024-08-17T17:47:13.675524+0000","last_fullsized":"2024-08-17T17:47:13.675524+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":404,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675099+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:47:13.675099+0000","last_peered":"2024-08-17T17:47:13.675099+0000","last_clean":"2024-08-17T17:47:13.675099+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:47:13.675099+0000","last_undegraded":"2024-08-17T17:47:13.675099+0000","last_fullsized":"2024-08-17T17:47:13.675099+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":366,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675335+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:47:13.675335+0000","last_peered":"2024-08-17T17:47:13.675335+0000","last_clean":"2024-08-17T17:47:13.675335+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:47:13.675335+0000","last_undegraded":"2024-08-17T17:47:13.675335+0000","last_fullsized":"2024-08-17T17:47:13.675335+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":640,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594672+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:47:14.594672+0000","last_peered":"2024-08-17T17:47:14.594672+0000","last_clean":"2024-08-17T17:47:14.594672+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:47:14.594672+0000","last_undegraded":"2024-08-17T17:47:14.594672+0000","last_fullsized":"2024-08-17T17:47:14.594672+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":577,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595137+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:47:14.595137+0000","last_peered":"2024-08-17T17:47:14.595137+0000","last_clean":"2024-08-17T17:47:14.595137+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:47:14.595137+0000","last_undegraded":"2024-08-17T17:47:14.595137+0000","last_fullsized":"2024-08-17T17:47:14.595137+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":784,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.531914+0000","last_change":"2024-08-17T17:47:14.531821+0000","last_active":"2024-08-17T17:47:14.531914+0000","last_peered":"2024-08-17T17:47:14.531914+0000","last_clean":"2024-08-17T17:47:14.531914+0000","last_became_active":"2024-08-17T17:47:14.531354+0000","last_became_peered":"2024-08-17T17:47:14.531354+0000","last_unstale":"2024-08-17T17:47:14.531914+0000","last_undegraded":"2024-08-17T17:47:14.531914+0000","last_fullsized":"2024-08-17T17:47:14.531914+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:54.544598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":347,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674848+0000","last_change":"2024-08-17T17:46:26.863739+0000","last_active":"2024-08-17T17:47:13.674848+0000","last_peered":"2024-08-17T17:47:13.674848+0000","last_clean":"2024-08-17T17:47:13.674848+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:47:13.674848+0000","last_undegraded":"2024-08-17T17:47:13.674848+0000","last_fullsized":"2024-08-17T17:47:13.674848+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":620,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594660+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:47:14.594660+0000","last_peered":"2024-08-17T17:47:14.594660+0000","last_clean":"2024-08-17T17:47:14.594660+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:47:14.594660+0000","last_undegraded":"2024-08-17T17:47:14.594660+0000","last_fullsized":"2024-08-17T17:47:14.594660+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":376,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516512+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:47:13.516512+0000","last_peered":"2024-08-17T17:47:13.516512+0000","last_clean":"2024-08-17T17:47:13.516512+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:47:13.516512+0000","last_undegraded":"2024-08-17T17:47:13.516512+0000","last_fullsized":"2024-08-17T17:47:13.516512+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":436,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.515160+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:47:14.515160+0000","last_peered":"2024-08-17T17:47:14.515160+0000","last_clean":"2024-08-17T17:47:14.515160+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:47:14.515160+0000","last_undegraded":"2024-08-17T17:47:14.515160+0000","last_fullsized":"2024-08-17T17:47:14.515160+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":348,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674722+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:47:13.674722+0000","last_peered":"2024-08-17T17:47:13.674722+0000","last_clean":"2024-08-17T17:47:13.674722+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:47:13.674722+0000","last_undegraded":"2024-08-17T17:47:13.674722+0000","last_fullsized":"2024-08-17T17:47:13.674722+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":362,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.517525+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:47:14.517525+0000","last_peered":"2024-08-17T17:47:14.517525+0000","last_clean":"2024-08-17T17:47:14.517525+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:47:14.517525+0000","last_undegraded":"2024-08-17T17:47:14.517525+0000","last_fullsized":"2024-08-17T17:47:14.517525+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":528,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594670+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:47:14.594670+0000","last_peered":"2024-08-17T17:47:14.594670+0000","last_clean":"2024-08-17T17:47:14.594670+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:47:14.594670+0000","last_undegraded":"2024-08-17T17:47:14.594670+0000","last_fullsized":"2024-08-17T17:47:14.594670+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516118+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:47:13.516118+0000","last_peered":"2024-08-17T17:47:13.516118+0000","last_clean":"2024-08-17T17:47:13.516118+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:47:13.516118+0000","last_undegraded":"2024-08-17T17:47:13.516118+0000","last_fullsized":"2024-08-17T17:47:13.516118+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":713,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.521819+0000","last_change":"2024-08-17T17:24:36.966387+0000","last_active":"2024-08-17T17:47:14.521819+0000","last_peered":"2024-08-17T17:47:14.521819+0000","last_clean":"2024-08-17T17:47:14.521819+0000","last_became_active":"2024-08-17T17:24:36.966053+0000","last_became_peered":"2024-08-17T17:24:36.966053+0000","last_unstale":"2024-08-17T17:47:14.521819+0000","last_undegraded":"2024-08-17T17:47:14.521819+0000","last_fullsized":"2024-08-17T17:47:14.521819+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:54.098730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":504,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.515361+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:47:14.515361+0000","last_peered":"2024-08-17T17:47:14.515361+0000","last_clean":"2024-08-17T17:47:14.515361+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:47:14.515361+0000","last_undegraded":"2024-08-17T17:47:14.515361+0000","last_fullsized":"2024-08-17T17:47:14.515361+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":387,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675224+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:47:13.675224+0000","last_peered":"2024-08-17T17:47:13.675224+0000","last_clean":"2024-08-17T17:47:13.675224+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:47:13.675224+0000","last_undegraded":"2024-08-17T17:47:13.675224+0000","last_fullsized":"2024-08-17T17:47:13.675224+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":299,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594820+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:47:14.594820+0000","last_peered":"2024-08-17T17:47:14.594820+0000","last_clean":"2024-08-17T17:47:14.594820+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:47:14.594820+0000","last_undegraded":"2024-08-17T17:47:14.594820+0000","last_fullsized":"2024-08-17T17:47:14.594820+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":326,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674952+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:47:13.674952+0000","last_peered":"2024-08-17T17:47:13.674952+0000","last_clean":"2024-08-17T17:47:13.674952+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:47:13.674952+0000","last_undegraded":"2024-08-17T17:47:13.674952+0000","last_fullsized":"2024-08-17T17:47:13.674952+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":168,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594677+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:47:14.594677+0000","last_peered":"2024-08-17T17:47:14.594677+0000","last_clean":"2024-08-17T17:47:14.594677+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:47:14.594677+0000","last_undegraded":"2024-08-17T17:47:14.594677+0000","last_fullsized":"2024-08-17T17:47:14.594677+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":663,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.515489+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:47:14.515489+0000","last_peered":"2024-08-17T17:47:14.515489+0000","last_clean":"2024-08-17T17:47:14.515489+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:47:14.515489+0000","last_undegraded":"2024-08-17T17:47:14.515489+0000","last_fullsized":"2024-08-17T17:47:14.515489+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":322,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675329+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:47:13.675329+0000","last_peered":"2024-08-17T17:47:13.675329+0000","last_clean":"2024-08-17T17:47:13.675329+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:47:13.675329+0000","last_undegraded":"2024-08-17T17:47:13.675329+0000","last_fullsized":"2024-08-17T17:47:13.675329+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":419,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.517404+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:47:14.517404+0000","last_peered":"2024-08-17T17:47:14.517404+0000","last_clean":"2024-08-17T17:47:14.517404+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:47:14.517404+0000","last_undegraded":"2024-08-17T17:47:14.517404+0000","last_fullsized":"2024-08-17T17:47:14.517404+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":464,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674873+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:47:13.674873+0000","last_peered":"2024-08-17T17:47:13.674873+0000","last_clean":"2024-08-17T17:47:13.674873+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:47:13.674873+0000","last_undegraded":"2024-08-17T17:47:13.674873+0000","last_fullsized":"2024-08-17T17:47:13.674873+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":694,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595235+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:47:14.595235+0000","last_peered":"2024-08-17T17:47:14.595235+0000","last_clean":"2024-08-17T17:47:14.595235+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:47:14.595235+0000","last_undegraded":"2024-08-17T17:47:14.595235+0000","last_fullsized":"2024-08-17T17:47:14.595235+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":662,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.515970+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:47:14.515970+0000","last_peered":"2024-08-17T17:47:14.515970+0000","last_clean":"2024-08-17T17:47:14.515970+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:47:14.515970+0000","last_undegraded":"2024-08-17T17:47:14.515970+0000","last_fullsized":"2024-08-17T17:47:14.515970+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":680,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594568+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:47:14.594568+0000","last_peered":"2024-08-17T17:47:14.594568+0000","last_clean":"2024-08-17T17:47:14.594568+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:47:14.594568+0000","last_undegraded":"2024-08-17T17:47:14.594568+0000","last_fullsized":"2024-08-17T17:47:14.594568+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":352,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.517476+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:47:14.517476+0000","last_peered":"2024-08-17T17:47:14.517476+0000","last_clean":"2024-08-17T17:47:14.517476+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:47:14.517476+0000","last_undegraded":"2024-08-17T17:47:14.517476+0000","last_fullsized":"2024-08-17T17:47:14.517476+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":416,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516587+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:47:13.516587+0000","last_peered":"2024-08-17T17:47:13.516587+0000","last_clean":"2024-08-17T17:47:13.516587+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:47:13.516587+0000","last_undegraded":"2024-08-17T17:47:13.516587+0000","last_fullsized":"2024-08-17T17:47:13.516587+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":366,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675022+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:47:13.675022+0000","last_peered":"2024-08-17T17:47:13.675022+0000","last_clean":"2024-08-17T17:47:13.675022+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:47:13.675022+0000","last_undegraded":"2024-08-17T17:47:13.675022+0000","last_fullsized":"2024-08-17T17:47:13.675022+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":366,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675136+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:47:13.675136+0000","last_peered":"2024-08-17T17:47:13.675136+0000","last_clean":"2024-08-17T17:47:13.675136+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:47:13.675136+0000","last_undegraded":"2024-08-17T17:47:13.675136+0000","last_fullsized":"2024-08-17T17:47:13.675136+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":409,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675029+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:47:13.675029+0000","last_peered":"2024-08-17T17:47:13.675029+0000","last_clean":"2024-08-17T17:47:13.675029+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:47:13.675029+0000","last_undegraded":"2024-08-17T17:47:13.675029+0000","last_fullsized":"2024-08-17T17:47:13.675029+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":668,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.515368+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:47:14.515368+0000","last_peered":"2024-08-17T17:47:14.515368+0000","last_clean":"2024-08-17T17:47:14.515368+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:47:14.515368+0000","last_undegraded":"2024-08-17T17:47:14.515368+0000","last_fullsized":"2024-08-17T17:47:14.515368+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":208,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595024+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:47:14.595024+0000","last_peered":"2024-08-17T17:47:14.595024+0000","last_clean":"2024-08-17T17:47:14.595024+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:47:14.595024+0000","last_undegraded":"2024-08-17T17:47:14.595024+0000","last_fullsized":"2024-08-17T17:47:14.595024+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":316,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675341+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:47:13.675341+0000","last_peered":"2024-08-17T17:47:13.675341+0000","last_clean":"2024-08-17T17:47:13.675341+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:47:13.675341+0000","last_undegraded":"2024-08-17T17:47:13.675341+0000","last_fullsized":"2024-08-17T17:47:13.675341+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":626,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594529+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:47:14.594529+0000","last_peered":"2024-08-17T17:47:14.594529+0000","last_clean":"2024-08-17T17:47:14.594529+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:47:14.594529+0000","last_undegraded":"2024-08-17T17:47:14.594529+0000","last_fullsized":"2024-08-17T17:47:14.594529+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":460,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675493+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:47:13.675493+0000","last_peered":"2024-08-17T17:47:13.675493+0000","last_clean":"2024-08-17T17:47:13.675493+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:47:13.675493+0000","last_undegraded":"2024-08-17T17:47:13.675493+0000","last_fullsized":"2024-08-17T17:47:13.675493+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":620,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594482+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:47:14.594482+0000","last_peered":"2024-08-17T17:47:14.594482+0000","last_clean":"2024-08-17T17:47:14.594482+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:47:14.594482+0000","last_undegraded":"2024-08-17T17:47:14.594482+0000","last_fullsized":"2024-08-17T17:47:14.594482+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":383,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.516520+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:47:13.516520+0000","last_peered":"2024-08-17T17:47:13.516520+0000","last_clean":"2024-08-17T17:47:13.516520+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:47:13.516520+0000","last_undegraded":"2024-08-17T17:47:13.516520+0000","last_fullsized":"2024-08-17T17:47:13.516520+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":652,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594600+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:47:14.594600+0000","last_peered":"2024-08-17T17:47:14.594600+0000","last_clean":"2024-08-17T17:47:14.594600+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:47:14.594600+0000","last_undegraded":"2024-08-17T17:47:14.594600+0000","last_fullsized":"2024-08-17T17:47:14.594600+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":368,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595100+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:47:14.595100+0000","last_peered":"2024-08-17T17:47:14.595100+0000","last_clean":"2024-08-17T17:47:14.595100+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:47:14.595100+0000","last_undegraded":"2024-08-17T17:47:14.595100+0000","last_fullsized":"2024-08-17T17:47:14.595100+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":660,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.515235+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:47:14.515235+0000","last_peered":"2024-08-17T17:47:14.515235+0000","last_clean":"2024-08-17T17:47:14.515235+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:47:14.515235+0000","last_undegraded":"2024-08-17T17:47:14.515235+0000","last_fullsized":"2024-08-17T17:47:14.515235+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":619,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594898+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:47:14.594898+0000","last_peered":"2024-08-17T17:47:14.594898+0000","last_clean":"2024-08-17T17:47:14.594898+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:47:14.594898+0000","last_undegraded":"2024-08-17T17:47:14.594898+0000","last_fullsized":"2024-08-17T17:47:14.594898+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":395,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675128+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:47:13.675128+0000","last_peered":"2024-08-17T17:47:13.675128+0000","last_clean":"2024-08-17T17:47:13.675128+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:47:13.675128+0000","last_undegraded":"2024-08-17T17:47:13.675128+0000","last_fullsized":"2024-08-17T17:47:13.675128+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":308,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594953+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:47:14.594953+0000","last_peered":"2024-08-17T17:47:14.594953+0000","last_clean":"2024-08-17T17:47:14.594953+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:47:14.594953+0000","last_undegraded":"2024-08-17T17:47:14.594953+0000","last_fullsized":"2024-08-17T17:47:14.594953+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":365,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.517134+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:47:14.517134+0000","last_peered":"2024-08-17T17:47:14.517134+0000","last_clean":"2024-08-17T17:47:14.517134+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:47:14.517134+0000","last_undegraded":"2024-08-17T17:47:14.517134+0000","last_fullsized":"2024-08-17T17:47:14.517134+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":403,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675136+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:47:13.675136+0000","last_peered":"2024-08-17T17:47:13.675136+0000","last_clean":"2024-08-17T17:47:13.675136+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:47:13.675136+0000","last_undegraded":"2024-08-17T17:47:13.675136+0000","last_fullsized":"2024-08-17T17:47:13.675136+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":266,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674940+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:47:13.674940+0000","last_peered":"2024-08-17T17:47:13.674940+0000","last_clean":"2024-08-17T17:47:13.674940+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:47:13.674940+0000","last_undegraded":"2024-08-17T17:47:13.674940+0000","last_fullsized":"2024-08-17T17:47:13.674940+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":403,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.674986+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:47:13.674986+0000","last_peered":"2024-08-17T17:47:13.674986+0000","last_clean":"2024-08-17T17:47:13.674986+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:47:13.674986+0000","last_undegraded":"2024-08-17T17:47:13.674986+0000","last_fullsized":"2024-08-17T17:47:13.674986+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":266,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675377+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:47:13.675377+0000","last_peered":"2024-08-17T17:47:13.675377+0000","last_clean":"2024-08-17T17:47:13.675377+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:47:13.675377+0000","last_undegraded":"2024-08-17T17:47:13.675377+0000","last_fullsized":"2024-08-17T17:47:13.675377+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":672,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.594754+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:47:14.594754+0000","last_peered":"2024-08-17T17:47:14.594754+0000","last_clean":"2024-08-17T17:47:14.594754+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:47:14.594754+0000","last_undegraded":"2024-08-17T17:47:14.594754+0000","last_fullsized":"2024-08-17T17:47:14.594754+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":646,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.595138+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:47:14.595138+0000","last_peered":"2024-08-17T17:47:14.595138+0000","last_clean":"2024-08-17T17:47:14.595138+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:47:14.595138+0000","last_undegraded":"2024-08-17T17:47:14.595138+0000","last_fullsized":"2024-08-17T17:47:14.595138+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":366,"reported_epoch":411,"state":"active+clean","last_fresh":"2024-08-17T17:47:13.675497+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:47:13.675497+0000","last_peered":"2024-08-17T17:47:13.675497+0000","last_clean":"2024-08-17T17:47:13.675497+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:47:13.675497+0000","last_undegraded":"2024-08-17T17:47:13.675497+0000","last_fullsized":"2024-08-17T17:47:13.675497+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":367,"reported_epoch":412,"state":"active+clean","last_fresh":"2024-08-17T17:47:14.517573+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:47:14.517573+0000","last_peered":"2024-08-17T17:47:14.517573+0000","last_clean":"2024-08-17T17:47:14.517573+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:47:14.517573+0000","last_undegraded":"2024-08-17T17:47:14.517573+0000","last_fullsized":"2024-08-17T17:47:14.517573+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170171,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463541,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1506964,"kb_used_data":5960,"kb_used_omap":1,"kb_used_meta":1500990,"kb_avail":92238188,"statfs":{"total":95995035648,"available":94451904512,"internally_reserved":0,"allocated":6103040,"data_stored":3798670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1537014104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492215,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":396544,"kb_used_data":5356,"kb_used_omap":7,"kb_used_meta":391160,"kb_avail":93348608,"statfs":{"total":95995035648,"available":95588974592,"internally_reserved":0,"allocated":5484544,"data_stored":3202474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":400548173},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986806,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505740,"kb_used_data":5372,"kb_used_omap":1,"kb_used_meta":1500350,"kb_avail":92239412,"statfs":{"total":95995035648,"available":94453157888,"internally_reserved":0,"allocated":5500928,"data_stored":3207498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1536358744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225634,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1007160,"kb_used_data":5356,"kb_used_omap":4,"kb_used_meta":1001787,"kb_avail":92737992,"statfs":{"total":95995035648,"available":94963703808,"internally_reserved":0,"allocated":5484544,"data_stored":3202772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1025830336},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075959,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533852,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097924,"kb_used_data":5936,"kb_used_omap":5,"kb_used_meta":1091962,"kb_avail":92647228,"statfs":{"total":95995035648,"available":94870761472,"internally_reserved":0,"allocated":6078464,"data_stored":3791890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1118169328},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089829,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1047256,"kb_used_data":5960,"kb_used_omap":5,"kb_used_meta":1041274,"kb_avail":92697896,"statfs":{"total":95995035648,"available":94922645504,"internally_reserved":0,"allocated":6103040,"data_stored":3798729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1066264635},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:47:25.161 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:47:25.161 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:47:25.161 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-17T17:47:25.161 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:47:25.161 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:47:25.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-17T17:47:25.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:25.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:25.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:25.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:25.351 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:25.431 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:47:25.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:25.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:25.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:47:25.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:25.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:25.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:25.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:25.787 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:25 smithi142 ceph-mon[106729]: from='client.38097 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:25.788 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:25 smithi142 ceph-mon[106729]: pgmap v1657: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:47:25.788 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:25 smithi142 ceph-mon[103614]: from='client.38097 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:25.788 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:25 smithi142 ceph-mon[103614]: pgmap v1657: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:47:25.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:25 smithi152 ceph-mon[93456]: from='client.38097 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:25.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:25 smithi152 ceph-mon[93456]: pgmap v1657: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:47:25.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:25.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.848 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:25.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:25.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:25.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:26.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:26.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.095 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:26.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:26.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:26.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:26.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:26.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:26.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:26.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:26.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:26.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:26.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:26.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:26.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:26.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:26.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:26.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:26.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:26.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:26.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:26.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:26.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:26.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:27.047 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:47:27.047 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:47:27.047 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:47:27.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:27.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:27.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:27.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:27.276 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:27.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:27.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:27.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:27.436 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:27.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:27.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:27.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:27.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:27.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:27.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:27 smithi152 ceph-mon[93456]: pgmap v1658: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:47:27.650 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:27.650 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:27.650 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:27 smithi142 ceph-mon[106729]: pgmap v1658: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:47:27.650 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:27.651 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:27.651 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:27 smithi142 ceph-mon[103614]: pgmap v1658: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:47:27.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:27.706 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:27.808 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:27.829 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:27.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:27.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:27.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:27.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:27.994 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:47:27.994 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:47:28.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:28.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:28.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:28.202 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:28.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:28.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:28.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:28.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:28.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:28.574 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:28.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:28.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:28.707 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:47:28.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:28.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:28.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:28.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:28.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:29.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:29.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:29.165 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:29.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:29.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:29.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:29.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:29.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:29.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:29.553 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:29 smithi142 ceph-mon[106729]: pgmap v1659: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:29.554 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:29 smithi142 ceph-mon[103614]: pgmap v1659: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:29.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:29.667 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:29.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:29.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:29.812 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:29.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:29 smithi152 ceph-mon[93456]: pgmap v1659: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:29.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:29.938 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:30.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:30.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:30.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:30.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:30.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:30.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:30.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:30.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:30.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:30.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:47:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:47:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:30.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:30.678 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:30.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:30.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:30.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:30.903 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:30.924 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:31.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:31.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:31.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:31.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:31.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:31.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:31.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:31.292 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:31.394 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:31.416 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:31.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:31.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:31.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:31.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:31.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:31 smithi142 ceph-mon[106729]: pgmap v1660: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:31.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:31 smithi142 ceph-mon[103614]: pgmap v1660: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:31.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:31.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:31.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:31.786 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:31 smithi152 ceph-mon[93456]: pgmap v1660: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:31.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:31.909 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:31.983 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:47:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:47:31] "GET /metrics HTTP/1.1" 200 34163 "" "Prometheus/2.51.0" 2024-08-17T17:47:32.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:32.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:32.154 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:47:32.154 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:47:32.157 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:32.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.161 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:47:32.161 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:32.177 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:47:32.177 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:32.193 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:47:32.193 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:32.204 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:32.213 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:47:32.213 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:32.229 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:47:32.229 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:32.244 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:47:32.245 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:32.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:32.261 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:47:32.261 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:32.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:32.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:32.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:32.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:32.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:32.466 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_8' created 2024-08-17T17:47:32.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:32.527 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:32.619 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:47:32.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:47:32.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2867911211' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-08-17T17:47:32.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-08-17T17:47:32.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2227952368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:32.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2867911211' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-08-17T17:47:32.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-08-17T17:47:32.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2227952368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:32.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:32.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:32.810 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2867911211' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-08-17T17:47:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-08-17T17:47:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2227952368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:32.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:32.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:33.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:33.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:33.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:33.160 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:33.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:33.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:33.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:33.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:33.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:33.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:33.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:33.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.558 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:33.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:33 smithi142 ceph-mon[106729]: pgmap v1661: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:33.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:33 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-08-17T17:47:33.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:33 smithi142 ceph-mon[106729]: osdmap e413: 8 total, 8 up, 6 in 2024-08-17T17:47:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:33 smithi142 ceph-mon[103614]: pgmap v1661: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:33 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-08-17T17:47:33.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:33 smithi142 ceph-mon[103614]: osdmap e413: 8 total, 8 up, 6 in 2024-08-17T17:47:33.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:33.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:33.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2024-08-17T17:47:33.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:33.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:33 smithi152 ceph-mon[93456]: pgmap v1661: 73 pgs: 73 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:33 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-08-17T17:47:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:33 smithi152 ceph-mon[93456]: osdmap e413: 8 total, 8 up, 6 in 2024-08-17T17:47:33.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:33.932 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:33.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:34.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:34.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:34.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:34.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:34.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:34.306 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:34.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:34.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:34.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:34.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:34.557 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:34.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:34 smithi142 ceph-mon[106729]: osdmap e414: 8 total, 8 up, 6 in 2024-08-17T17:47:34.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:34 smithi142 ceph-mon[103614]: osdmap e414: 8 total, 8 up, 6 in 2024-08-17T17:47:34.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:34.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:34.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:34.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:34.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:34 smithi152 ceph-mon[93456]: osdmap e414: 8 total, 8 up, 6 in 2024-08-17T17:47:34.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:34.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:34.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:35.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:35.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:35.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:35.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:35.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:35.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.307 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:35.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:35.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:35.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:35.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:35.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:35.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:35.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:35.567 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:35 smithi142 ceph-mon[103614]: pgmap v1664: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:35.567 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:35 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:47:35.567 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:35 smithi142 ceph-mon[103614]: osdmap e415: 8 total, 8 up, 6 in 2024-08-17T17:47:35.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:35 smithi142 ceph-mon[106729]: pgmap v1664: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:35.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:35 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:47:35.568 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:35 smithi142 ceph-mon[106729]: osdmap e415: 8 total, 8 up, 6 in 2024-08-17T17:47:35.574 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:35.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:35.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:35.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:35.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:35 smithi152 ceph-mon[93456]: pgmap v1664: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:35 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:47:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:35 smithi152 ceph-mon[93456]: osdmap e415: 8 total, 8 up, 6 in 2024-08-17T17:47:35.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:35.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:36.038 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:36.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:36.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:36.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:36.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:36.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:36.305 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:36.312 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:36.414 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:36.440 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:36.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:36.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:36.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:36.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.814 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:36.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:36.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:36.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:37.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:37.065 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:37.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:37.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:37.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:37.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:37.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:37.416 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:37.438 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:37.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:37.486 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:47:37.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:37.554 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:37 smithi142 ceph-mon[106729]: pgmap v1666: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:37.554 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1591431085' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:37.554 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:37 smithi142 ceph-mon[103614]: pgmap v1666: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:37.554 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1591431085' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:37.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:37.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:47:37.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:37.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:37.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:37.812 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:37 smithi152 ceph-mon[93456]: pgmap v1666: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1591431085' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:37.914 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:37.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:37.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:38.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:38.063 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:38.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:38.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:38.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:38.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:38.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:38.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:38.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:38.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:38.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:38.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:38.687 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:38.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:38.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:38 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:38.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1591431085' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:38.757 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:38 smithi142 ceph-mon[106729]: osdmap e416: 8 total, 8 up, 6 in 2024-08-17T17:47:38.757 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:38 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:38.757 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1591431085' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:38.757 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:38 smithi142 ceph-mon[103614]: osdmap e416: 8 total, 8 up, 6 in 2024-08-17T17:47:38.789 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:38.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:38 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1591431085' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:38.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:38 smithi152 ceph-mon[93456]: osdmap e416: 8 total, 8 up, 6 in 2024-08-17T17:47:38.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:38.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:38.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:39.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:39.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:39.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:39.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:39.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:39.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:39.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:39.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:39.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.427 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:39.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:39.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:39.553 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:39.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:39.607 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:39 smithi142 ceph-mon[103614]: pgmap v1668: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:39.608 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:39 smithi142 ceph-mon[106729]: pgmap v1668: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:39.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:39.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:39.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:39.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:39.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:39 smithi152 ceph-mon[93456]: pgmap v1668: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:39.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:39.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:40.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:40.051 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:40.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:40.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:40.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:40.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:40.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:40.304 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:40.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:40.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:40.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:40.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:40.539 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:47:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:47:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:40.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:40.587 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:47:40.588 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:47:40.588 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:47:40.588 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:47:40.588 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:47:40.610 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:47:40.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:40.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:40.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:40.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:40.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:40.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:41.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:41.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:41.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.179 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:41.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:41.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:41.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:41.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:41.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:41.528 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:41.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:41.678 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-mon[106729]: pgmap v1669: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-mon[103614]: pgmap v1669: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:41.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:41.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:41.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:41 smithi152 ceph-mon[93456]: pgmap v1669: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:41.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:41.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:42.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:42.049 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:42.067 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:47:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:47:41] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-08-17T17:47:42.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:47:42.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:42.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:42.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:42.304 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:42.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:42.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:42.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:42.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:42.505 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_8' 2024-08-17T17:47:42.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:42.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:42.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:42.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1268677019' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:42 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1268677019' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:42 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1268677019' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:42 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:42.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.925 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:43.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:43.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:43.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:43.172 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:43.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:43.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:43.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:43.384 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:43.385 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:43.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:43.428 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:43.502 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:47:43.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:43.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:43.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:43.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[106729]: pgmap v1670: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:43.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:47:43.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[106729]: osdmap e417: 8 total, 8 up, 6 in 2024-08-17T17:47:43.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/482610401' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:47:43.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[103614]: pgmap v1670: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:43.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:47:43.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[103614]: osdmap e417: 8 total, 8 up, 6 in 2024-08-17T17:47:43.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/482610401' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:47:43.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:43.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:43.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:43.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:43.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:43 smithi152 ceph-mon[93456]: pgmap v1670: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:43 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:47:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:43 smithi152 ceph-mon[93456]: osdmap e417: 8 total, 8 up, 6 in 2024-08-17T17:47:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/482610401' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:47:43.912 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:43.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:43.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:43.949 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:44.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:44.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:44.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:44.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:44.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:44.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:44.291 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2024-08-17T17:47:44.294 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:47:44.294 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:47:44.316 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:44.340 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:47:44.340 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:47:44.340 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_393845 2024-08-17T17:47:44.340 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:47:44.340 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:47:44.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:44.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:44.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:44.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/482610401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[106729]: osdmap e418: 8 total, 8 up, 6 in 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/482610401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[103614]: osdmap e418: 8 total, 8 up, 6 in 2024-08-17T17:47:44.683 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:44.684 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:44 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:44.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:44.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/482610401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:47:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:44 smithi152 ceph-mon[93456]: osdmap e418: 8 total, 8 up, 6 in 2024-08-17T17:47:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:44 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:44.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:44.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:44.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:45.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.019 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:45.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:45.120 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:45.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:45.262 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:45.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:45.340 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 4126 4110 256.861 256.875 0.00652419 0.00388179 2024-08-17T17:47:45.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:45.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:45.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:45.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.565 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[106729]: pgmap v1673: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:45.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:47:45.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[103614]: pgmap v1673: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:45.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:45.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:45 smithi152 ceph-mon[93456]: pgmap v1673: 89 pgs: 89 active+clean; 582 KiB data, 6.3 GiB used, 530 GiB / 536 GiB avail 2024-08-17T17:47:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:45 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:47:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:45.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:45.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:45.977 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:45.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:46.079 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:46.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:46.215 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:46.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:46.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:46.341 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 7982 7966 248.92 241 0.0047626 0.00400895 2024-08-17T17:47:46.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:46.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:46.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:46.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:46.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:46.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:46.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:46.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:46.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:46.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:46.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:46.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:46.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:47.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:47.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:47.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:47.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:47.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:47.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:47:47.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:47.341 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 11535 11519 239.888 222.062 0.00496924 0.00416045 2024-08-17T17:47:47.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:47.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:47.456 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:47.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:47.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:47.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:47.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:47.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:47.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:47.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:47.806 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:47 smithi142 ceph-mon[106729]: pgmap v1674: 89 pgs: 89 active+clean; 121 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 15 MiB/s wr, 241 op/s 2024-08-17T17:47:47.806 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:47.806 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:47 smithi142 ceph-mon[103614]: pgmap v1674: 89 pgs: 89 active+clean; 121 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 15 MiB/s wr, 241 op/s 2024-08-17T17:47:47.806 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:47 smithi152 ceph-mon[93456]: pgmap v1674: 89 pgs: 89 active+clean; 121 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 15 MiB/s wr, 241 op/s 2024-08-17T17:47:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:47.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:47.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.007 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:48.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:48.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:48.179 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:48.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:48.281 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:48.318 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:48.341 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 14740 14724 229.992 200.312 0.00787298 0.00434173 2024-08-17T17:47:48.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:48.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:48.590 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:48.692 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:48.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:48.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:48.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:48.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:49.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:49.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:49.109 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.142 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:49.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:49.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 17787 17771 222.079 190.438 0.00338776 0.00449766 2024-08-17T17:47:49.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:49.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:49.534 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:47:49.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:49.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:49.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:49.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:49.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:49 smithi152 ceph-mon[93456]: pgmap v1675: 89 pgs: 89 active+clean; 408 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 51 MiB/s wr, 815 op/s 2024-08-17T17:47:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2123892614' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:49.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:49.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:49 smithi142 ceph-mon[106729]: pgmap v1675: 89 pgs: 89 active+clean; 408 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 51 MiB/s wr, 815 op/s 2024-08-17T17:47:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2123892614' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:49 smithi142 ceph-mon[103614]: pgmap v1675: 89 pgs: 89 active+clean; 408 MiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 51 MiB/s wr, 815 op/s 2024-08-17T17:47:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2123892614' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:47:49.935 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:50.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:50.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:50.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:50.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:50.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:50.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:50.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 20464 20448 212.95 167.312 0.00419637 0.00469025 2024-08-17T17:47:50.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:50.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:50.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:50.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:50.544 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:47:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:47:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:50.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:50.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:50.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:50.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:50 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:50 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:47:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:50 smithi152 ceph-mon[93456]: osdmap e419: 8 total, 8 up, 6 in 2024-08-17T17:47:50.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:50 smithi152 ceph-mon[93456]: pgmap v1677: 89 pgs: 89 active+clean; 870 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[106729]: osdmap e419: 8 total, 8 up, 6 in 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[106729]: pgmap v1677: 89 pgs: 89 active+clean; 870 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[103614]: osdmap e419: 8 total, 8 up, 6 in 2024-08-17T17:47:50.832 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:50 smithi142 ceph-mon[103614]: pgmap v1677: 89 pgs: 89 active+clean; 870 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-08-17T17:47:50.871 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:50.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:51.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:51.040 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:51.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:51.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:51.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:51.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:51.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:51.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 22594 22578 201.547 133.125 0.0123313 0.00495602 2024-08-17T17:47:51.397 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:51.430 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:51.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:51.568 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:51.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:51.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:51.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:47:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:47:51] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-08-17T17:47:51.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:51.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:51.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:52.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:52.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:52.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:52.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.269 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:52.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:52.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 25068 25052 195.68 154.625 0.00341259 0.00510454 2024-08-17T17:47:52.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:52.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:52.508 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:52.543 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:52.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:52.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:52.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:52.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:52.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:52.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:53.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:53.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:53.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:53.201 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:53.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:53.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 27532 27516 191.048 154 0.00634299 0.00522953 2024-08-17T17:47:53.345 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:53.375 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:53.401 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:47:53.401 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:47:53.428 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:47:53.428 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:53.445 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:47:53.445 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:53.462 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:47:53.462 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:53.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:53.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:53.518 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:47:53.518 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:53.548 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:47:53.549 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:53.582 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:47:53.582 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:53.596 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:53 smithi142 ceph-mon[106729]: pgmap v1678: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:47:53.596 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/802833862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:53.596 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:53 smithi142 ceph-mon[103614]: pgmap v1678: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:47:53.596 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/802833862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:53.609 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:53.609 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:47:53.610 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:53.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:53.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:53.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:53.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:53.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:53 smithi152 ceph-mon[93456]: pgmap v1678: 89 pgs: 89 active+clean; 1.2 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-17T17:47:53.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/802833862' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:53.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:53.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:53.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:54.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:54.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:54.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:54.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:54.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:54.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:54.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:54.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:54.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:54.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 29755 29739 185.837 138.938 0.00761634 0.00537598 2024-08-17T17:47:54.436 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:54.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:54.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:54.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:54.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:54.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:54.727 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:47:54.727 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:47:54.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:54.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:54.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:54.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:54.982 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:55.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:55.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:47:55.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:55.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:55.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:55.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:55.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 32063 32047 182.055 144.25 0.00616419 0.00548796 2024-08-17T17:47:55.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:55.421 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:55.485 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:47:55.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:55.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:55.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:55.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:55.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:55 smithi152 ceph-mon[93456]: pgmap v1679: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:47:55.830 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:55 smithi142 ceph-mon[106729]: pgmap v1679: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:47:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:55 smithi142 ceph-mon[103614]: pgmap v1679: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:47:55.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:55.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:55.966 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:56.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:56.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:56.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:56.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:56.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:56.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 33823 33807 176.05 110 0.00487727 0.00567548 2024-08-17T17:47:56.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:56.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:56.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:56.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:56.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:56.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:56.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:56.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:56.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:56.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:56.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:57.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:57.046 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:57.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:57.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:57.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:57.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:57.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:57.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 35910 35894 172.541 130.438 0.00293128 0.00579058 2024-08-17T17:47:57.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:57.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:57.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:57.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:57.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:57.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:57.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:57.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:57 smithi152 ceph-mon[93456]: pgmap v1680: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 170 MiB/s wr, 2.71k op/s 2024-08-17T17:47:57.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:57 smithi142 ceph-mon[106729]: pgmap v1680: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 170 MiB/s wr, 2.71k op/s 2024-08-17T17:47:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:47:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:57.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:57 smithi142 ceph-mon[103614]: pgmap v1680: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 170 MiB/s wr, 2.71k op/s 2024-08-17T17:47:57.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:57.993 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:57.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:58.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:58.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:58.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:58.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:58.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 37840 37824 168.832 120.625 0.00479263 0.00591823 2024-08-17T17:47:58.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:58.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:58.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:58.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:58.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:58.669 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:58.771 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:58.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:58.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:58.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:59.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:59.076 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:59.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:47:59.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:47:59.342 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 39773 39757 165.63 120.812 0.0093678 0.00603282 2024-08-17T17:47:59.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:59.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:59.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:59.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:47:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:59.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:59.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:47:59 smithi152 ceph-mon[93456]: pgmap v1681: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-08-17T17:47:59.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:47:59 smithi142 ceph-mon[106729]: pgmap v1681: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-08-17T17:47:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:47:59 smithi142 ceph-mon[103614]: pgmap v1681: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-08-17T17:47:59.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:47:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:00.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:00.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.140 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:00.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:00.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:00.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 41700 41684 162.805 120.438 0.0132017 0.00613761 2024-08-17T17:48:00.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:00.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:00.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:00.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:00.539 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:00.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:00.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:48:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:48:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:00.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.671 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.773 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:00.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:00.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:00.942 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:00.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:01.043 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:01.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:01.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:01.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:01.195 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:01.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:01.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:01.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 43599 43583 160.209 118.688 0.00721415 0.00623692 2024-08-17T17:48:01.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:01.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:01.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:01.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:01.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:01.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:01.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:01.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:01 smithi142 ceph-mon[106729]: pgmap v1682: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 159 MiB/s wr, 2.54k op/s 2024-08-17T17:48:01.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:01 smithi142 ceph-mon[103614]: pgmap v1682: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 159 MiB/s wr, 2.54k op/s 2024-08-17T17:48:01.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:01 smithi152 ceph-mon[93456]: pgmap v1682: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 159 MiB/s wr, 2.54k op/s 2024-08-17T17:48:01.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:02.002 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:02.026 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:48:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:48:01] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-08-17T17:48:02.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:02.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:02.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:02.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:02.308 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:02.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 45423 45407 157.642 114 0.0102896 0.00633797 2024-08-17T17:48:02.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:02.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:02.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:02.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:02.688 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:02.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:02.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:02.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:02.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:02.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:02.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:03.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:03.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:03.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:03.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:03.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 47227 47211 155.279 112.75 0.00470432 0.00643365 2024-08-17T17:48:03.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:03.406 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:03.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:03.527 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:48:03.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:03.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:03 smithi142 ceph-mon[106729]: pgmap v1683: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:48:03.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3955562863' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:48:03.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:03 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:48:03.616 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:03 smithi142 ceph-mon[103614]: pgmap v1683: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:48:03.616 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3955562863' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:48:03.617 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:03 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:48:03.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:03.687 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:03.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:03 smithi152 ceph-mon[93456]: pgmap v1683: 89 pgs: 89 active+clean; 2.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:48:03.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3955562863' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:48:03.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:03 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:48:03.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:03.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:03.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:04.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.094 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:04.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:04.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:04.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:48:04.343610+0000 min lat: 0.00119197 max lat: 0.119428 avg lat: 0.00655268 2024-08-17T17:48:04.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:48:04.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 48816 48800 152.48 99.3125 0.00543573 0.00655268 2024-08-17T17:48:04.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:04.375 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:04.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:04.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:04.548 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:48:04.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.646 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:48:04.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[103614]: osdmap e420: 8 total, 8 up, 6 in 2024-08-17T17:48:04.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3029657777' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.719 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:48:04.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[106729]: osdmap e420: 8 total, 8 up, 6 in 2024-08-17T17:48:04.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3029657777' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:04.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:04.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:48:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:04 smithi152 ceph-mon[93456]: osdmap e420: 8 total, 8 up, 6 in 2024-08-17T17:48:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3029657777' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:04.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:04.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:05.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:05.060 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:05.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:05.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:05.191 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:05.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:05.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:05.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:05.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 50517 50501 150.281 106.312 0.0088685 0.00664974 2024-08-17T17:48:05.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:05.463 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:05.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:05.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:05.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:05.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:05.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:05.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:05 smithi152 ceph-mon[93456]: pgmap v1685: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-08-17T17:48:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:05 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:05 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:48:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:05 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:05.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:05 smithi152 ceph-mon[93456]: osdmap e421: 8 total, 8 up, 6 in 2024-08-17T17:48:05.827 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:05.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[106729]: pgmap v1685: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[106729]: osdmap e421: 8 total, 8 up, 6 in 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[103614]: pgmap v1685: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-08-17T17:48:05.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:05.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:48:05.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:05.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:05 smithi142 ceph-mon[103614]: osdmap e421: 8 total, 8 up, 6 in 2024-08-17T17:48:05.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:48:05.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:06.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:06.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:06.148 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:48:06.148 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:48:06.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:06.269 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:06.302 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:06.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 52224 52208 148.299 106.688 0.00730027 0.00673847 2024-08-17T17:48:06.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:06.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:06.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:06.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:06.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:06.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:06.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:06.814 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:06.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:06 smithi152 ceph-mon[93456]: pgmap v1687: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:48:06.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:06 smithi142 ceph-mon[103614]: pgmap v1687: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:48:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:06 smithi142 ceph-mon[106729]: pgmap v1687: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-17T17:48:06.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:06.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:07.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:07.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:07.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:07.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:07.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:07.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:07.334 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:07.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 53859 53843 146.294 102.188 0.0117077 0.00683065 2024-08-17T17:48:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.371 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:07.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:07.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:48:07.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:48:07.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:07.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:07.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:07.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:07.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:07.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:07.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:07.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:07.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:07.957 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:08.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:08.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:08.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:08.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:08.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:08.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 55472 55456 144.398 100.812 0.0205899 0.00692048 2024-08-17T17:48:08.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:08.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:08.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:08.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:08.719 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:08.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:08.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:08.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:08.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:08.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:09.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:09.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:09.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:09.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:09.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:09.343 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 56907 56891 142.21 89.6875 0.00694072 0.00702622 2024-08-17T17:48:09.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:09.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:09.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:09.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:09.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:09.693 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:48:09.693 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:09.720 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:48:09.720 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:09.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:09.746 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:48:09.746 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:09.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:09.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:09 smithi142 ceph-mon[103614]: pgmap v1688: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T17:48:09.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:09 smithi142 ceph-mon[106729]: pgmap v1688: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T17:48:09.773 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:48:09.773 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:09.802 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:48:09.802 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:09 smithi152 ceph-mon[93456]: pgmap v1688: 89 pgs: 89 active+clean; 3.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T17:48:09.830 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:48:09.830 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:09.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:09.857 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:48:09.857 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:09.884 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:48:09.884 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:09.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:09.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.025 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:10.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:10.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.304 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.344 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 58489 58473 140.543 98.875 0.00736526 0.00711044 2024-08-17T17:48:10.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:10.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:10.520 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:48:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:48:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:10.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.575 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:10.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:10.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:10.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3643953715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:10.828 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3643953715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:10.828 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3643953715' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:10.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:10.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:10.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:10.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:11.073 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:11.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:11.125 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:11.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:11.256 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:11.344 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 60018 60002 138.876 95.5625 0.00858697 0.00719551 2024-08-17T17:48:11.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:11.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:11.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:11.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:11.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:11.559 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:11.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:11.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:11.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-mon[106729]: pgmap v1689: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:48:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-mon[103614]: pgmap v1689: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:48:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:11.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:11 smithi152 ceph-mon[93456]: pgmap v1689: 89 pgs: 89 active+clean; 3.4 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:48:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:11.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:11.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:12.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:12.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:12.097 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:48:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:48:11] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-08-17T17:48:12.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:12.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:12.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:12.344 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 61539 61523 137.311 95.0625 0.0133282 0.00727827 2024-08-17T17:48:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:12.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:12.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:12.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:12.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:12.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:12.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:12.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:12.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 4]} : dispatch 2024-08-17T17:48:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 4]}]': finished 2024-08-17T17:48:12.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:12 smithi152 ceph-mon[93456]: osdmap e422: 8 total, 8 up, 6 in 2024-08-17T17:48:12.846 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 4]} : dispatch 2024-08-17T17:48:12.846 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 4]}]': finished 2024-08-17T17:48:12.846 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:12 smithi142 ceph-mon[106729]: osdmap e422: 8 total, 8 up, 6 in 2024-08-17T17:48:12.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 4]} : dispatch 2024-08-17T17:48:12.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 4]}]': finished 2024-08-17T17:48:12.847 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:12 smithi142 ceph-mon[103614]: osdmap e422: 8 total, 8 up, 6 in 2024-08-17T17:48:12.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:12.908 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:13.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:13.039 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:13.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:13.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:13.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:13.299 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:13.344 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 63008 62992 135.742 91.8125 0.00406829 0.00736233 2024-08-17T17:48:13.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:13.441 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:13.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:13.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:48:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:13.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:13.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:13.719 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:13.802 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:13 smithi142 ceph-mon[106729]: pgmap v1691: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:48:13.803 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:13 smithi142 ceph-mon[106729]: osdmap e423: 8 total, 8 up, 6 in 2024-08-17T17:48:13.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:13 smithi142 ceph-mon[103614]: pgmap v1691: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:48:13.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:13 smithi142 ceph-mon[103614]: osdmap e423: 8 total, 8 up, 6 in 2024-08-17T17:48:13.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:13 smithi152 ceph-mon[93456]: pgmap v1691: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:48:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:13 smithi152 ceph-mon[93456]: osdmap e423: 8 total, 8 up, 6 in 2024-08-17T17:48:13.858 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:13.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:13.998 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:13.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:14.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:14.142 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:14.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.282 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:14.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:14.344 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 64419 64403 134.157 88.1875 0.0160736 0.00744915 2024-08-17T17:48:14.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:14.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:14.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:14.549 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:48:14.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:14.646 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:48:14.646 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:48:14.646 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:48:14.647 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:48:14.647 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:48:14.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:14.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:14.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:14.817 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3254563735' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:48:14.817 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:14 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:48:14.817 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3254563735' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:48:14.817 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:14 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:48:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3254563735' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:48:14.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:14 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:48:14.824 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:14.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:14.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:15.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:15.107 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:15.209 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:15.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:15.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:15.344 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 65864 65848 132.742 90.3125 0.0113752 0.00752829 2024-08-17T17:48:15.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:15.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:15.524 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:15.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:15.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:15.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:48:15.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:15.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:15.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:15 smithi152 ceph-mon[93456]: pgmap v1693: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:48:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:15 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:48:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:15 smithi152 ceph-mon[93456]: osdmap e424: 8 total, 8 up, 6 in 2024-08-17T17:48:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:15 smithi142 ceph-mon[106729]: pgmap v1693: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:48:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:15 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:48:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:15 smithi142 ceph-mon[106729]: osdmap e424: 8 total, 8 up, 6 in 2024-08-17T17:48:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:15 smithi142 ceph-mon[103614]: pgmap v1693: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:48:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:15 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:48:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:15 smithi142 ceph-mon[103614]: osdmap e424: 8 total, 8 up, 6 in 2024-08-17T17:48:15.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:15.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:16.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:16.079 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:16.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:16.180 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:16.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:16.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:16.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:48:16.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:16.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:16.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:16.345 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 16 67261 67245 131.321 87.3125 0.0146052 0.00761039 2024-08-17T17:48:16.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:16.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:16.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:16.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:16.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:16.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:16.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:16.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:16.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:16.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:16.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:16.939 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:16.952 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:17.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:17.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:17.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:17.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:17.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:17.345 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 68628 68612 129.927 85.4375 0.0102236 0.00769137 2024-08-17T17:48:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:17.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:17.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:17.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:17.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:17.635 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:17.641 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:17 smithi142 ceph-mon[106729]: pgmap v1695: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:48:17.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:17 smithi142 ceph-mon[103614]: pgmap v1695: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:48:17.737 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:17.774 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:17 smithi152 ceph-mon[93456]: pgmap v1695: 89 pgs: 89 active+clean; 3.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:48:17.876 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:17.899 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:17.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:18.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:18.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:18.345 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 69953 69937 128.541 82.8125 0.0125613 0.0077747 2024-08-17T17:48:18.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:18.464 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:18.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.720 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:18 smithi142 ceph-mon[106729]: pgmap v1696: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T17:48:18.720 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:18 smithi142 ceph-mon[103614]: pgmap v1696: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T17:48:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.880 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:19.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:19.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:19.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:18 smithi152 ceph-mon[93456]: pgmap v1696: 89 pgs: 89 active+clean; 4.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-17T17:48:19.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:19.162 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:19.190 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:19.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:19.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:19.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:19.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 71285 71269 127.247 83.25 0.00387463 0.00785427 2024-08-17T17:48:19.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:19.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:19.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:19.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:19.600 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:48:19.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:19.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:19.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:19.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2028416326' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2028416326' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:19.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:19.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2028416326' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:20.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:20.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:20.231 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:20.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.261 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:20.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:20.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 72620 72604 126.03 83.4375 0.0135633 0.00793007 2024-08-17T17:48:20.362 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:20.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:20.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:20.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:48:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:48:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:20.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:20.662 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:20.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:20.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2028416326' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[106729]: osdmap e425: 8 total, 8 up, 6 in 2024-08-17T17:48:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[106729]: pgmap v1698: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:48:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2028416326' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[103614]: osdmap e425: 8 total, 8 up, 6 in 2024-08-17T17:48:20.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:20 smithi142 ceph-mon[103614]: pgmap v1698: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:48:20.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:20.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:21.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:20 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2028416326' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:20 smithi152 ceph-mon[93456]: osdmap e425: 8 total, 8 up, 6 in 2024-08-17T17:48:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:20 smithi152 ceph-mon[93456]: pgmap v1698: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:48:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.081 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:21.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:21.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:21.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:21.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:21.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 73947 73931 124.865 82.9375 0.0113574 0.00800366 2024-08-17T17:48:21.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:21.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:21.493 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:21.595 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:21.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:21.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:21.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:21.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:21.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:48:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:48:21] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-08-17T17:48:21.900 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:22.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:22.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:22.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:22.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:22.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:22.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:22.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:22.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 75239 75223 123.704 80.75 0.00617278 0.00807915 2024-08-17T17:48:22.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:22.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:22.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:22.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:22.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:22.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:22.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:22.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:22.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:22.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:23.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:23.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:23.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:23.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:23.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:23.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:23.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:23.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 76508 76492 122.566 79.3125 0.0218653 0.00815417 2024-08-17T17:48:23.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.426 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:23.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:23.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:23.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:23.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:23.693 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:23.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:23.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:23.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:23 smithi152 ceph-mon[93456]: pgmap v1699: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:48:23.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:23 smithi142 ceph-mon[106729]: pgmap v1699: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:48:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:23 smithi142 ceph-mon[103614]: pgmap v1699: 89 pgs: 89 active+clean; 4.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:48:23.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:23.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:23.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:48:24.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:24.079 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:24.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:24.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:24.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:24.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:24.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:48:24.346622+0000 min lat: 0.00119197 max lat: 0.119428 avg lat: 0.00824216 2024-08-17T17:48:24.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:48:24.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 77619 77603 121.238 69.4375 0.0110122 0.00824216 2024-08-17T17:48:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.364 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:24.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:24.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:24.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:24.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:24.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:24.728 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:24.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:24.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:24.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:24.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:25.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:25.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:25.143 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:25.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:25.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:25.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 78848 78832 120.154 76.8125 0.00617571 0.00831813 2024-08-17T17:48:25.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:25.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:25.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:25.425 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:48:25.425 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:48:25.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:48:25.425 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:195031 12.1 e398/337 12:8fe5c2f9:::benchmark_data_smithi142_261410_object76834:head v 398'12382, mlcod=398'12380)", 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:20.021214+0000", 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 244.102944713, 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.185343119, 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.426 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021214+0000", 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021214+0000", 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021216+0000", 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.703e-06 2024-08-17T17:48:25.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021226+0000", 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0081999999999999e-05 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021246+0000", 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9890000000000001e-05 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021256+0000", 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0229e-05 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021317+0000", 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1567000000000002e-05 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.021334+0000", 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6611000000000001e-05 2024-08-17T17:48:25.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.206511+0000", 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.18517723699999999 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.206557+0000", 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5800000000000002e-05 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.430 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:195033 12.d 12:b82a675e:::benchmark_data_smithi142_261410_object76836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:20.048651+0000", 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 244.07550791899999, 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.17574040899999999, 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.431 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 195033 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.048651+0000", 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.048651+0000", 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.048653+0000", 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0880000000000002e-06 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.048662+0000", 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4730000000000006e-06 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204035+0000", 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.15537228 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204050+0000", 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5826999999999999e-05 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204129+0000", 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8217e-05 2024-08-17T17:48:25.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204722+0000", 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059323500000000001 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.212701+0000", 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079795279999999996 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.212816+0000", 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114138 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.212992+0000", 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000176393 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.222146+0000", 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0091541279999999992 2024-08-17T17:48:25.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.222195+0000", 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9285e-05 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.222204+0000", 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.952e-06 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.222255+0000", 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1227999999999998e-05 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.222321+0000", 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5594999999999998e-05 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.224391+0000", 2024-08-17T17:48:25.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020700419999999998 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:194835 12.7 e398/337 12:e9ca3b48:::benchmark_data_smithi142_261410_object76638:head v 398'12162, mlcod=324'722)", 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:20.078789+0000", 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 244.04536927300001, 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.141248925, 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.439 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.078789+0000", 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.078789+0000", 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.078791+0000", 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.406e-06 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.078802+0000", 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0832000000000001e-05 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204037+0000", 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.125235079 2024-08-17T17:48:25.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204050+0000", 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.341e-05 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204132+0000", 2024-08-17T17:48:25.442 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1433000000000005e-05 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.204162+0000", 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.0923000000000002e-05 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.219991+0000", 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015828181 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:20.220038+0000", 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7661000000000003e-05 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:195258 12.d 12:b733b66d:::benchmark_data_smithi142_261410_object77061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:21.077255+0000", 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 243.04690371300001, 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.088434659999999998, 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.444 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 195258 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077255+0000", 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077255+0000", 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077257+0000", 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2539999999999999e-06 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.446 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077267+0000", 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6560000000000004e-06 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077295+0000", 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.832e-05 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077321+0000", 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5778e-05 2024-08-17T17:48:25.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077370+0000", 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9209000000000003e-05 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.077926+0000", 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055585900000000002 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.154519+0000", 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.076592746000000003 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.154642+0000", 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000123411 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.154829+0000", 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018682400000000001 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.158447+0000", 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0036182470000000002 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.159702+0000", 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012543840000000001 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.161336+0000", 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001634358 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.162844+0000", 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0015079799999999999 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.162922+0000", 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7824000000000006e-05 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:21.165690+0000", 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027678099999999999 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230550 12.d 12:b9b3fad2:::benchmark_data_smithi142_261410_object112353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:19.953359+0000", 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 124.170799987, 2024-08-17T17:48:25.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.086343761000000005, 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230550 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.453 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953359+0000", 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953359+0000", 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953361+0000", 2024-08-17T17:48:25.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9599999999999999e-06 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953368+0000", 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5440000000000001e-06 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953383+0000", 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4761e-05 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953393+0000", 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6390000000000004e-06 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.953629+0000", 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00023594800000000001 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.955347+0000", 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001718114 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.973434+0000", 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.018087465 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.973640+0000", 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000206245 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.974021+0000", 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00038020600000000001 2024-08-17T17:48:25.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.990817+0000", 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.016796719000000002 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.995072+0000", 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042547829999999998 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.995369+0000", 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000297261 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.038916+0000", 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.043546965999999999 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.039032+0000", 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115697 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.459 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.039702+0000", 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000670453 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230955 12.d 12:b6067d3d:::benchmark_data_smithi142_261410_object112758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.679237+0000", 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 123.444921261, 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.099004928000000006, 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.460 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230955 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.679237+0000", 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.679237+0000", 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.679239+0000", 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.714e-06 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.679244+0000", 2024-08-17T17:48:25.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5600000000000004e-06 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.679260+0000", 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6540000000000001e-05 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.679267+0000", 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4139999999999999e-06 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.689509+0000", 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010241956999999999 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.691797+0000", 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0022885319999999998 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.717397+0000", 2024-08-17T17:48:25.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.025599407000000001 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.717493+0000", 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6587000000000002e-05 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.717654+0000", 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016099199999999999 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.722299+0000", 2024-08-17T17:48:25.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046445139999999998 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.725197+0000", 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028987349999999999 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.728522+0000", 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033248349999999999 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.777529+0000", 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.049006914999999998 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.777598+0000", 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9219000000000003e-05 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.778242+0000", 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00064400700000000004 2024-08-17T17:48:25.467 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232241 12.d 12:b6e882c7:::benchmark_data_smithi142_261410_object114044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.311435+0000", 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.812723791, 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.13312030999999999, 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.468 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232241 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311435+0000", 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311435+0000", 2024-08-17T17:48:25.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311436+0000", 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2470000000000001e-06 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311441+0000", 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4660000000000001e-06 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311450+0000", 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2070000000000001e-06 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311455+0000", 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1229999999999999e-06 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.311502+0000", 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7006999999999999e-05 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.312036+0000", 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00053454000000000002 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316011+0000", 2024-08-17T17:48:25.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039741660000000003 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316090+0000", 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9845999999999994e-05 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316210+0000", 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000119065 2024-08-17T17:48:25.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442912+0000", 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12670262299999999 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.443677+0000", 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00076503300000000003 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.443775+0000", 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7397000000000003e-05 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.443845+0000", 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0482999999999996e-05 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.444094+0000", 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024851799999999998 2024-08-17T17:48:25.475 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.444555+0000", 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00046158899999999999 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232243 12.6 12:6cb7e383:::benchmark_data_smithi142_261410_object114046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.316125+0000", 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.80803375399999, 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.13097766999999999, 2024-08-17T17:48:25.476 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232243 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316125+0000", 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316125+0000", 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316127+0000", 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9300000000000002e-06 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316132+0000", 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4589999999999997e-06 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316144+0000", 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.166e-05 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316151+0000", 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7220000000000002e-06 2024-08-17T17:48:25.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316202+0000", 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1656000000000001e-05 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316293+0000", 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0867999999999998e-05 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6,7", 2024-08-17T17:48:25.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316408+0000", 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000114478 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319071+0000", 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026634760000000001 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446272+0000", 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.127200747 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446338+0000", 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6305999999999998e-05 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446399+0000", 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1228000000000004e-05 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446510+0000", 2024-08-17T17:48:25.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000110121 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.447103+0000", 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059301899999999997 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232244 12.6 12:6b27abf4:::benchmark_data_smithi142_261410_object114047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.316759+0000", 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.807399698, 2024-08-17T17:48:25.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.137996325, 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232244 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316759+0000", 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316759+0000", 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316760+0000", 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0299999999999997e-07 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316764+0000", 2024-08-17T17:48:25.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.408e-06 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316776+0000", 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1620999999999999e-05 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.316783+0000", 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0099999999999998e-06 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442728+0000", 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.125945586 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442863+0000", 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013460699999999999 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,6,7", 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.443005+0000", 2024-08-17T17:48:25.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014194399999999999 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.447151+0000", 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041455260000000001 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.447195+0000", 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3954e-05 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.454474+0000", 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0072799700000000002 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.454536+0000", 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1273999999999996e-05 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.454614+0000", 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8141999999999998e-05 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.489 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.454755+0000", 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014138000000000001 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232245 12.d 12:b64eda22:::benchmark_data_smithi142_261410_object114048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.318393+0000", 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.80576540200001, 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.15765489299999999, 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.490 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232245 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.318393+0000", 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.318393+0000", 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.318394+0000", 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8100000000000001e-07 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.318398+0000", 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.126e-06 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.318409+0000", 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0259e-05 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.318414+0000", 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1479999999999997e-06 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442944+0000", 2024-08-17T17:48:25.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12453048 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.443613+0000", 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00066844099999999998 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.455539+0000", 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011925938000000001 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.455627+0000", 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8018999999999999e-05 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.455750+0000", 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012296799999999999 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.462911+0000", 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.007161789 2024-08-17T17:48:25.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.475383+0000", 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012471711 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.475429+0000", 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5640999999999999e-05 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.496 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.475441+0000", 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2472e-05 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.475529+0000", 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7896e-05 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.476048+0000", 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00051912400000000004 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.497 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.46715.0:232246 12.a e408/355 12:5f44d8ed:::benchmark_data_smithi142_261410_object114049:head v 408'14577, mlcod=408'14575)", 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.319361+0000", 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.804797452, 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.127948654, 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.498 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319361+0000", 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319361+0000", 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319362+0000", 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3500000000000005e-07 2024-08-17T17:48:25.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319369+0000", 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5590000000000002e-06 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442557+0000", 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12318789099999999 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442564+0000", 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9179999999999994e-06 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442629+0000", 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4035999999999995e-05 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442654+0000", 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5242e-05 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.447267+0000", 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046128879999999999 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.447310+0000", 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3185000000000001e-05 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.502 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:232243 12.6 e408/337 ondisk, result = 0)", 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.319745+0000", 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.804413498, 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12660510899999999, 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.503 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319745+0000", 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319745+0000", 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319746+0000", 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0399999999999996e-07 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319751+0000", 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8570000000000003e-06 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442553+0000", 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.122802308 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.505 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442564+0000", 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0893000000000001e-05 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446324+0000", 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037603720000000001 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446332+0000", 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0020000000000006e-06 2024-08-17T17:48:25.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446350+0000", 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8073000000000001e-05 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.46715.0:232243 12.6 e408/337 ondisk, result = 0)", 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.319762+0000", 2024-08-17T17:48:25.507 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 120.80439631599999, 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12651984499999999, 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319762+0000", 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.508 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319762+0000", 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319763+0000", 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7000000000000005e-07 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.319766+0000", 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.129e-06 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442522+0000", 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.122755559 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.442539+0000", 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7829000000000002e-05 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446252+0000", 2024-08-17T17:48:25.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037120909999999998 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446262+0000", 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0795000000000001e-05 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.446282+0000", 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9871999999999999e-05 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:233100 12.d 12:ba5646bb:::benchmark_data_smithi142_261410_object114903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:25.044019+0000", 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 119.080139859, 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.086722093, 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.512 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 233100 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044019+0000", 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044019+0000", 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044020+0000", 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.646e-06 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044026+0000", 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0619999999999997e-06 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044040+0000", 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3923e-05 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044048+0000", 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3189999999999997e-06 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044120+0000", 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2229000000000003e-05 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.515 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.044758+0000", 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00063777500000000004 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.108314+0000", 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.063556457999999996 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.108418+0000", 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000104217 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.108566+0000", 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014734899999999999 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.120729+0000", 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012162951999999999 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.129935+0000", 2024-08-17T17:48:25.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0092062069999999992 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.130007+0000", 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2571999999999996e-05 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.130198+0000", 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000190013 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.130242+0000", 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4681000000000001e-05 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.130741+0000", 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00049868999999999998 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.519 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:233140 12.d 12:b4ab2def:::benchmark_data_smithi142_261410_object114943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:25.118446+0000", 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 119.00571215799999, 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11826344799999999, 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 233140 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.520 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.118446+0000", 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.118446+0000", 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.118448+0000", 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.677e-06 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.521 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.118454+0000", 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6420000000000004e-06 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.118468+0000", 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.416e-05 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.118475+0000", 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8279999999999997e-06 2024-08-17T17:48:25.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.137485+0000", 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01901036 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.139021+0000", 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001535855 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.190964+0000", 2024-08-17T17:48:25.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.051942544 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.191056+0000", 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2455999999999997e-05 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.191192+0000", 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000135909 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.203650+0000", 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012458175 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.203697+0000", 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.706e-05 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.203712+0000", 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4681000000000001e-05 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.525 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.236175+0000", 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032462924999999997 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.236365+0000", 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000190426 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.236710+0000", 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00034475 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.526 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:233143 12.d 12:b50e7cdc:::benchmark_data_smithi142_261410_object114946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:25.119810+0000", 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 119.004348893, 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.117264272, 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.527 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 233143 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.119810+0000", 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.119810+0000", 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.119811+0000", 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1599999999999999e-06 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.119816+0000", 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7929999999999997e-06 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.119831+0000", 2024-08-17T17:48:25.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5628e-05 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.119839+0000", 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5480000000000004e-06 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.139055+0000", 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.019216586000000001 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.141098+0000", 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002042767 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.195455+0000", 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.054356429999999997 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.195561+0000", 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000106569 2024-08-17T17:48:25.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.195721+0000", 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016005899999999999 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.203731+0000", 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0080095470000000005 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.532 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.203779+0000", 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7954000000000002e-05 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.203833+0000", 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3757e-05 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.236761+0000", 2024-08-17T17:48:25.533 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.032928195 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.236814+0000", 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3584000000000001e-05 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.237074+0000", 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000259695 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.534 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:233146 12.d 12:bc3e6bc4:::benchmark_data_smithi142_261410_object114949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:25.127303+0000", 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 118.9968552, 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11012177200000001, 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.535 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 233146 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.127303+0000", 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.127303+0000", 2024-08-17T17:48:25.536 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.127305+0000", 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.559e-06 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.127310+0000", 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.853e-06 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.537 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.127326+0000", 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6010000000000001e-05 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.127334+0000", 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9710000000000004e-06 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.141138+0000", 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.013803678999999999 2024-08-17T17:48:25.538 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.142922+0000", 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001784491 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.199845+0000", 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.056923389999999997 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.539 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.199917+0000", 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1682999999999998e-05 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.200477+0000", 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055990300000000005 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.204517+0000", 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040396199999999998 2024-08-17T17:48:25.540 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.210226+0000", 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0057093420000000001 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.210301+0000", 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4590000000000002e-05 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.541 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.237127+0000", 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.026826406000000001 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.237179+0000", 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2493e-05 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.542 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.237425+0000", 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000245782 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:233261 12.d 12:bad92648:::benchmark_data_smithi142_261410_object115064:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:25.325440+0000", 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 118.798718745, 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.085877261999999996, 2024-08-17T17:48:25.543 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 233261 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325440+0000", 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.544 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325440+0000", 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325442+0000", 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9269999999999999e-06 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325449+0000", 2024-08-17T17:48:25.545 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3250000000000002e-06 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325471+0000", 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1413e-05 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325480+0000", 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8190000000000003e-06 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.546 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.325554+0000", 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3936000000000005e-05 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for degraded object", 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.326335+0000", 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00078028899999999998 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.350777+0000", 2024-08-17T17:48:25.547 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.024442379 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.350899+0000", 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012211500000000001 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4,7", 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.351057+0000", 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000158107 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.548 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.355336+0000", 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042792810000000002 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.357331+0000", 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0019946249999999999 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.357349+0000", 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8204999999999999e-05 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.549 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.410910+0000", 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.053560201000000002 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.410946+0000", 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6915000000000001e-05 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.411317+0000", 2024-08-17T17:48:25.550 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00037072500000000003 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:20109 13.a 13:55808814:::benchmark_data_smithi142_393845_object20108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:47:50.139347+0000", 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 33.984812032999997, 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.088285988999999995, 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:48:25.551 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 20109 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139347+0000", 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139347+0000", 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.552 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139348+0000", 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9599999999999998e-07 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139368+0000", 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0199000000000001e-05 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139379+0000", 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1527e-05 2024-08-17T17:48:25.553 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139386+0000", 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3380000000000001e-06 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139732+0000", 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00034605699999999998 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.554 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139851+0000", 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000118937 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139954+0000", 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000103702 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.141690+0000", 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0017356699999999999 2024-08-17T17:48:25.555 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.142471+0000", 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00078139400000000003 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.227535+0000", 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.085063642999999994 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.556 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.227591+0000", 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6459000000000003e-05 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.227633+0000", 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1167000000000001e-05 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:20111 13.a 13:5cff19a3:::benchmark_data_smithi142_393845_object20110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:47:50.140659+0000", 2024-08-17T17:48:25.557 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 33.983499690999999, 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.088426315000000005, 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 20111 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:48:25.558 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140659+0000", 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140659+0000", 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140660+0000", 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.108e-06 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:48:25.559 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140679+0000", 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9403999999999999e-05 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140691+0000", 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1588e-05 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140698+0000", 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7479999999999996e-06 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.560 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140754+0000", 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5831000000000001e-05 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140984+0000", 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022996599999999999 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.141102+0000", 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000118004 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.561 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.141865+0000", 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00076309299999999995 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.189188+0000", 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.047323523999999999 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.228963+0000", 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.039775053999999997 2024-08-17T17:48:25.562 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.229037+0000", 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3279000000000001e-05 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.229085+0000", 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8715999999999999e-05 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:48:25.563 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:48:25.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:25.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:25.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:25.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:25.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:25.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:25 smithi152 ceph-mon[93456]: pgmap v1700: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-08-17T17:48:25.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:25 smithi142 ceph-mon[106729]: pgmap v1700: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-08-17T17:48:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:25 smithi142 ceph-mon[103614]: pgmap v1700: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-08-17T17:48:25.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:25.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:25.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:26.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:26.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:26.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:26.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:26.339 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:26.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 79826 79810 118.748 61.125 0.00684373 0.00841627 2024-08-17T17:48:26.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:26.467 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:26.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:26.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:26.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:26.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:26.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.785 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:26.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:26.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:26.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:27.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:27.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:27.159 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:27.188 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:27.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:27.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:27.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 81005 80989 117.7 73.6875 0.00910145 0.0084908 2024-08-17T17:48:27.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:27.460 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:27.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:27.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:27.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:27.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:27 smithi152 ceph-mon[93456]: pgmap v1701: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:48:27.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:27 smithi142 ceph-mon[106729]: pgmap v1701: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:48:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:27 smithi142 ceph-mon[103614]: pgmap v1701: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:48:27.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:27.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:28.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:28.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:28.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:28.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:28.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:28.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 82209 82193 116.736 75.25 0.0182008 0.00856168 2024-08-17T17:48:28.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:28.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:28.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:28.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:28.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:28.687 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:28.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:28.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:28.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:28.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:28.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:29.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:29.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:29.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:29.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:29.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:29.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:29.341 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:48:29.341 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:29.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 83387 83371 115.777 73.625 0.0119486 0.00863227 2024-08-17T17:48:29.369 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:48:29.369 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:29.372 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:29.399 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:48:29.401 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:29.425 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:48:29.425 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:29.452 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:48:29.452 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:29.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:29.483 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:48:29.483 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.503 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:29.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:29.517 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:48:29.518 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:29.530 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:48:29.537 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[106729]: pgmap v1702: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:48:29.537 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/132449976' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:29.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[103614]: pgmap v1702: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:48:29.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/132449976' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:29.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1100348627' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:48:29.540 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:48:29.548 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:48:29.548 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:29.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:29.647 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:29.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:29.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:29.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:29 smithi152 ceph-mon[93456]: pgmap v1702: 89 pgs: 89 active+clean; 4.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:48:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/132449976' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:29 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1100348627' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:48:29.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:48:29.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1100348627' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:48:29.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:48:29.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:29.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:29.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:48:30.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:30.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:30.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:30.205 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:30.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:30.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.339 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:30.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:30.346 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 84544 84528 114.832 72.3125 0.014069 0.0087032 2024-08-17T17:48:30.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:30.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:30.540 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:48:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:48:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:30.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:30.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:30.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:30.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:30 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:48:30.625 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:30 smithi142 ceph-mon[106729]: osdmap e426: 8 total, 8 up, 6 in 2024-08-17T17:48:30.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:30 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:48:30.626 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:30 smithi142 ceph-mon[103614]: osdmap e426: 8 total, 8 up, 6 in 2024-08-17T17:48:30.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:30.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:30.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:30.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:48:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:30 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:48:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:30 smithi152 ceph-mon[93456]: osdmap e426: 8 total, 8 up, 6 in 2024-08-17T17:48:30.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:30.908 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:30.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:31.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:31.048 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:31.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:31.150 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:31.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:31.280 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:31.308 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:31.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:31.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 85712 85696 113.942 73 0.0103984 0.00877142 2024-08-17T17:48:31.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:31.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:31.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:31.536 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:31.541 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:31.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:31.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:31.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:31.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:31.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:31 smithi142 ceph-mon[103614]: pgmap v1704: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T17:48:31.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:31 smithi142 ceph-mon[106729]: pgmap v1704: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T17:48:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:31 smithi152 ceph-mon[93456]: pgmap v1704: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T17:48:31.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:31.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:31.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:31.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.058 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:48:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:48:31] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-08-17T17:48:32.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:32.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.242 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:32.267 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:32.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 86868 86852 113.074 72.25 0.0124424 0.00883957 2024-08-17T17:48:32.369 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:32.402 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:32.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:32.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:32.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:32.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:32.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:32.802 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:32.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:32 smithi142 ceph-mon[106729]: pgmap v1705: 89 pgs: 89 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-08-17T17:48:32.874 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:32 smithi142 ceph-mon[103614]: pgmap v1705: 89 pgs: 89 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-08-17T17:48:32.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:32.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:33.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:33.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:33.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:32 smithi152 ceph-mon[93456]: pgmap v1705: 89 pgs: 89 active+clean; 5.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-08-17T17:48:33.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:33.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:33.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:33.304 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:33.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:33.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 88003 87987 112.214 70.9375 0.0285794 0.00890652 2024-08-17T17:48:33.444 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:33.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:33.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:33.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:33.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:33.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:33.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:33.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:33.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:33.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:33.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:34.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:34.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:34.138 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:34.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:34.240 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:34.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:34.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 89140 89124 111.39 71.0625 0.0105846 0.00897198 2024-08-17T17:48:34.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:34.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:34.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:34.559 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:34.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:34.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:34.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:34.798 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:34.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:34.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:34.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:34.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:35.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:35.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:35.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:35.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:35.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:35.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:35.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 15 90234 90219 110.548 68.4375 0.00573006 0.00904025 2024-08-17T17:48:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.360 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:35.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:35.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:35.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:35.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:35.536 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:48:35.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:35 smithi142 ceph-mon[106729]: pgmap v1706: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:48:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2215573638' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:35 smithi142 ceph-mon[103614]: pgmap v1706: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:48:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2215573638' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:35.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:35.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:35.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:35 smithi152 ceph-mon[93456]: pgmap v1706: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:48:35.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2215573638' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:35.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:35.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:35.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:36.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:36.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:36.084 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:48:36.084 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:48:36.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:36.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:36.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:36.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:36.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 91352 91336 109.765 69.8125 0.0144909 0.00910563 2024-08-17T17:48:36.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:36.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:36.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:36.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:36.638 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:36.688 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:36.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:36.736 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:36 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:36.736 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2215573638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:36.737 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:36 smithi142 ceph-mon[103614]: osdmap e427: 8 total, 8 up, 6 in 2024-08-17T17:48:36.737 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:36 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:36.737 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2215573638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:36.737 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:36 smithi142 ceph-mon[106729]: osdmap e427: 8 total, 8 up, 6 in 2024-08-17T17:48:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:36 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2215573638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:36.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:36 smithi152 ceph-mon[93456]: osdmap e427: 8 total, 8 up, 6 in 2024-08-17T17:48:36.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:36.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:36.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:36.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:37.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:37.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:37.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:37.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:37.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:37.288 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:37.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 92445 92429 108.983 68.3125 0.00641848 0.00917087 2024-08-17T17:48:37.389 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:37.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:37.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:37.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:37.691 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:37.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:37.821 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:37 smithi152 ceph-mon[93456]: pgmap v1708: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:48:37.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:37 smithi142 ceph-mon[106729]: pgmap v1708: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:48:37.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:37 smithi142 ceph-mon[103614]: pgmap v1708: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:48:37.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:37.952 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:38.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:38.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:38.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:38.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.240 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:38.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:38.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:38.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 93570 93554 108.266 70.3125 0.013883 0.0092318 2024-08-17T17:48:38.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:38.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:38.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:38.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:38.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:38 smithi142 ceph-mon[106729]: pgmap v1709: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T17:48:38.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:38 smithi142 ceph-mon[103614]: pgmap v1709: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T17:48:38.653 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:38.755 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:38.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:38.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:48:38.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:38.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:38.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:39.033 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:39.069 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:38 smithi152 ceph-mon[93456]: pgmap v1709: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T17:48:39.171 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:39.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:39.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:39.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:39.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 94667 94651 107.544 68.5625 0.022409 0.00929375 2024-08-17T17:48:39.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:39.478 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:39.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:39.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:39.614 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:39.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:39.746 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:39.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:39.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:48:39.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:39.884 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:39.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.019 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:48:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.165 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:40.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:40.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:40.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1844004125' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:48:40.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:40 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:48:40.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:48:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:48:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:40.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 95679 95663 106.753 63.25 0.00452511 0.00936063 2024-08-17T17:48:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1844004125' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:48:40.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:40 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:48:40.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1844004125' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:48:40.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:40 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:48:40.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:40.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:40.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.569 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:40.837 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:40.887 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:48:40.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:40.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:41.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:41.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:41.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:48:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-mon[106729]: osdmap e428: 8 total, 8 up, 6 in 2024-08-17T17:48:41.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-mon[106729]: pgmap v1711: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:48:41.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:48:41.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-mon[103614]: osdmap e428: 8 total, 8 up, 6 in 2024-08-17T17:48:41.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-mon[103614]: pgmap v1711: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:48:41.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:41.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:41.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:41 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:48:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:41 smithi152 ceph-mon[93456]: osdmap e428: 8 total, 8 up, 6 in 2024-08-17T17:48:41.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:41 smithi152 ceph-mon[93456]: pgmap v1711: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T17:48:41.328 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:41.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:41.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 96594 96578 105.884 57.1875 0.0117871 0.0094398 2024-08-17T17:48:41.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.427 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:41.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:41.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:41.573 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:41.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:41.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:41.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:41.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:41.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:41.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:41.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:41.950 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:48:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:48:41] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:48:41.983 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:42.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:42.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:42.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:42.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:42.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:42.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:42.342 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:42.347 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 97639 97623 105.184 65.3125 0.00817133 0.00950196 2024-08-17T17:48:42.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:42.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:42.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:42.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:42.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:42.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:42.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:42.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:42.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:42.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:42.827 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:42.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:42.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:43.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.113 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:43 smithi152 ceph-mon[93456]: pgmap v1712: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T17:48:43.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 98675 98659 104.499 64.75 0.023648 0.00956428 2024-08-17T17:48:43.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:43 smithi142 ceph-mon[106729]: pgmap v1712: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T17:48:43.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:43 smithi142 ceph-mon[103614]: pgmap v1712: 89 pgs: 89 active+clean; 5.9 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T17:48:43.388 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:43.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:43.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:43.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:43.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:43.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:43.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:43.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:43.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:43.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:43.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:44.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:44.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:44.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/993749117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:44.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/993749117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:44.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:44.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:44.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:44.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/993749117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:44.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:48:44.348400+0000 min lat: 0.00119197 max lat: 0.239115 avg lat: 0.0096248 2024-08-17T17:48:44.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:48:44.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 99719 99703 103.844 65.25 0.0204159 0.0096248 2024-08-17T17:48:44.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:44.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:44.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:44.566 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:48:44.566 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:44.567 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:44.568 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:44.568 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:48:44.568 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:48:44.568 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:44.568 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:44.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:44.632 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:44.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:44.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:44.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:44.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:44.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:44.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:45.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:45.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:45.159 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:45.261 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.292 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:45.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:45.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:45 smithi152 ceph-mon[93456]: pgmap v1713: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:48:45.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 100710 100694 103.157 61.9375 0.0147542 0.00968892 2024-08-17T17:48:45.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:45 smithi142 ceph-mon[106729]: pgmap v1713: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:48:45.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:45 smithi142 ceph-mon[103614]: pgmap v1713: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T17:48:45.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.424 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:45.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:45.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:45.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:45.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:45.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:45.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:45.798 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:45.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:45.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:45.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:45.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:46.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:46.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:46.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:46.271 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:46.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:46.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:46 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:46.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 101636 101620 102.427 57.875 0.0151901 0.00975825 2024-08-17T17:48:46.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:46 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:46.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:46 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:46.373 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:46.403 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:46.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:46.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:46.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:46.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:46.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:46.811 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:46.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:46.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:46.953 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:46.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:47.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:47.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:47.191 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:47 smithi142 ceph-mon[106729]: pgmap v1714: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:48:47.191 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:47 smithi142 ceph-mon[103614]: pgmap v1714: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:48:47.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:47.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:47.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:47 smithi152 ceph-mon[93456]: pgmap v1714: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:48:47.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 102579 102563 101.737 58.9375 0.0208549 0.00982452 2024-08-17T17:48:47.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:47.411 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:47.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:47.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:47.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:47.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:47.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:47.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:47.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:47.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:47.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:47.994 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:47.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:48.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:48.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:48.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:48.204 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:48:48.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:48.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:48.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:48.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 103490 103474 101.037 56.9375 0.0152538 0.00989263 2024-08-17T17:48:48.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:48.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:48.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:48.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:48.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:48.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:48.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:48.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:48.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:48.869 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:48:48.870 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:48:48.870 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:48:48.870 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:48:48.870 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:48:48.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:48.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:49.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:49.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:49.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:49.191 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:49.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:49.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:49.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 104446 104430 100.401 59.75 0.010609 0.00995445 2024-08-17T17:48:49.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:49.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:49.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:49.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:49.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:49.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:49.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:49.749 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:49 smithi142 ceph-mon[106729]: pgmap v1715: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:48:49.750 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:49 smithi142 ceph-mon[103614]: pgmap v1715: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:48:49.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:49.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:49 smithi152 ceph-mon[93456]: pgmap v1715: 89 pgs: 89 active+clean; 6.2 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:48:49.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:49.903 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:50.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:50.052 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:50.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:50.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:50.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:50.284 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:50.319 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:50.322 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:48:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:48:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:50.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 105389 105373 99.7731 58.9375 0.0149876 0.0100179 2024-08-17T17:48:50.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:50.448 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:50.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:48:50.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:50.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:50.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:50.735 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:50.837 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.868 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:50.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:50.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:51.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:51.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:51.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:51.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:51.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:51.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:51.349 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 106311 106295 99.144 57.625 0.0100624 0.0100819 2024-08-17T17:48:51.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:51.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:51.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:51 smithi152 ceph-mon[93456]: pgmap v1716: 89 pgs: 89 active+clean; 6.3 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 62 MiB/s wr, 985 op/s 2024-08-17T17:48:51.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:51.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:51 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:51.548 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:51.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:51.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:51.674 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-mon[106729]: pgmap v1716: 89 pgs: 89 active+clean; 6.3 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 62 MiB/s wr, 985 op/s 2024-08-17T17:48:51.674 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:51.675 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:51.675 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-mon[103614]: pgmap v1716: 89 pgs: 89 active+clean; 6.3 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 62 MiB/s wr, 985 op/s 2024-08-17T17:48:51.675 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:51.675 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:48:51.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:51.781 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:51.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:51.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:51.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:51.956 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:48:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:48:51] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:48:52.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:52.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:52.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:52.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:52.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:52.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:52.349 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 107166 107150 98.4717 53.4375 0.0124998 0.0101496 2024-08-17T17:48:52.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:52.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:52.559 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:48:52.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:52.607 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:52.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:52.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:52.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:52.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:52.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:52 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3422712743' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:48:52.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:52 smithi152 ceph-mon[93456]: pgmap v1717: 89 pgs: 89 active+clean; 6.4 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 62 MiB/s wr, 999 op/s 2024-08-17T17:48:52.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:52.871 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:52 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3422712743' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:48:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:52 smithi142 ceph-mon[106729]: pgmap v1717: 89 pgs: 89 active+clean; 6.4 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 62 MiB/s wr, 999 op/s 2024-08-17T17:48:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:52 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3422712743' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:48:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:52 smithi142 ceph-mon[103614]: pgmap v1717: 89 pgs: 89 active+clean; 6.4 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 62 MiB/s wr, 999 op/s 2024-08-17T17:48:52.974 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:53.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:53.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:53.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:53.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.312 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:53.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:53.348 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 108180 108164 97.963 63.375 0.0149009 0.0102033 2024-08-17T17:48:53.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:53.440 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:53.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:53.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:53.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:53.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:53.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:53.814 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3422712743' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:48:53.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:53 smithi152 ceph-mon[93456]: osdmap e429: 8 total, 8 up, 6 in 2024-08-17T17:48:53.843 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3422712743' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:48:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:53 smithi142 ceph-mon[106729]: osdmap e429: 8 total, 8 up, 6 in 2024-08-17T17:48:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3422712743' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:48:53.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:53 smithi142 ceph-mon[103614]: osdmap e429: 8 total, 8 up, 6 in 2024-08-17T17:48:53.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:53.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:54.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:54.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:54.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:54.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:54.349 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 109165 109149 97.443 61.5625 0.0283509 0.010257 2024-08-17T17:48:54.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:54.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:54.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:54.534 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:54.573 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:48:54.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:54.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:54.680 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:48:54.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:48:54.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:54.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:54.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:54.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/430187749' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:54 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:54.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:54 smithi152 ceph-mon[93456]: pgmap v1719: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:48:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/430187749' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:54 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:54 smithi142 ceph-mon[106729]: pgmap v1719: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:48:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/430187749' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:54 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:54 smithi142 ceph-mon[103614]: pgmap v1719: 89 pgs: 89 active+clean; 6.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:48:54.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:54.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:55.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:55.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:55.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:55.221 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:55.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:55.349 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 110131 110115 96.9208 60.375 0.0120054 0.0103132 2024-08-17T17:48:55.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:55.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:55.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:55.501 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:55.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:55.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:55 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:55 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:55.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:55 smithi142 ceph-mon[106729]: osdmap e430: 8 total, 8 up, 6 in 2024-08-17T17:48:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:55 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:55.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:55 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:55.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:55 smithi142 ceph-mon[103614]: osdmap e430: 8 total, 8 up, 6 in 2024-08-17T17:48:55.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:55.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:55.785 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:55.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:55 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:55.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:55 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:55.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:55 smithi152 ceph-mon[93456]: osdmap e430: 8 total, 8 up, 6 in 2024-08-17T17:48:55.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.937 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:55.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:56.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:56.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:56.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:56.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:56.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:56.209 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:48:56.209 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:48:56.209 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:48:56.209 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:48:56.209 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:56.209 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:56.210 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:56.210 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:56.210 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:56.210 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:48:56.210 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:48:56.210 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:56.222 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:48:56.223 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:56.250 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:48:56.250 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:56.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:56.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:56.349 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 111108 111092 96.4227 61.0625 0.00581524 0.0103659 2024-08-17T17:48:56.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:56.471 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:56.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:56.528 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:56.573 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.610 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:56.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:56.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:56.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:56.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:56.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:56.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:56.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2094928513' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:56 smithi142 ceph-mon[106729]: pgmap v1721: 89 pgs: 89 active+clean; 6.6 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s 2024-08-17T17:48:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2094928513' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:56.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:56 smithi142 ceph-mon[103614]: pgmap v1721: 89 pgs: 89 active+clean; 6.6 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s 2024-08-17T17:48:56.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:57.007 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2094928513' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:56 smithi152 ceph-mon[93456]: pgmap v1721: 89 pgs: 89 active+clean; 6.6 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 53 MiB/s wr, 841 op/s 2024-08-17T17:48:57.108 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:57.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:57.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:57.290 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:57.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:57.349 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 112080 112064 95.934 60.75 0.0162488 0.0104196 2024-08-17T17:48:57.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:48:57.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:57.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:57.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:57.561 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:57.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:57.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:57.688 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:57.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:57.848 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:57.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:57.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:58.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:58.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:58.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:58.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:58.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:58.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:58.353 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 113051 113035 95.4522 60.6875 0.0219941 0.0104712 2024-08-17T17:48:58.385 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:58.486 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:58.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:58.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:58.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:58.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:58.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:58.754 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:58.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.786 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:58.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:58.801 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:58.838 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:58.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.923 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:58.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:59.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:59.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:59.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:59.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:59.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:59.353 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 114037 114021 95.0012 61.625 0.00835671 0.0105206 2024-08-17T17:48:59.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:59.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:59.627 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:48:59 smithi142 ceph-mon[106729]: pgmap v1722: 89 pgs: 89 active+clean; 6.8 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 58 MiB/s wr, 925 op/s 2024-08-17T17:48:59.628 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:48:59 smithi142 ceph-mon[103614]: pgmap v1722: 89 pgs: 89 active+clean; 6.8 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 58 MiB/s wr, 925 op/s 2024-08-17T17:48:59.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:59.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:59.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:59.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:48:59 smithi152 ceph-mon[93456]: pgmap v1722: 89 pgs: 89 active+clean; 6.8 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 58 MiB/s wr, 925 op/s 2024-08-17T17:48:59.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:48:59.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:48:59.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:00.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.142 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:00.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:00.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:00.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:00.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 114962 114946 94.5118 57.8125 0.0115464 0.0105758 2024-08-17T17:49:00.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:00.416 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:00.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:00.520 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:49:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:49:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:00.565 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:00.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:00.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:00.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:00.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:00.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:00.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:01.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:01.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:01.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:01.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:01.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:01.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:01.353 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 115798 115782 93.9629 52.25 0.0271536 0.0106375 2024-08-17T17:49:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.373 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:01.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:01.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.520 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:01.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:01.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:01.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:01.793 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:01 smithi152 ceph-mon[93456]: pgmap v1723: 89 pgs: 89 active+clean; 6.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:49:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:01.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:01.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:01 smithi142 ceph-mon[106729]: pgmap v1723: 89 pgs: 89 active+clean; 6.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:49:01.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:01 smithi142 ceph-mon[103614]: pgmap v1723: 89 pgs: 89 active+clean; 6.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:49:01.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:49:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:49:01] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:49:01.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:01.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:02.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:02.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:02.175 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:02.217 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:02.318 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:02.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:02.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 116681 116665 93.4658 55.1875 0.0412802 0.0106941 2024-08-17T17:49:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:02.404 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:02.404 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:02.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:02.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:02.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:02.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:02.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:02.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:02.844 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:02 smithi142 ceph-mon[103614]: pgmap v1724: 89 pgs: 89 active+clean; 7.0 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:49:02.845 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:02 smithi142 ceph-mon[106729]: pgmap v1724: 89 pgs: 89 active+clean; 7.0 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:49:02.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:02.904 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:03.006 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:02 smithi152 ceph-mon[93456]: pgmap v1724: 89 pgs: 89 active+clean; 7.0 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 60 MiB/s wr, 959 op/s 2024-08-17T17:49:03.141 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:03.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:03.207 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:03.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:03.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:03.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 117641 117625 93.0421 60 0.0258942 0.0107432 2024-08-17T17:49:03.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:03.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:03.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:03.561 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:03.608 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:49:03.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:03.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:03.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:03.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:03.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3184505194' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:03.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:49:03.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3184505194' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:03.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:03 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:49:03.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.965 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:03.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:03.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:49:04.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3184505194' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:03 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:03 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:03 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:49:04.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:04.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:04.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:04.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:04.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:49:04.354331+0000 min lat: 0.00119197 max lat: 0.239115 avg lat: 0.0107915 2024-08-17T17:49:04.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:49:04.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 118589 118573 92.6197 59.25 0.0157408 0.0107915 2024-08-17T17:49:04.375 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:04.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:04.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:04.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:04.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:04.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:04.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:04.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:04.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:04.786 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:04 smithi142 ceph-mon[103614]: osdmap e431: 8 total, 8 up, 6 in 2024-08-17T17:49:04.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:04 smithi142 ceph-mon[103614]: pgmap v1726: 89 pgs: 89 active+clean; 7.1 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s 2024-08-17T17:49:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:04.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:04 smithi142 ceph-mon[106729]: osdmap e431: 8 total, 8 up, 6 in 2024-08-17T17:49:04.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:04 smithi142 ceph-mon[106729]: pgmap v1726: 89 pgs: 89 active+clean; 7.1 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s 2024-08-17T17:49:04.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:04.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:05.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:04 smithi152 ceph-mon[93456]: osdmap e431: 8 total, 8 up, 6 in 2024-08-17T17:49:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:04 smithi152 ceph-mon[93456]: pgmap v1726: 89 pgs: 89 active+clean; 7.1 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s 2024-08-17T17:49:05.076 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:05.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:05.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:05.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:05.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:05.348 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:05.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 119518 119502 92.1931 58.0625 0.0227837 0.010842 2024-08-17T17:49:05.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:05.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:05.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:05.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:05.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:05.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:05.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:05.782 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:49:05.783 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:49:05.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:05.919 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:06.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:06.059 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:06.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:06.188 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:06.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:06.331 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:06.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 120409 120393 91.7478 55.6875 0.0115048 0.0108947 2024-08-17T17:49:06.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:06.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:06.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:06.566 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:06.591 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:06.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:06.693 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:06.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:06.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:06.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:06.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:06.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:49:06.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:06.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:07.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:07.124 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:07.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:07.226 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:07.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:07.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 121378 121362 91.3721 60.5625 0.0154155 0.0109396 2024-08-17T17:49:07.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:07.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:07.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:07.521 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:07.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:07.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:07.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:07.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:07.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:07 smithi152 ceph-mon[93456]: from='client.48089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:49:07.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:07 smithi152 ceph-mon[93456]: pgmap v1727: 89 pgs: 89 active+clean; 7.2 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s 2024-08-17T17:49:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:07 smithi142 ceph-mon[106729]: from='client.48089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:49:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:07 smithi142 ceph-mon[106729]: pgmap v1727: 89 pgs: 89 active+clean; 7.2 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s 2024-08-17T17:49:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:07 smithi142 ceph-mon[103614]: from='client.48089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:49:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:07 smithi142 ceph-mon[103614]: pgmap v1727: 89 pgs: 89 active+clean; 7.2 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s 2024-08-17T17:49:07.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:07.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:08.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:08.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:08.164 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:08.204 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:08.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:08.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:08.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 122336 122320 90.9971 59.875 0.0136991 0.0109849 2024-08-17T17:49:08.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:08.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:08.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:08.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:08.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:08.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:08.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:08.775 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1726,"stamp":"2024-08-17T17:49:04.481084+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7639865293,"num_objects":116592,"num_object_clones":0,"num_object_copies":349776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116592,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":116844,"num_write_kb":7462671,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2958510,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107473,"ondisk_log_size":107473,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":29418404,"kb_used_data":22254000,"kb_used_omap":27,"kb_used_meta":7164324,"kb_avail":533052508,"statfs":{"total":575970213888,"available":545845768192,"internally_reserved":0,"allocated":22788096000,"data_stored":22773905106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":7336268520},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":622395392,"num_objects":9497,"num_object_clones":0,"num_object_copies":28491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9497,"num_write_kb":607808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.912488"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":706,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572631+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:48:54.572631+0000","last_peered":"2024-08-17T17:48:54.572631+0000","last_clean":"2024-08-17T17:48:54.572631+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:48:54.572631+0000","last_undegraded":"2024-08-17T17:48:54.572631+0000","last_fullsized":"2024-08-17T17:48:54.572631+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":139,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569052+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:48:52.569052+0000","last_peered":"2024-08-17T17:48:52.569052+0000","last_clean":"2024-08-17T17:48:52.569052+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:48:52.569052+0000","last_undegraded":"2024-08-17T17:48:52.569052+0000","last_fullsized":"2024-08-17T17:48:52.569052+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":333,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.564299+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:48:52.564299+0000","last_peered":"2024-08-17T17:48:52.564299+0000","last_clean":"2024-08-17T17:48:52.564299+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:48:52.564299+0000","last_undegraded":"2024-08-17T17:48:52.564299+0000","last_fullsized":"2024-08-17T17:48:52.564299+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":404,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572640+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:48:54.572640+0000","last_peered":"2024-08-17T17:48:54.572640+0000","last_clean":"2024-08-17T17:48:54.572640+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:48:54.572640+0000","last_undegraded":"2024-08-17T17:48:54.572640+0000","last_fullsized":"2024-08-17T17:48:54.572640+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":581,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.573113+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:48:54.573113+0000","last_peered":"2024-08-17T17:48:54.573113+0000","last_clean":"2024-08-17T17:48:54.573113+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:48:54.573113+0000","last_undegraded":"2024-08-17T17:48:54.573113+0000","last_fullsized":"2024-08-17T17:48:54.573113+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":282,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572993+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:48:52.572993+0000","last_peered":"2024-08-17T17:48:52.572993+0000","last_clean":"2024-08-17T17:48:52.572993+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:48:52.572993+0000","last_undegraded":"2024-08-17T17:48:52.572993+0000","last_fullsized":"2024-08-17T17:48:52.572993+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":675,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572645+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:48:54.572645+0000","last_peered":"2024-08-17T17:48:54.572645+0000","last_clean":"2024-08-17T17:48:54.572645+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:48:54.572645+0000","last_undegraded":"2024-08-17T17:48:54.572645+0000","last_fullsized":"2024-08-17T17:48:54.572645+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":450,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.567475+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:48:52.567475+0000","last_peered":"2024-08-17T17:48:52.567475+0000","last_clean":"2024-08-17T17:48:52.567475+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:48:52.567475+0000","last_undegraded":"2024-08-17T17:48:52.567475+0000","last_fullsized":"2024-08-17T17:48:52.567475+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"430'7221","reported_seq":7246,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.422319+0000","last_change":"2024-08-17T17:47:33.474484+0000","last_active":"2024-08-17T17:49:02.422319+0000","last_peered":"2024-08-17T17:49:02.422319+0000","last_clean":"2024-08-17T17:49:02.422319+0000","last_became_active":"2024-08-17T17:47:33.474193+0000","last_became_peered":"2024-08-17T17:47:33.474193+0000","last_unstale":"2024-08-17T17:49:02.422319+0000","last_undegraded":"2024-08-17T17:49:02.422319+0000","last_fullsized":"2024-08-17T17:49:02.422319+0000","mapping_epoch":413,"log_start":"418'200","ondisk_log_start":"418'200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:23.071916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473235456,"num_objects":7221,"num_object_clones":0,"num_object_copies":21663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7221,"num_write_kb":462144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":135,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.570330+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:48:52.570330+0000","last_peered":"2024-08-17T17:48:52.570330+0000","last_clean":"2024-08-17T17:48:52.570330+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:48:52.570330+0000","last_undegraded":"2024-08-17T17:48:52.570330+0000","last_fullsized":"2024-08-17T17:48:52.570330+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":316,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569318+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:48:52.569318+0000","last_peered":"2024-08-17T17:48:52.569318+0000","last_clean":"2024-08-17T17:48:52.569318+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:48:52.569318+0000","last_undegraded":"2024-08-17T17:48:52.569318+0000","last_fullsized":"2024-08-17T17:48:52.569318+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"431'7351","reported_seq":7397,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:04.167646+0000","last_change":"2024-08-17T17:47:33.469969+0000","last_active":"2024-08-17T17:49:04.167646+0000","last_peered":"2024-08-17T17:49:04.167646+0000","last_clean":"2024-08-17T17:49:04.167646+0000","last_became_active":"2024-08-17T17:47:33.469675+0000","last_became_peered":"2024-08-17T17:47:33.469675+0000","last_unstale":"2024-08-17T17:49:04.167646+0000","last_undegraded":"2024-08-17T17:49:04.167646+0000","last_fullsized":"2024-08-17T17:49:04.167646+0000","mapping_epoch":413,"log_start":"419'1400","ondisk_log_start":"419'1400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:24.678241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":481755136,"num_objects":7351,"num_object_clones":0,"num_object_copies":22053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7351,"num_write_kb":470464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":137,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.571594+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:48:52.571594+0000","last_peered":"2024-08-17T17:48:52.571594+0000","last_clean":"2024-08-17T17:48:52.571594+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:48:52.571594+0000","last_undegraded":"2024-08-17T17:48:52.571594+0000","last_fullsized":"2024-08-17T17:48:52.571594+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":401,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572924+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:48:52.572924+0000","last_peered":"2024-08-17T17:48:52.572924+0000","last_clean":"2024-08-17T17:48:52.572924+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:48:52.572924+0000","last_undegraded":"2024-08-17T17:48:52.572924+0000","last_fullsized":"2024-08-17T17:48:52.572924+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"430'7105","reported_seq":7150,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:00.122808+0000","last_change":"2024-08-17T17:47:33.474669+0000","last_active":"2024-08-17T17:49:00.122808+0000","last_peered":"2024-08-17T17:49:00.122808+0000","last_clean":"2024-08-17T17:49:00.122808+0000","last_became_active":"2024-08-17T17:47:33.474373+0000","last_became_peered":"2024-08-17T17:47:33.474373+0000","last_unstale":"2024-08-17T17:49:00.122808+0000","last_undegraded":"2024-08-17T17:49:00.122808+0000","last_fullsized":"2024-08-17T17:49:00.122808+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7105,"log_dups_size":0,"ondisk_log_size":7105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:34:23.147112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":465633280,"num_objects":7105,"num_object_clones":0,"num_object_copies":21315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7105,"num_write_kb":454720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":133,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.564231+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:48:52.564231+0000","last_peered":"2024-08-17T17:48:52.564231+0000","last_clean":"2024-08-17T17:48:52.564231+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:48:52.564231+0000","last_undegraded":"2024-08-17T17:48:52.564231+0000","last_fullsized":"2024-08-17T17:48:52.564231+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":411,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.567050+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:48:52.567050+0000","last_peered":"2024-08-17T17:48:52.567050+0000","last_clean":"2024-08-17T17:48:52.567050+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:48:52.567050+0000","last_undegraded":"2024-08-17T17:48:52.567050+0000","last_fullsized":"2024-08-17T17:48:52.567050+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"430'7424","reported_seq":7468,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.067536+0000","last_change":"2024-08-17T17:47:33.471249+0000","last_active":"2024-08-17T17:49:03.067536+0000","last_peered":"2024-08-17T17:49:03.067536+0000","last_clean":"2024-08-17T17:49:03.067536+0000","last_became_active":"2024-08-17T17:47:33.470968+0000","last_became_peered":"2024-08-17T17:47:33.470968+0000","last_unstale":"2024-08-17T17:49:03.067536+0000","last_undegraded":"2024-08-17T17:49:03.067536+0000","last_fullsized":"2024-08-17T17:49:03.067536+0000","mapping_epoch":413,"log_start":"418'600","ondisk_log_start":"418'600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:45.656151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":486539264,"num_objects":7424,"num_object_clones":0,"num_object_copies":22272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7424,"num_write_kb":475136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":618,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572755+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:48:54.572755+0000","last_peered":"2024-08-17T17:48:54.572755+0000","last_clean":"2024-08-17T17:48:54.572755+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:48:54.572755+0000","last_undegraded":"2024-08-17T17:48:54.572755+0000","last_fullsized":"2024-08-17T17:48:54.572755+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":384,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.575845+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:48:54.575845+0000","last_peered":"2024-08-17T17:48:54.575845+0000","last_clean":"2024-08-17T17:48:54.575845+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:48:54.575845+0000","last_undegraded":"2024-08-17T17:48:54.575845+0000","last_fullsized":"2024-08-17T17:48:54.575845+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"430'7295","reported_seq":7320,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.110025+0000","last_change":"2024-08-17T17:47:33.472177+0000","last_active":"2024-08-17T17:49:02.110025+0000","last_peered":"2024-08-17T17:49:02.110025+0000","last_clean":"2024-08-17T17:49:02.110025+0000","last_became_active":"2024-08-17T17:47:33.471903+0000","last_became_peered":"2024-08-17T17:47:33.471903+0000","last_unstale":"2024-08-17T17:49:02.110025+0000","last_undegraded":"2024-08-17T17:49:02.110025+0000","last_fullsized":"2024-08-17T17:49:02.110025+0000","mapping_epoch":413,"log_start":"418'1000","ondisk_log_start":"418'1000","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6295,"log_dups_size":0,"ondisk_log_size":6295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:39:03.482291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":478085120,"num_objects":7295,"num_object_clones":0,"num_object_copies":21885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7295,"num_write_kb":466880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":708,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572990+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:48:54.572990+0000","last_peered":"2024-08-17T17:48:54.572990+0000","last_clean":"2024-08-17T17:48:54.572990+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:48:54.572990+0000","last_undegraded":"2024-08-17T17:48:54.572990+0000","last_fullsized":"2024-08-17T17:48:54.572990+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":384,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572612+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:48:52.572612+0000","last_peered":"2024-08-17T17:48:52.572612+0000","last_clean":"2024-08-17T17:48:52.572612+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:48:52.572612+0000","last_undegraded":"2024-08-17T17:48:52.572612+0000","last_fullsized":"2024-08-17T17:48:52.572612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"430'7276","reported_seq":7300,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.421127+0000","last_change":"2024-08-17T17:47:33.472785+0000","last_active":"2024-08-17T17:49:02.421127+0000","last_peered":"2024-08-17T17:49:02.421127+0000","last_clean":"2024-08-17T17:49:02.421127+0000","last_became_active":"2024-08-17T17:47:33.472497+0000","last_became_peered":"2024-08-17T17:47:33.472497+0000","last_unstale":"2024-08-17T17:49:02.421127+0000","last_undegraded":"2024-08-17T17:49:02.421127+0000","last_fullsized":"2024-08-17T17:49:02.421127+0000","mapping_epoch":413,"log_start":"418'200","ondisk_log_start":"418'200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:39:33.059184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":476839936,"num_objects":7276,"num_object_clones":0,"num_object_copies":21828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7276,"num_write_kb":465664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":385,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.567171+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:48:52.567171+0000","last_peered":"2024-08-17T17:48:52.567171+0000","last_clean":"2024-08-17T17:48:52.567171+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:48:52.567171+0000","last_undegraded":"2024-08-17T17:48:52.567171+0000","last_fullsized":"2024-08-17T17:48:52.567171+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":411,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569820+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:48:52.569820+0000","last_peered":"2024-08-17T17:48:52.569820+0000","last_clean":"2024-08-17T17:48:52.569820+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:48:52.569820+0000","last_undegraded":"2024-08-17T17:48:52.569820+0000","last_fullsized":"2024-08-17T17:48:52.569820+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"430'7372","reported_seq":7397,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.394987+0000","last_change":"2024-08-17T17:47:33.473418+0000","last_active":"2024-08-17T17:49:02.394987+0000","last_peered":"2024-08-17T17:49:02.394987+0000","last_clean":"2024-08-17T17:49:02.394987+0000","last_became_active":"2024-08-17T17:47:33.473144+0000","last_became_peered":"2024-08-17T17:47:33.473144+0000","last_unstale":"2024-08-17T17:49:02.394987+0000","last_undegraded":"2024-08-17T17:49:02.394987+0000","last_fullsized":"2024-08-17T17:49:02.394987+0000","mapping_epoch":413,"log_start":"418'300","ondisk_log_start":"418'300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:19:35.223379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483131392,"num_objects":7372,"num_object_clones":0,"num_object_copies":22116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7372,"num_write_kb":471808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":422,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.573130+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:48:52.573130+0000","last_peered":"2024-08-17T17:48:52.573130+0000","last_clean":"2024-08-17T17:48:52.573130+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:48:52.573130+0000","last_undegraded":"2024-08-17T17:48:52.573130+0000","last_fullsized":"2024-08-17T17:48:52.573130+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":384,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.571718+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:48:52.571718+0000","last_peered":"2024-08-17T17:48:52.571718+0000","last_clean":"2024-08-17T17:48:52.571718+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:48:52.571718+0000","last_undegraded":"2024-08-17T17:48:52.571718+0000","last_fullsized":"2024-08-17T17:48:52.571718+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"430'7367","reported_seq":7392,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.143410+0000","last_change":"2024-08-17T17:47:33.473396+0000","last_active":"2024-08-17T17:49:02.143410+0000","last_peered":"2024-08-17T17:49:02.143410+0000","last_clean":"2024-08-17T17:49:02.143410+0000","last_became_active":"2024-08-17T17:47:33.472959+0000","last_became_peered":"2024-08-17T17:47:33.472959+0000","last_unstale":"2024-08-17T17:49:02.143410+0000","last_undegraded":"2024-08-17T17:49:02.143410+0000","last_fullsized":"2024-08-17T17:49:02.143410+0000","mapping_epoch":413,"log_start":"418'1100","ondisk_log_start":"418'1100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6267,"log_dups_size":0,"ondisk_log_size":6267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:26.187848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":482803712,"num_objects":7367,"num_object_clones":0,"num_object_copies":22101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7367,"num_write_kb":471488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":677,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572157+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:48:54.572157+0000","last_peered":"2024-08-17T17:48:54.572157+0000","last_clean":"2024-08-17T17:48:54.572157+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:48:54.572157+0000","last_undegraded":"2024-08-17T17:48:54.572157+0000","last_fullsized":"2024-08-17T17:48:54.572157+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":614,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572704+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:48:54.572704+0000","last_peered":"2024-08-17T17:48:54.572704+0000","last_clean":"2024-08-17T17:48:54.572704+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:48:54.572704+0000","last_undegraded":"2024-08-17T17:48:54.572704+0000","last_fullsized":"2024-08-17T17:48:54.572704+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.c","version":"431'7475","reported_seq":7522,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:04.149976+0000","last_change":"2024-08-17T17:47:33.468618+0000","last_active":"2024-08-17T17:49:04.149976+0000","last_peered":"2024-08-17T17:49:04.149976+0000","last_clean":"2024-08-17T17:49:04.149976+0000","last_became_active":"2024-08-17T17:47:33.468320+0000","last_became_peered":"2024-08-17T17:47:33.468320+0000","last_unstale":"2024-08-17T17:49:04.149976+0000","last_undegraded":"2024-08-17T17:49:04.149976+0000","last_fullsized":"2024-08-17T17:49:04.149976+0000","mapping_epoch":413,"log_start":"419'1500","ondisk_log_start":"419'1500","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:12.338465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":489881600,"num_objects":7475,"num_object_clones":0,"num_object_copies":22425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7475,"num_write_kb":478400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":821,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577997+0000","last_change":"2024-08-17T17:47:14.531821+0000","last_active":"2024-08-17T17:48:54.577997+0000","last_peered":"2024-08-17T17:48:54.577997+0000","last_clean":"2024-08-17T17:48:54.577997+0000","last_became_active":"2024-08-17T17:47:14.531354+0000","last_became_peered":"2024-08-17T17:47:14.531354+0000","last_unstale":"2024-08-17T17:48:54.577997+0000","last_undegraded":"2024-08-17T17:48:54.577997+0000","last_fullsized":"2024-08-17T17:48:54.577997+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:54.544598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":365,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.566362+0000","last_change":"2024-08-17T17:46:26.863739+0000","last_active":"2024-08-17T17:48:52.566362+0000","last_peered":"2024-08-17T17:48:52.566362+0000","last_clean":"2024-08-17T17:48:52.566362+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:48:52.566362+0000","last_undegraded":"2024-08-17T17:48:52.566362+0000","last_fullsized":"2024-08-17T17:48:52.566362+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":657,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572559+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:48:54.572559+0000","last_peered":"2024-08-17T17:48:54.572559+0000","last_clean":"2024-08-17T17:48:54.572559+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:48:54.572559+0000","last_undegraded":"2024-08-17T17:48:54.572559+0000","last_fullsized":"2024-08-17T17:48:54.572559+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":394,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.570674+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:48:52.570674+0000","last_peered":"2024-08-17T17:48:52.570674+0000","last_clean":"2024-08-17T17:48:52.570674+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:48:52.570674+0000","last_undegraded":"2024-08-17T17:48:52.570674+0000","last_fullsized":"2024-08-17T17:48:52.570674+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"430'7282","reported_seq":7307,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.420415+0000","last_change":"2024-08-17T17:47:33.474825+0000","last_active":"2024-08-17T17:49:02.420415+0000","last_peered":"2024-08-17T17:49:02.420415+0000","last_clean":"2024-08-17T17:49:02.420415+0000","last_became_active":"2024-08-17T17:47:33.474247+0000","last_became_peered":"2024-08-17T17:47:33.474247+0000","last_unstale":"2024-08-17T17:49:02.420415+0000","last_undegraded":"2024-08-17T17:49:02.420415+0000","last_fullsized":"2024-08-17T17:49:02.420415+0000","mapping_epoch":413,"log_start":"418'300","ondisk_log_start":"418'300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:46:34.619479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477233152,"num_objects":7282,"num_object_clones":0,"num_object_copies":21846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7282,"num_write_kb":466048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":475,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.604954+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:49:03.604954+0000","last_peered":"2024-08-17T17:49:03.604954+0000","last_clean":"2024-08-17T17:49:03.604954+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:49:03.604954+0000","last_undegraded":"2024-08-17T17:49:03.604954+0000","last_fullsized":"2024-08-17T17:49:03.604954+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":366,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572794+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:48:52.572794+0000","last_peered":"2024-08-17T17:48:52.572794+0000","last_clean":"2024-08-17T17:48:52.572794+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:48:52.572794+0000","last_undegraded":"2024-08-17T17:48:52.572794+0000","last_fullsized":"2024-08-17T17:48:52.572794+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":399,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.575924+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:48:54.575924+0000","last_peered":"2024-08-17T17:48:54.575924+0000","last_clean":"2024-08-17T17:48:54.575924+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:48:54.575924+0000","last_undegraded":"2024-08-17T17:48:54.575924+0000","last_fullsized":"2024-08-17T17:48:54.575924+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"430'7146","reported_seq":7171,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:00.001286+0000","last_change":"2024-08-17T17:47:33.471881+0000","last_active":"2024-08-17T17:49:00.001286+0000","last_peered":"2024-08-17T17:49:00.001286+0000","last_clean":"2024-08-17T17:49:00.001286+0000","last_became_active":"2024-08-17T17:47:33.471614+0000","last_became_peered":"2024-08-17T17:47:33.471614+0000","last_unstale":"2024-08-17T17:49:00.001286+0000","last_undegraded":"2024-08-17T17:49:00.001286+0000","last_fullsized":"2024-08-17T17:49:00.001286+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7146,"log_dups_size":0,"ondisk_log_size":7146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:40.873776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":468320256,"num_objects":7146,"num_object_clones":0,"num_object_copies":21438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7146,"num_write_kb":457344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":565,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572484+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:48:54.572484+0000","last_peered":"2024-08-17T17:48:54.572484+0000","last_clean":"2024-08-17T17:48:54.572484+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:48:54.572484+0000","last_undegraded":"2024-08-17T17:48:54.572484+0000","last_fullsized":"2024-08-17T17:48:54.572484+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":166,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.567353+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:48:52.567353+0000","last_peered":"2024-08-17T17:48:52.567353+0000","last_clean":"2024-08-17T17:48:52.567353+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:48:52.567353+0000","last_undegraded":"2024-08-17T17:48:52.567353+0000","last_fullsized":"2024-08-17T17:48:52.567353+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":17,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569955+0000","last_change":"2024-08-17T17:48:13.475561+0000","last_active":"2024-08-17T17:48:52.569955+0000","last_peered":"2024-08-17T17:48:52.569955+0000","last_clean":"2024-08-17T17:48:52.569955+0000","last_became_active":"2024-08-17T17:48:13.475084+0000","last_became_peered":"2024-08-17T17:48:13.475084+0000","last_unstale":"2024-08-17T17:48:52.569955+0000","last_undegraded":"2024-08-17T17:48:52.569955+0000","last_fullsized":"2024-08-17T17:48:52.569955+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:23.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.d","version":"430'7247","reported_seq":7272,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:02.174215+0000","last_change":"2024-08-17T17:47:33.473230+0000","last_active":"2024-08-17T17:49:02.174215+0000","last_peered":"2024-08-17T17:49:02.174215+0000","last_clean":"2024-08-17T17:49:02.174215+0000","last_became_active":"2024-08-17T17:47:33.472924+0000","last_became_peered":"2024-08-17T17:47:33.472924+0000","last_unstale":"2024-08-17T17:49:02.174215+0000","last_undegraded":"2024-08-17T17:49:02.174215+0000","last_fullsized":"2024-08-17T17:49:02.174215+0000","mapping_epoch":413,"log_start":"418'900","ondisk_log_start":"418'900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:58.863358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474939392,"num_objects":7247,"num_object_clones":0,"num_object_copies":21741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7247,"num_write_kb":463808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":543,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603140+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:49:03.603140+0000","last_peered":"2024-08-17T17:49:03.603140+0000","last_clean":"2024-08-17T17:49:03.603140+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:49:03.603140+0000","last_undegraded":"2024-08-17T17:49:03.603140+0000","last_fullsized":"2024-08-17T17:49:03.603140+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":405,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569181+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:48:52.569181+0000","last_peered":"2024-08-17T17:48:52.569181+0000","last_clean":"2024-08-17T17:48:52.569181+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:48:52.569181+0000","last_undegraded":"2024-08-17T17:48:52.569181+0000","last_fullsized":"2024-08-17T17:48:52.569181+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":336,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572552+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:48:54.572552+0000","last_peered":"2024-08-17T17:48:54.572552+0000","last_clean":"2024-08-17T17:48:54.572552+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:48:54.572552+0000","last_undegraded":"2024-08-17T17:48:54.572552+0000","last_fullsized":"2024-08-17T17:48:54.572552+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"430'7239","reported_seq":7284,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.074171+0000","last_change":"2024-08-17T17:47:33.471160+0000","last_active":"2024-08-17T17:49:03.074171+0000","last_peered":"2024-08-17T17:49:03.074171+0000","last_clean":"2024-08-17T17:49:03.074171+0000","last_became_active":"2024-08-17T17:47:33.470939+0000","last_became_peered":"2024-08-17T17:47:33.470939+0000","last_unstale":"2024-08-17T17:49:03.074171+0000","last_undegraded":"2024-08-17T17:49:03.074171+0000","last_fullsized":"2024-08-17T17:49:03.074171+0000","mapping_epoch":413,"log_start":"418'400","ondisk_log_start":"418'400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:09.156891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474415104,"num_objects":7239,"num_object_clones":0,"num_object_copies":21717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7239,"num_write_kb":463296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":344,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.564145+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:48:52.564145+0000","last_peered":"2024-08-17T17:48:52.564145+0000","last_clean":"2024-08-17T17:48:52.564145+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:48:52.564145+0000","last_undegraded":"2024-08-17T17:48:52.564145+0000","last_fullsized":"2024-08-17T17:48:52.564145+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":205,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572462+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:48:54.572462+0000","last_peered":"2024-08-17T17:48:54.572462+0000","last_clean":"2024-08-17T17:48:54.572462+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:48:54.572462+0000","last_undegraded":"2024-08-17T17:48:54.572462+0000","last_fullsized":"2024-08-17T17:48:54.572462+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":702,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603382+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:49:03.603382+0000","last_peered":"2024-08-17T17:49:03.603382+0000","last_clean":"2024-08-17T17:49:03.603382+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:49:03.603382+0000","last_undegraded":"2024-08-17T17:49:03.603382+0000","last_fullsized":"2024-08-17T17:49:03.603382+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"431'7337","reported_seq":7384,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:04.150139+0000","last_change":"2024-08-17T17:47:33.468618+0000","last_active":"2024-08-17T17:49:04.150139+0000","last_peered":"2024-08-17T17:49:04.150139+0000","last_clean":"2024-08-17T17:49:04.150139+0000","last_became_active":"2024-08-17T17:47:33.468343+0000","last_became_peered":"2024-08-17T17:47:33.468343+0000","last_unstale":"2024-08-17T17:49:04.150139+0000","last_undegraded":"2024-08-17T17:49:04.150139+0000","last_fullsized":"2024-08-17T17:49:04.150139+0000","mapping_epoch":413,"log_start":"419'1400","ondisk_log_start":"419'1400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5937,"log_dups_size":0,"ondisk_log_size":5937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:35:12.974392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480837632,"num_objects":7337,"num_object_clones":0,"num_object_copies":22011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7337,"num_write_kb":469568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":340,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.568874+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:48:52.568874+0000","last_peered":"2024-08-17T17:48:52.568874+0000","last_clean":"2024-08-17T17:48:52.568874+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:48:52.568874+0000","last_undegraded":"2024-08-17T17:48:52.568874+0000","last_fullsized":"2024-08-17T17:48:52.568874+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":456,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.576098+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:48:54.576098+0000","last_peered":"2024-08-17T17:48:54.576098+0000","last_clean":"2024-08-17T17:48:54.576098+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:48:54.576098+0000","last_undegraded":"2024-08-17T17:48:54.576098+0000","last_fullsized":"2024-08-17T17:48:54.576098+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":482,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.566290+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:48:52.566290+0000","last_peered":"2024-08-17T17:48:52.566290+0000","last_clean":"2024-08-17T17:48:52.566290+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:48:52.566290+0000","last_undegraded":"2024-08-17T17:48:52.566290+0000","last_fullsized":"2024-08-17T17:48:52.566290+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"430'7075","reported_seq":7100,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:59.993111+0000","last_change":"2024-08-17T17:47:33.473836+0000","last_active":"2024-08-17T17:48:59.993111+0000","last_peered":"2024-08-17T17:48:59.993111+0000","last_clean":"2024-08-17T17:48:59.993111+0000","last_became_active":"2024-08-17T17:47:33.473133+0000","last_became_peered":"2024-08-17T17:47:33.473133+0000","last_unstale":"2024-08-17T17:48:59.993111+0000","last_undegraded":"2024-08-17T17:48:59.993111+0000","last_fullsized":"2024-08-17T17:48:59.993111+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:19.139546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":463667200,"num_objects":7075,"num_object_clones":0,"num_object_copies":21225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7075,"num_write_kb":452800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":730,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572800+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:48:54.572800+0000","last_peered":"2024-08-17T17:48:54.572800+0000","last_clean":"2024-08-17T17:48:54.572800+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:48:54.572800+0000","last_undegraded":"2024-08-17T17:48:54.572800+0000","last_fullsized":"2024-08-17T17:48:54.572800+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":701,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603518+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:49:03.603518+0000","last_peered":"2024-08-17T17:49:03.603518+0000","last_clean":"2024-08-17T17:49:03.603518+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:49:03.603518+0000","last_undegraded":"2024-08-17T17:49:03.603518+0000","last_fullsized":"2024-08-17T17:49:03.603518+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":717,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.571967+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:48:54.571967+0000","last_peered":"2024-08-17T17:48:54.571967+0000","last_clean":"2024-08-17T17:48:54.571967+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:48:54.571967+0000","last_undegraded":"2024-08-17T17:48:54.571967+0000","last_fullsized":"2024-08-17T17:48:54.571967+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"430'7354","reported_seq":7378,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:49:00.017184+0000","last_change":"2024-08-17T17:47:33.468582+0000","last_active":"2024-08-17T17:49:00.017184+0000","last_peered":"2024-08-17T17:49:00.017184+0000","last_clean":"2024-08-17T17:49:00.017184+0000","last_became_active":"2024-08-17T17:47:33.468018+0000","last_became_peered":"2024-08-17T17:47:33.468018+0000","last_unstale":"2024-08-17T17:49:00.017184+0000","last_undegraded":"2024-08-17T17:49:00.017184+0000","last_fullsized":"2024-08-17T17:49:00.017184+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7354,"log_dups_size":0,"ondisk_log_size":7354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:55:27.900073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":481951744,"num_objects":7354,"num_object_clones":0,"num_object_copies":22062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7354,"num_write_kb":470656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":389,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.575918+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:48:54.575918+0000","last_peered":"2024-08-17T17:48:54.575918+0000","last_clean":"2024-08-17T17:48:54.575918+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:48:54.575918+0000","last_undegraded":"2024-08-17T17:48:54.575918+0000","last_fullsized":"2024-08-17T17:48:54.575918+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":434,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572283+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:48:52.572283+0000","last_peered":"2024-08-17T17:48:52.572283+0000","last_clean":"2024-08-17T17:48:52.572283+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:48:52.572283+0000","last_undegraded":"2024-08-17T17:48:52.572283+0000","last_fullsized":"2024-08-17T17:48:52.572283+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":384,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.573061+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:48:52.573061+0000","last_peered":"2024-08-17T17:48:52.573061+0000","last_clean":"2024-08-17T17:48:52.573061+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:48:52.573061+0000","last_undegraded":"2024-08-17T17:48:52.573061+0000","last_fullsized":"2024-08-17T17:48:52.573061+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":384,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569053+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:48:52.569053+0000","last_peered":"2024-08-17T17:48:52.569053+0000","last_clean":"2024-08-17T17:48:52.569053+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:48:52.569053+0000","last_undegraded":"2024-08-17T17:48:52.569053+0000","last_fullsized":"2024-08-17T17:48:52.569053+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":427,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572784+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:48:52.572784+0000","last_peered":"2024-08-17T17:48:52.572784+0000","last_clean":"2024-08-17T17:48:52.572784+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:48:52.572784+0000","last_undegraded":"2024-08-17T17:48:52.572784+0000","last_fullsized":"2024-08-17T17:48:52.572784+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":707,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603256+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:49:03.603256+0000","last_peered":"2024-08-17T17:49:03.603256+0000","last_clean":"2024-08-17T17:49:03.603256+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:49:03.603256+0000","last_undegraded":"2024-08-17T17:49:03.603256+0000","last_fullsized":"2024-08-17T17:49:03.603256+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":245,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.574374+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:48:54.574374+0000","last_peered":"2024-08-17T17:48:54.574374+0000","last_clean":"2024-08-17T17:48:54.574374+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:48:54.574374+0000","last_undegraded":"2024-08-17T17:48:54.574374+0000","last_fullsized":"2024-08-17T17:48:54.574374+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":334,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.566620+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:48:52.566620+0000","last_peered":"2024-08-17T17:48:52.566620+0000","last_clean":"2024-08-17T17:48:52.566620+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:48:52.566620+0000","last_undegraded":"2024-08-17T17:48:52.566620+0000","last_fullsized":"2024-08-17T17:48:52.566620+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":663,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572281+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:48:54.572281+0000","last_peered":"2024-08-17T17:48:54.572281+0000","last_clean":"2024-08-17T17:48:54.572281+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:48:54.572281+0000","last_undegraded":"2024-08-17T17:48:54.572281+0000","last_fullsized":"2024-08-17T17:48:54.572281+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":478,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.580401+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:48:52.580401+0000","last_peered":"2024-08-17T17:48:52.580401+0000","last_clean":"2024-08-17T17:48:52.580401+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:48:52.580401+0000","last_undegraded":"2024-08-17T17:48:52.580401+0000","last_fullsized":"2024-08-17T17:48:52.580401+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":657,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.571852+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:48:54.571852+0000","last_peered":"2024-08-17T17:48:54.571852+0000","last_clean":"2024-08-17T17:48:54.571852+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:48:54.571852+0000","last_undegraded":"2024-08-17T17:48:54.571852+0000","last_fullsized":"2024-08-17T17:48:54.571852+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":401,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572220+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:48:52.572220+0000","last_peered":"2024-08-17T17:48:52.572220+0000","last_clean":"2024-08-17T17:48:52.572220+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:48:52.572220+0000","last_undegraded":"2024-08-17T17:48:52.572220+0000","last_fullsized":"2024-08-17T17:48:52.572220+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":689,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572108+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:48:54.572108+0000","last_peered":"2024-08-17T17:48:54.572108+0000","last_clean":"2024-08-17T17:48:54.572108+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:48:54.572108+0000","last_undegraded":"2024-08-17T17:48:54.572108+0000","last_fullsized":"2024-08-17T17:48:54.572108+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":405,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.573033+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:48:54.573033+0000","last_peered":"2024-08-17T17:48:54.573033+0000","last_clean":"2024-08-17T17:48:54.573033+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:48:54.573033+0000","last_undegraded":"2024-08-17T17:48:54.573033+0000","last_fullsized":"2024-08-17T17:48:54.573033+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":699,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.602915+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:49:03.602915+0000","last_peered":"2024-08-17T17:49:03.602915+0000","last_clean":"2024-08-17T17:49:03.602915+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:49:03.602915+0000","last_undegraded":"2024-08-17T17:49:03.602915+0000","last_fullsized":"2024-08-17T17:49:03.602915+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":656,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.574177+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:48:54.574177+0000","last_peered":"2024-08-17T17:48:54.574177+0000","last_clean":"2024-08-17T17:48:54.574177+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:48:54.574177+0000","last_undegraded":"2024-08-17T17:48:54.574177+0000","last_fullsized":"2024-08-17T17:48:54.574177+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":413,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569409+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:48:52.569409+0000","last_peered":"2024-08-17T17:48:52.569409+0000","last_clean":"2024-08-17T17:48:52.569409+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:48:52.569409+0000","last_undegraded":"2024-08-17T17:48:52.569409+0000","last_fullsized":"2024-08-17T17:48:52.569409+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":345,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572714+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:48:54.572714+0000","last_peered":"2024-08-17T17:48:54.572714+0000","last_clean":"2024-08-17T17:48:54.572714+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:48:54.572714+0000","last_undegraded":"2024-08-17T17:48:54.572714+0000","last_fullsized":"2024-08-17T17:48:54.572714+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":401,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.575695+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:48:54.575695+0000","last_peered":"2024-08-17T17:48:54.575695+0000","last_clean":"2024-08-17T17:48:54.575695+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:48:54.575695+0000","last_undegraded":"2024-08-17T17:48:54.575695+0000","last_fullsized":"2024-08-17T17:48:54.575695+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":421,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.568992+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:48:52.568992+0000","last_peered":"2024-08-17T17:48:52.568992+0000","last_clean":"2024-08-17T17:48:52.568992+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:48:52.568992+0000","last_undegraded":"2024-08-17T17:48:52.568992+0000","last_fullsized":"2024-08-17T17:48:52.568992+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":284,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.572693+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:48:52.572693+0000","last_peered":"2024-08-17T17:48:52.572693+0000","last_clean":"2024-08-17T17:48:52.572693+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:48:52.572693+0000","last_undegraded":"2024-08-17T17:48:52.572693+0000","last_fullsized":"2024-08-17T17:48:52.572693+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":421,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569282+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:48:52.569282+0000","last_peered":"2024-08-17T17:48:52.569282+0000","last_clean":"2024-08-17T17:48:52.569282+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:48:52.569282+0000","last_undegraded":"2024-08-17T17:48:52.569282+0000","last_fullsized":"2024-08-17T17:48:52.569282+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":284,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.569691+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:48:52.569691+0000","last_peered":"2024-08-17T17:48:52.569691+0000","last_clean":"2024-08-17T17:48:52.569691+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:48:52.569691+0000","last_undegraded":"2024-08-17T17:48:52.569691+0000","last_fullsized":"2024-08-17T17:48:52.569691+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":709,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.572609+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:48:54.572609+0000","last_peered":"2024-08-17T17:48:54.572609+0000","last_clean":"2024-08-17T17:48:54.572609+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:48:54.572609+0000","last_undegraded":"2024-08-17T17:48:54.572609+0000","last_fullsized":"2024-08-17T17:48:54.572609+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":683,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.573193+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:48:54.573193+0000","last_peered":"2024-08-17T17:48:54.573193+0000","last_clean":"2024-08-17T17:48:54.573193+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:48:54.573193+0000","last_undegraded":"2024-08-17T17:48:54.573193+0000","last_fullsized":"2024-08-17T17:48:54.573193+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":384,"reported_epoch":429,"state":"active+clean","last_fresh":"2024-08-17T17:48:52.571481+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:48:52.571481+0000","last_peered":"2024-08-17T17:48:52.571481+0000","last_clean":"2024-08-17T17:48:52.571481+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:48:52.571481+0000","last_undegraded":"2024-08-17T17:48:52.571481+0000","last_fullsized":"2024-08-17T17:48:52.571481+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":404,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.576154+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:48:54.576154+0000","last_peered":"2024-08-17T17:48:54.576154+0000","last_clean":"2024-08-17T17:48:54.576154+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:48:54.576154+0000","last_undegraded":"2024-08-17T17:48:54.576154+0000","last_fullsized":"2024-08-17T17:48:54.576154+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":7639269376,"num_objects":116566,"num_object_clones":0,"num_object_copies":349698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":116566,"num_write_kb":7460224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22867345408,"data_stored":22867345408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107266,"ondisk_log_size":107266,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170191,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463561,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4772520,"kb_used_data":3263200,"kb_used_omap":1,"kb_used_meta":1509310,"kb_avail":88972632,"statfs":{"total":95995035648,"available":91107975168,"internally_reserved":0,"allocated":3341516800,"data_stored":3339140047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1545533784},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492235,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5720780,"kb_used_data":5032900,"kb_used_omap":7,"kb_used_meta":687864,"kb_avail":88024372,"statfs":{"total":95995035648,"available":90136956928,"internally_reserved":0,"allocated":5153689600,"data_stored":5151335147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":704373069},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986827,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7065628,"kb_used_data":5565268,"kb_used_omap":1,"kb_used_meta":1500350,"kb_avail":86679524,"statfs":{"total":95995035648,"available":88759832576,"internally_reserved":0,"allocated":5698834432,"data_stored":5696468619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1536358744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225655,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4357004,"kb_used_data":3270532,"kb_used_omap":4,"kb_used_meta":1086459,"kb_avail":89388148,"statfs":{"total":95995035648,"available":91533463552,"internally_reserved":0,"allocated":3349024768,"data_stored":3346670613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1112534464},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075979,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533873,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4063456,"kb_used_data":2840980,"kb_used_omap":5,"kb_used_meta":1222458,"kb_avail":89681696,"statfs":{"total":95995035648,"available":91834056704,"internally_reserved":0,"allocated":2909163520,"data_stored":2906800494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1251797232},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089849,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3439016,"kb_used_data":2281120,"kb_used_omap":5,"kb_used_meta":1157882,"kb_avail":90306136,"statfs":{"total":95995035648,"available":92473483264,"internally_reserved":0,"allocated":2335866880,"data_stored":2333490186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1185671227},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2338258944,"data_stored":2338258944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2917138432,"data_stored":2917138432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3355246592,"data_stored":3355246592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":5724438528,"data_stored":5724438528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":5183504384,"data_stored":5183504384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3348758528,"data_stored":3348758528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:49:08.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:49:08.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:08.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:08.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:08.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:09.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:09.134 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:09.236 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:09.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:09.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 123295 123279 90.6317 59.9375 0.0227716 0.0110284 2024-08-17T17:49:09.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:09.424 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:09.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.563 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:09.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:09.604 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:09.622 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:49:09.623 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:09.651 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:49:09.651 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:09.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:09.672 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:49:09.673 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:09.695 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:49:09.696 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:09.700 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:09.725 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:09 smithi142 ceph-mon[106729]: pgmap v1728: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 508 GiB / 536 GiB avail; 59 MiB/s wr, 938 op/s 2024-08-17T17:49:09.725 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:49:09.726 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:09.726 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:09 smithi142 ceph-mon[103614]: pgmap v1728: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 508 GiB / 536 GiB avail; 59 MiB/s wr, 938 op/s 2024-08-17T17:49:09.753 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:49:09.754 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:09.780 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:49:09.780 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:09.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:09.802 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:49:09.803 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:49:09.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:09 smithi152 ceph-mon[93456]: pgmap v1728: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 508 GiB / 536 GiB avail; 59 MiB/s wr, 938 op/s 2024-08-17T17:49:09.836 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:09.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:09.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:10.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:10.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:10.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:10.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:10.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:10.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 124228 124212 90.2558 58.3125 0.0239619 0.0110746 2024-08-17T17:49:10.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:10.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:10.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.516 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:10.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.550 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:49:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:49:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:10.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:10.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:10.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:10.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2079330229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2079330229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2079330229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:10.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:10.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:11.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:11.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.105 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:11.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:11.133 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:11.178 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:11.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:11.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:11.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:11.354 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 125093 125077 89.8398 54.0625 0.0186701 0.0111259 2024-08-17T17:49:11.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:11.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:11.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:11.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:11.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:11 smithi152 ceph-mon[93456]: pgmap v1729: 89 pgs: 89 active+clean; 7.5 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 58 MiB/s wr, 928 op/s 2024-08-17T17:49:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:11.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:11.862 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:49:11] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:49:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-mon[106729]: pgmap v1729: 89 pgs: 89 active+clean; 7.5 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 58 MiB/s wr, 928 op/s 2024-08-17T17:49:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-mon[103614]: pgmap v1729: 89 pgs: 89 active+clean; 7.5 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 58 MiB/s wr, 928 op/s 2024-08-17T17:49:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:11.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:11.888 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:11.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:11.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.025 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:12.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:12.126 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.167 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:12.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:12.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:12.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:12.355 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 126015 125999 89.4736 57.625 0.00985594 0.0111716 2024-08-17T17:49:12.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:12.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:12.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:12.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:12.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:12 smithi142 ceph-mon[106729]: pgmap v1730: 89 pgs: 89 active+clean; 7.6 GiB data, 30 GiB used, 507 GiB / 536 GiB avail; 58 MiB/s wr, 932 op/s 2024-08-17T17:49:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:12 smithi142 ceph-mon[103614]: pgmap v1730: 89 pgs: 89 active+clean; 7.6 GiB data, 30 GiB used, 507 GiB / 536 GiB avail; 58 MiB/s wr, 932 op/s 2024-08-17T17:49:12.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:49:12.686 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:12.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:12.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:12.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:12.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:12.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:13.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:12 smithi152 ceph-mon[93456]: pgmap v1730: 89 pgs: 89 active+clean; 7.6 GiB data, 30 GiB used, 507 GiB / 536 GiB avail; 58 MiB/s wr, 932 op/s 2024-08-17T17:49:13.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:13.160 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:13.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:13.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:13.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:13.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:13.355 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 126885 126869 89.0792 54.375 0.0228561 0.0112212 2024-08-17T17:49:13.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:13.424 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:13.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:13.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:13.665 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.695 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:13.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:13.797 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:13.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:13.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:13.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:13.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:14.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:14.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:14.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:14.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.232 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:14.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:14.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:14.355 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0072 2024-08-17T17:49:14.356 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 127813 2024-08-17T17:49:14.356 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:49:14.356 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:49:14.356 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 88.7519 2024-08-17T17:49:14.356 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 42.4407 2024-08-17T17:49:14.357 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 256.875 2024-08-17T17:49:14.357 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 52.25 2024-08-17T17:49:14.357 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1420 2024-08-17T17:49:14.357 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 679.05 2024-08-17T17:49:14.357 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 4110 2024-08-17T17:49:14.358 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 836 2024-08-17T17:49:14.358 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.0112634 2024-08-17T17:49:14.358 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.00798995 2024-08-17T17:49:14.358 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 0.239115 2024-08-17T17:49:14.358 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00119197 2024-08-17T17:49:14.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:14.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:14.361 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:49:14.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:14.486 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:14.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:14.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:14.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:14.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:14.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:14.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:14.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:14.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:14.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:15.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:15.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:15.156 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:15.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:15.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:15.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:15.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:15.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:15.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:15.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:15.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:15.734 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:15.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:15.808 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:15 smithi142 ceph-mon[103614]: pgmap v1731: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 58 MiB/s wr, 929 op/s 2024-08-17T17:49:15.809 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:15 smithi142 ceph-mon[106729]: pgmap v1731: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 58 MiB/s wr, 929 op/s 2024-08-17T17:49:15.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:15 smithi152 ceph-mon[93456]: pgmap v1731: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 58 MiB/s wr, 929 op/s 2024-08-17T17:49:15.836 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:15.869 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:15.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:15.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:15.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:16.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:16.123 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:16.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:16.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:16.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:16.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:16.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:16.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:16.497 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:16.530 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:16.632 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:16.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:16.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:16.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:16.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:16.937 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:16.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:17.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:17.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:17.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:17.211 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:17.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:17.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:17.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:17.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:17.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:17.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:17.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:17.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:17.771 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:49:17.771 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:49:17.771 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.771 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:131235 13.b 13:d130c02b:::benchmark_data_smithi142_393845_object3419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:49:17.771 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:17.751719+0000", 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.017937657999999999, 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017965049, 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131235 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.772 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.751719+0000", 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.751719+0000", 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.751720+0000", 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.249e-06 2024-08-17T17:49:17.773 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.751726+0000", 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4369999999999998e-06 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.751741+0000", 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4942e-05 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:49:17.774 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.751748+0000", 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4919999999999996e-06 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.755410+0000", 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0036613230000000002 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.755538+0000", 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000128462 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.775 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.755652+0000", 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011396999999999999 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.759266+0000", 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0036135400000000002 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.762646+0000", 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0033803660000000001 2024-08-17T17:49:17.776 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:131247 13.2 13:48142928:::benchmark_data_smithi142_393845_object3431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:17.767900+0000", 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.001756446, 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0018568129999999999, 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:49:17.777 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131247 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767900+0000", 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767900+0000", 2024-08-17T17:49:17.778 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767901+0000", 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6200000000000006e-07 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767905+0000", 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0790000000000002e-06 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:49:17.779 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767917+0000", 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1082e-05 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767922+0000", 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4979999999999997e-06 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.767972+0000", 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0040999999999997e-05 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.780 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768109+0000", 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013659699999999999 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768182+0000", 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3543999999999999e-05 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:131248 13.6 e431/413 13:6ce55a0e:::benchmark_data_smithi142_393845_object3432:head v 431'8281, mlcod=431'8277)", 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:17.768410+0000", 2024-08-17T17:49:17.781 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.001247279, 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001404097, 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768410+0000", 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768410+0000", 2024-08-17T17:49:17.782 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768410+0000", 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4199999999999996e-07 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768414+0000", 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.765e-06 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:49:17.783 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768424+0000", 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9089999999999995e-06 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768430+0000", 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8239999999999996e-06 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768497+0000", 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6984999999999996e-05 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:49:17.784 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768517+0000", 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0230000000000001e-05 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:131249 13.d e431/413 13:b36534e3:::benchmark_data_smithi142_393845_object3433:head v 431'8150, mlcod=431'8148)", 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:17.768928+0000", 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.00072876799999999997, 2024-08-17T17:49:17.785 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00092964399999999998, 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768928+0000", 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768928+0000", 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.786 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768929+0000", 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3900000000000005e-07 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768933+0000", 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1810000000000003e-06 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768942+0000", 2024-08-17T17:49:17.787 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8459999999999997e-06 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768947+0000", 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1610000000000002e-06 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.768998+0000", 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0779999999999998e-05 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:49:17.788 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.769020+0000", 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2867999999999999e-05 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:131229 13.b 13:d3d56d91:::benchmark_data_smithi142_393845_object3413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:17.747925+0000", 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.021731744000000001, 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.021987610000000001, 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:49:17.789 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131229 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.747925+0000", 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.790 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.747925+0000", 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.747927+0000", 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5409999999999999e-06 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.747932+0000", 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8440000000000003e-06 2024-08-17T17:49:17.791 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.747944+0000", 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1951e-05 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.747952+0000", 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5349999999999999e-06 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.748064+0000", 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000112154 2024-08-17T17:49:17.792 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.748251+0000", 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000186761 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.748359+0000", 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000108447 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:49:17.793 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.755242+0000", 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0068823640000000002 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.759140+0000", 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038979729999999999 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:131233 13.b 13:dfcfafa0:::benchmark_data_smithi142_393845_object3417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:49:17.794 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:17.749286+0000", 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 0.020370816999999999, 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.020691633000000001, 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 131233 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.749286+0000", 2024-08-17T17:49:17.795 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.749286+0000", 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.749287+0000", 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7700000000000003e-07 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.796 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.749290+0000", 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3189999999999999e-06 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.749299+0000", 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9320000000000002e-06 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.749305+0000", 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6300000000000003e-06 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.797 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.752084+0000", 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027793459999999998 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.752210+0000", 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012563099999999999 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.752321+0000", 2024-08-17T17:49:17.798 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000111333 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.759062+0000", 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0067408989999999998 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:17.762597+0000", 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035351660000000002 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:49:17.799 INFO:teuthology.orchestra.run.smithi142.stdout: ], 2024-08-17T17:49:17.800 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 6 2024-08-17T17:49:17.800 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:49:17.800 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:17 smithi142 ceph-mon[106729]: pgmap v1732: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 53 MiB/s wr, 867 op/s 2024-08-17T17:49:17.800 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:17 smithi142 ceph-mon[103614]: pgmap v1732: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 53 MiB/s wr, 867 op/s 2024-08-17T17:49:17.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:17 smithi152 ceph-mon[93456]: pgmap v1732: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 53 MiB/s wr, 867 op/s 2024-08-17T17:49:17.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:17.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:18.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:18.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:18.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:18.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:18.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:18.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:18.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:18.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:18.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:18.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:18.573 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:18.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:49:18.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:18.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:18.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:18.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:18.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:19.003 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:19.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:19.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:19.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:19.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:19.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:19.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:19.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:19.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:19.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.531 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:19.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:19.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:19.659 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:19.737 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:19.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:19.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:19.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:19 smithi152 ceph-mon[93456]: pgmap v1733: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 85 B/s rd, 52 MiB/s wr, 1k op/s 2024-08-17T17:49:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:19 smithi142 ceph-mon[106729]: pgmap v1733: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 85 B/s rd, 52 MiB/s wr, 1k op/s 2024-08-17T17:49:19.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:19 smithi142 ceph-mon[103614]: pgmap v1733: 89 pgs: 89 active+clean; 7.7 GiB data, 30 GiB used, 506 GiB / 536 GiB avail; 85 B/s rd, 52 MiB/s wr, 1k op/s 2024-08-17T17:49:19.905 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:19.940 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:20.042 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:20.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:20.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:20.224 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:20.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:20.358 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:20.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:20.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:49:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:49:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:20.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:20.615 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:49:20.616 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:49:20.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:20.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:20.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:20.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:20.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:20.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:20.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[106729]: from='client.38232 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:49:20.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[106729]: pgmap v1734: 89 pgs: 89 active+clean; 7.6 GiB data, 30 GiB used, 507 GiB / 536 GiB avail; 85 B/s rd, 39 MiB/s wr, 959 op/s 2024-08-17T17:49:20.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:20.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:20.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[103614]: from='client.38232 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:49:20.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:20 smithi142 ceph-mon[103614]: pgmap v1734: 89 pgs: 89 active+clean; 7.6 GiB data, 30 GiB used, 507 GiB / 536 GiB avail; 85 B/s rd, 39 MiB/s wr, 959 op/s 2024-08-17T17:49:21.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:21.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:20 smithi152 ceph-mon[93456]: from='client.38232 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:49:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:20 smithi152 ceph-mon[93456]: pgmap v1734: 89 pgs: 89 active+clean; 7.6 GiB data, 30 GiB used, 507 GiB / 536 GiB avail; 85 B/s rd, 39 MiB/s wr, 959 op/s 2024-08-17T17:49:21.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:21.199 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:21.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.340 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:21.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:21.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:21.489 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:21.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:21.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:21.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:21.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.760 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:21.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:21.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.904 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:21.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:22.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:22.053 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:22.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:49:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:49:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:22.079 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:49:22.080 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:49:22.080 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:22.080 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:49:21] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:49:22.080 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:49:22.080 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:49:22.080 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:22.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:22.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:22.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:22.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:22.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:22.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:22.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:22.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:22.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:22.595 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:22.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:22.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:22.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:22.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:22.763 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1733,"stamp":"2024-08-17T17:49:18.484399+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8246532069,"num_objects":125850,"num_object_clones":0,"num_object_copies":377550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":125850,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":130022,"num_write_kb":8180560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2958510,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109251,"ondisk_log_size":109251,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":31658852,"kb_used_data":24246388,"kb_used_omap":27,"kb_used_meta":7412068,"kb_avail":530812060,"statfs":{"total":575970213888,"available":543551549440,"internally_reserved":0,"allocated":24828301312,"data_stored":24814081929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":7589958376},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,9,3,7,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":529924120,"num_objects":8087,"num_object_clones":0,"num_object_copies":24261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8087,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12007,"num_write_kb":642945,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002882"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":708,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606802+0000","last_change":"2024-08-17T17:33:40.653332+0000","last_active":"2024-08-17T17:49:03.606802+0000","last_peered":"2024-08-17T17:49:03.606802+0000","last_clean":"2024-08-17T17:49:03.606802+0000","last_became_active":"2024-08-17T17:33:40.652968+0000","last_became_peered":"2024-08-17T17:33:40.652968+0000","last_unstale":"2024-08-17T17:49:03.606802+0000","last_undegraded":"2024-08-17T17:49:03.606802+0000","last_fullsized":"2024-08-17T17:49:03.606802+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:47.278518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":140,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577809+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:48:54.577809+0000","last_peered":"2024-08-17T17:48:54.577809+0000","last_clean":"2024-08-17T17:48:54.577809+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:48:54.577809+0000","last_undegraded":"2024-08-17T17:48:54.577809+0000","last_fullsized":"2024-08-17T17:48:54.577809+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.570528+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:48:54.570528+0000","last_peered":"2024-08-17T17:48:54.570528+0000","last_clean":"2024-08-17T17:48:54.570528+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:48:54.570528+0000","last_undegraded":"2024-08-17T17:48:54.570528+0000","last_fullsized":"2024-08-17T17:48:54.570528+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":406,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.608015+0000","last_change":"2024-08-17T17:33:40.673382+0000","last_active":"2024-08-17T17:49:03.608015+0000","last_peered":"2024-08-17T17:49:03.608015+0000","last_clean":"2024-08-17T17:49:03.608015+0000","last_became_active":"2024-08-17T17:33:40.669156+0000","last_became_peered":"2024-08-17T17:33:40.669156+0000","last_unstale":"2024-08-17T17:49:03.608015+0000","last_undegraded":"2024-08-17T17:49:03.608015+0000","last_fullsized":"2024-08-17T17:49:03.608015+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:03.366758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":583,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607656+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:49:03.607656+0000","last_peered":"2024-08-17T17:49:03.607656+0000","last_clean":"2024-08-17T17:49:03.607656+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:49:03.607656+0000","last_undegraded":"2024-08-17T17:49:03.607656+0000","last_fullsized":"2024-08-17T17:49:03.607656+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":283,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.579820+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:48:54.579820+0000","last_peered":"2024-08-17T17:48:54.579820+0000","last_clean":"2024-08-17T17:48:54.579820+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:48:54.579820+0000","last_undegraded":"2024-08-17T17:48:54.579820+0000","last_fullsized":"2024-08-17T17:48:54.579820+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":677,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607149+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:49:03.607149+0000","last_peered":"2024-08-17T17:49:03.607149+0000","last_clean":"2024-08-17T17:49:03.607149+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:49:03.607149+0000","last_undegraded":"2024-08-17T17:49:03.607149+0000","last_fullsized":"2024-08-17T17:49:03.607149+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":451,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.579724+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:48:54.579724+0000","last_peered":"2024-08-17T17:48:54.579724+0000","last_clean":"2024-08-17T17:48:54.579724+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:48:54.579724+0000","last_undegraded":"2024-08-17T17:48:54.579724+0000","last_fullsized":"2024-08-17T17:48:54.579724+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"431'8118","reported_seq":8144,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.434890+0000","last_change":"2024-08-17T17:47:33.474484+0000","last_active":"2024-08-17T17:49:17.434890+0000","last_peered":"2024-08-17T17:49:17.434890+0000","last_clean":"2024-08-17T17:49:17.434890+0000","last_became_active":"2024-08-17T17:47:33.474193+0000","last_became_peered":"2024-08-17T17:47:33.474193+0000","last_unstale":"2024-08-17T17:49:17.434890+0000","last_undegraded":"2024-08-17T17:49:17.434890+0000","last_fullsized":"2024-08-17T17:49:17.434890+0000","mapping_epoch":413,"log_start":"418'1100","ondisk_log_start":"418'1100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:23.071916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":507379712,"num_objects":7742,"num_object_clones":0,"num_object_copies":23226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8118,"num_write_kb":507520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":136,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.584361+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:48:54.584361+0000","last_peered":"2024-08-17T17:48:54.584361+0000","last_clean":"2024-08-17T17:48:54.584361+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:48:54.584361+0000","last_undegraded":"2024-08-17T17:48:54.584361+0000","last_fullsized":"2024-08-17T17:48:54.584361+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":317,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.571736+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:48:54.571736+0000","last_peered":"2024-08-17T17:48:54.571736+0000","last_clean":"2024-08-17T17:48:54.571736+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:48:54.571736+0000","last_undegraded":"2024-08-17T17:48:54.571736+0000","last_fullsized":"2024-08-17T17:48:54.571736+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"431'7927","reported_seq":7974,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:14.148068+0000","last_change":"2024-08-17T17:47:33.469969+0000","last_active":"2024-08-17T17:49:14.148068+0000","last_peered":"2024-08-17T17:49:14.148068+0000","last_clean":"2024-08-17T17:49:14.148068+0000","last_became_active":"2024-08-17T17:47:33.469675+0000","last_became_peered":"2024-08-17T17:47:33.469675+0000","last_unstale":"2024-08-17T17:49:14.148068+0000","last_undegraded":"2024-08-17T17:49:14.148068+0000","last_fullsized":"2024-08-17T17:49:14.148068+0000","mapping_epoch":413,"log_start":"419'2000","ondisk_log_start":"419'2000","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:24.678241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":519503872,"num_objects":7927,"num_object_clones":0,"num_object_copies":23781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7927,"num_write_kb":507328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":138,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.578418+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:48:54.578418+0000","last_peered":"2024-08-17T17:48:54.578418+0000","last_clean":"2024-08-17T17:48:54.578418+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:48:54.578418+0000","last_undegraded":"2024-08-17T17:48:54.578418+0000","last_fullsized":"2024-08-17T17:48:54.578418+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":402,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.579749+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:48:54.579749+0000","last_peered":"2024-08-17T17:48:54.579749+0000","last_clean":"2024-08-17T17:48:54.579749+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:48:54.579749+0000","last_undegraded":"2024-08-17T17:48:54.579749+0000","last_fullsized":"2024-08-17T17:48:54.579749+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"431'8008","reported_seq":8055,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:15.111694+0000","last_change":"2024-08-17T17:47:33.474669+0000","last_active":"2024-08-17T17:49:15.111694+0000","last_peered":"2024-08-17T17:49:15.111694+0000","last_clean":"2024-08-17T17:49:15.111694+0000","last_became_active":"2024-08-17T17:47:33.474373+0000","last_became_peered":"2024-08-17T17:47:33.474373+0000","last_unstale":"2024-08-17T17:49:15.111694+0000","last_undegraded":"2024-08-17T17:49:15.111694+0000","last_fullsized":"2024-08-17T17:49:15.111694+0000","mapping_epoch":413,"log_start":"418'800","ondisk_log_start":"418'800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7208,"log_dups_size":0,"ondisk_log_size":7208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:34:23.147112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":518782976,"num_objects":7916,"num_object_clones":0,"num_object_copies":23748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8008,"num_write_kb":509568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":134,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.570450+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:48:54.570450+0000","last_peered":"2024-08-17T17:48:54.570450+0000","last_clean":"2024-08-17T17:48:54.570450+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:48:54.570450+0000","last_undegraded":"2024-08-17T17:48:54.570450+0000","last_fullsized":"2024-08-17T17:48:54.570450+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":412,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.576337+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:48:54.576337+0000","last_peered":"2024-08-17T17:48:54.576337+0000","last_clean":"2024-08-17T17:48:54.576337+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:48:54.576337+0000","last_undegraded":"2024-08-17T17:48:54.576337+0000","last_fullsized":"2024-08-17T17:48:54.576337+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"431'8321","reported_seq":8366,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:18.074775+0000","last_change":"2024-08-17T17:47:33.471249+0000","last_active":"2024-08-17T17:49:18.074775+0000","last_peered":"2024-08-17T17:49:18.074775+0000","last_clean":"2024-08-17T17:49:18.074775+0000","last_became_active":"2024-08-17T17:47:33.470968+0000","last_became_peered":"2024-08-17T17:47:33.470968+0000","last_unstale":"2024-08-17T17:49:18.074775+0000","last_undegraded":"2024-08-17T17:49:18.074775+0000","last_fullsized":"2024-08-17T17:49:18.074775+0000","mapping_epoch":413,"log_start":"419'1500","ondisk_log_start":"419'1500","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:45.656151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515571712,"num_objects":7867,"num_object_clones":0,"num_object_copies":23601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8321,"num_write_kb":518016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":620,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606968+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:49:03.606968+0000","last_peered":"2024-08-17T17:49:03.606968+0000","last_clean":"2024-08-17T17:49:03.606968+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:49:03.606968+0000","last_undegraded":"2024-08-17T17:49:03.606968+0000","last_fullsized":"2024-08-17T17:49:03.606968+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":386,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.621405+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:49:03.621405+0000","last_peered":"2024-08-17T17:49:03.621405+0000","last_clean":"2024-08-17T17:49:03.621405+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:49:03.621405+0000","last_undegraded":"2024-08-17T17:49:03.621405+0000","last_fullsized":"2024-08-17T17:49:03.621405+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"431'8188","reported_seq":8214,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.197685+0000","last_change":"2024-08-17T17:47:33.472177+0000","last_active":"2024-08-17T17:49:17.197685+0000","last_peered":"2024-08-17T17:49:17.197685+0000","last_clean":"2024-08-17T17:49:17.197685+0000","last_became_active":"2024-08-17T17:47:33.471903+0000","last_became_peered":"2024-08-17T17:47:33.471903+0000","last_unstale":"2024-08-17T17:49:17.197685+0000","last_undegraded":"2024-08-17T17:49:17.197685+0000","last_fullsized":"2024-08-17T17:49:17.197685+0000","mapping_epoch":413,"log_start":"419'1900","ondisk_log_start":"419'1900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6288,"log_dups_size":0,"ondisk_log_size":6288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:39:03.482291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":512884736,"num_objects":7826,"num_object_clones":0,"num_object_copies":23478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8188,"num_write_kb":512448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":710,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.608161+0000","last_change":"2024-08-17T17:24:37.068966+0000","last_active":"2024-08-17T17:49:03.608161+0000","last_peered":"2024-08-17T17:49:03.608161+0000","last_clean":"2024-08-17T17:49:03.608161+0000","last_became_active":"2024-08-17T17:24:37.044126+0000","last_became_peered":"2024-08-17T17:24:37.044126+0000","last_unstale":"2024-08-17T17:49:03.608161+0000","last_undegraded":"2024-08-17T17:49:03.608161+0000","last_fullsized":"2024-08-17T17:49:03.608161+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:03.443209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":385,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577082+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:48:54.577082+0000","last_peered":"2024-08-17T17:48:54.577082+0000","last_clean":"2024-08-17T17:48:54.577082+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:48:54.577082+0000","last_undegraded":"2024-08-17T17:48:54.577082+0000","last_fullsized":"2024-08-17T17:48:54.577082+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"431'8157","reported_seq":8183,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.440694+0000","last_change":"2024-08-17T17:47:33.472785+0000","last_active":"2024-08-17T17:49:17.440694+0000","last_peered":"2024-08-17T17:49:17.440694+0000","last_clean":"2024-08-17T17:49:17.440694+0000","last_became_active":"2024-08-17T17:47:33.472497+0000","last_became_peered":"2024-08-17T17:47:33.472497+0000","last_unstale":"2024-08-17T17:49:17.440694+0000","last_undegraded":"2024-08-17T17:49:17.440694+0000","last_fullsized":"2024-08-17T17:49:17.440694+0000","mapping_epoch":413,"log_start":"418'1100","ondisk_log_start":"418'1100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:39:33.059184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":511639552,"num_objects":7807,"num_object_clones":0,"num_object_copies":23421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8157,"num_write_kb":510848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":386,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.576442+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:48:54.576442+0000","last_peered":"2024-08-17T17:48:54.576442+0000","last_clean":"2024-08-17T17:48:54.576442+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:48:54.576442+0000","last_undegraded":"2024-08-17T17:48:54.576442+0000","last_fullsized":"2024-08-17T17:48:54.576442+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":412,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.578814+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:48:54.578814+0000","last_peered":"2024-08-17T17:48:54.578814+0000","last_clean":"2024-08-17T17:48:54.578814+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:48:54.578814+0000","last_undegraded":"2024-08-17T17:48:54.578814+0000","last_fullsized":"2024-08-17T17:48:54.578814+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"431'8275","reported_seq":8301,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.424513+0000","last_change":"2024-08-17T17:47:33.473418+0000","last_active":"2024-08-17T17:49:17.424513+0000","last_peered":"2024-08-17T17:49:17.424513+0000","last_clean":"2024-08-17T17:49:17.424513+0000","last_became_active":"2024-08-17T17:47:33.473144+0000","last_became_peered":"2024-08-17T17:47:33.473144+0000","last_unstale":"2024-08-17T17:49:17.424513+0000","last_undegraded":"2024-08-17T17:49:17.424513+0000","last_fullsized":"2024-08-17T17:49:17.424513+0000","mapping_epoch":413,"log_start":"419'1200","ondisk_log_start":"419'1200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:19:35.223379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515571712,"num_objects":7867,"num_object_clones":0,"num_object_copies":23601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8275,"num_write_kb":516544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":423,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.580146+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:48:54.580146+0000","last_peered":"2024-08-17T17:48:54.580146+0000","last_clean":"2024-08-17T17:48:54.580146+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:48:54.580146+0000","last_undegraded":"2024-08-17T17:48:54.580146+0000","last_fullsized":"2024-08-17T17:48:54.580146+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":385,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.578496+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:48:54.578496+0000","last_peered":"2024-08-17T17:48:54.578496+0000","last_clean":"2024-08-17T17:48:54.578496+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:48:54.578496+0000","last_undegraded":"2024-08-17T17:48:54.578496+0000","last_fullsized":"2024-08-17T17:48:54.578496+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"431'8250","reported_seq":8276,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.166809+0000","last_change":"2024-08-17T17:47:33.473396+0000","last_active":"2024-08-17T17:49:17.166809+0000","last_peered":"2024-08-17T17:49:17.166809+0000","last_clean":"2024-08-17T17:49:17.166809+0000","last_became_active":"2024-08-17T17:47:33.472959+0000","last_became_peered":"2024-08-17T17:47:33.472959+0000","last_unstale":"2024-08-17T17:49:17.166809+0000","last_undegraded":"2024-08-17T17:49:17.166809+0000","last_fullsized":"2024-08-17T17:49:17.166809+0000","mapping_epoch":413,"log_start":"419'1900","ondisk_log_start":"419'1900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6350,"log_dups_size":0,"ondisk_log_size":6350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:26.187848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":516816896,"num_objects":7886,"num_object_clones":0,"num_object_copies":23658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8250,"num_write_kb":516352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":679,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607000+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:49:03.607000+0000","last_peered":"2024-08-17T17:49:03.607000+0000","last_clean":"2024-08-17T17:49:03.607000+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:49:03.607000+0000","last_undegraded":"2024-08-17T17:49:03.607000+0000","last_fullsized":"2024-08-17T17:49:03.607000+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":616,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607228+0000","last_change":"2024-08-17T17:21:31.886188+0000","last_active":"2024-08-17T17:49:03.607228+0000","last_peered":"2024-08-17T17:49:03.607228+0000","last_clean":"2024-08-17T17:49:03.607228+0000","last_became_active":"2024-08-17T17:21:31.885974+0000","last_became_peered":"2024-08-17T17:21:31.885974+0000","last_unstale":"2024-08-17T17:49:03.607228+0000","last_undegraded":"2024-08-17T17:49:03.607228+0000","last_fullsized":"2024-08-17T17:49:03.607228+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:27.786527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.c","version":"431'8038","reported_seq":8085,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:14.036601+0000","last_change":"2024-08-17T17:47:33.468618+0000","last_active":"2024-08-17T17:49:14.036601+0000","last_peered":"2024-08-17T17:49:14.036601+0000","last_clean":"2024-08-17T17:49:14.036601+0000","last_became_active":"2024-08-17T17:47:33.468320+0000","last_became_peered":"2024-08-17T17:47:33.468320+0000","last_unstale":"2024-08-17T17:49:14.036601+0000","last_undegraded":"2024-08-17T17:49:14.036601+0000","last_fullsized":"2024-08-17T17:49:14.036601+0000","mapping_epoch":413,"log_start":"419'2100","ondisk_log_start":"419'2100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:12.338465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":526778368,"num_objects":8038,"num_object_clones":0,"num_object_copies":24114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8038,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":823,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.620955+0000","last_change":"2024-08-17T17:47:14.531821+0000","last_active":"2024-08-17T17:49:03.620955+0000","last_peered":"2024-08-17T17:49:03.620955+0000","last_clean":"2024-08-17T17:49:03.620955+0000","last_became_active":"2024-08-17T17:47:14.531354+0000","last_became_peered":"2024-08-17T17:47:14.531354+0000","last_unstale":"2024-08-17T17:49:03.620955+0000","last_undegraded":"2024-08-17T17:49:03.620955+0000","last_fullsized":"2024-08-17T17:49:03.620955+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:54.544598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":366,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.570593+0000","last_change":"2024-08-17T17:46:26.863739+0000","last_active":"2024-08-17T17:48:54.570593+0000","last_peered":"2024-08-17T17:48:54.570593+0000","last_clean":"2024-08-17T17:48:54.570593+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:48:54.570593+0000","last_undegraded":"2024-08-17T17:48:54.570593+0000","last_fullsized":"2024-08-17T17:48:54.570593+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":659,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607910+0000","last_change":"2024-08-17T17:21:31.882746+0000","last_active":"2024-08-17T17:49:03.607910+0000","last_peered":"2024-08-17T17:49:03.607910+0000","last_clean":"2024-08-17T17:49:03.607910+0000","last_became_active":"2024-08-17T17:21:31.882338+0000","last_became_peered":"2024-08-17T17:21:31.882338+0000","last_unstale":"2024-08-17T17:49:03.607910+0000","last_undegraded":"2024-08-17T17:49:03.607910+0000","last_fullsized":"2024-08-17T17:49:03.607910+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:13.307194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":395,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.576908+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:48:54.576908+0000","last_peered":"2024-08-17T17:48:54.576908+0000","last_clean":"2024-08-17T17:48:54.576908+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:48:54.576908+0000","last_undegraded":"2024-08-17T17:48:54.576908+0000","last_fullsized":"2024-08-17T17:48:54.576908+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"431'8172","reported_seq":8198,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.405816+0000","last_change":"2024-08-17T17:47:33.474825+0000","last_active":"2024-08-17T17:49:17.405816+0000","last_peered":"2024-08-17T17:49:17.405816+0000","last_clean":"2024-08-17T17:49:17.405816+0000","last_became_active":"2024-08-17T17:47:33.474247+0000","last_became_peered":"2024-08-17T17:47:33.474247+0000","last_unstale":"2024-08-17T17:49:17.405816+0000","last_undegraded":"2024-08-17T17:49:17.405816+0000","last_fullsized":"2024-08-17T17:49:17.405816+0000","mapping_epoch":413,"log_start":"418'1100","ondisk_log_start":"418'1100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:46:34.619479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":509739008,"num_objects":7778,"num_object_clones":0,"num_object_copies":23334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8172,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":475,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.604954+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:49:03.604954+0000","last_peered":"2024-08-17T17:49:03.604954+0000","last_clean":"2024-08-17T17:49:03.604954+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:49:03.604954+0000","last_undegraded":"2024-08-17T17:49:03.604954+0000","last_fullsized":"2024-08-17T17:49:03.604954+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":367,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.579626+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:48:54.579626+0000","last_peered":"2024-08-17T17:48:54.579626+0000","last_clean":"2024-08-17T17:48:54.579626+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:48:54.579626+0000","last_undegraded":"2024-08-17T17:48:54.579626+0000","last_fullsized":"2024-08-17T17:48:54.579626+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":401,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.624397+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:49:03.624397+0000","last_peered":"2024-08-17T17:49:03.624397+0000","last_clean":"2024-08-17T17:49:03.624397+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:49:03.624397+0000","last_undegraded":"2024-08-17T17:49:03.624397+0000","last_fullsized":"2024-08-17T17:49:03.624397+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"431'7993","reported_seq":8019,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:15.044020+0000","last_change":"2024-08-17T17:47:33.471881+0000","last_active":"2024-08-17T17:49:15.044020+0000","last_peered":"2024-08-17T17:49:15.044020+0000","last_clean":"2024-08-17T17:49:15.044020+0000","last_became_active":"2024-08-17T17:47:33.471614+0000","last_became_peered":"2024-08-17T17:47:33.471614+0000","last_unstale":"2024-08-17T17:49:15.044020+0000","last_undegraded":"2024-08-17T17:49:15.044020+0000","last_fullsized":"2024-08-17T17:49:15.044020+0000","mapping_epoch":413,"log_start":"418'300","ondisk_log_start":"418'300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7693,"log_dups_size":0,"ondisk_log_size":7693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:40.873776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":519241728,"num_objects":7923,"num_object_clones":0,"num_object_copies":23769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7993,"num_write_kb":509312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":567,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.608926+0000","last_change":"2024-08-17T17:33:20.287506+0000","last_active":"2024-08-17T17:49:03.608926+0000","last_peered":"2024-08-17T17:49:03.608926+0000","last_clean":"2024-08-17T17:49:03.608926+0000","last_became_active":"2024-08-17T17:33:20.287124+0000","last_became_peered":"2024-08-17T17:33:20.287124+0000","last_unstale":"2024-08-17T17:49:03.608926+0000","last_undegraded":"2024-08-17T17:49:03.608926+0000","last_fullsized":"2024-08-17T17:49:03.608926+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:18.526960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":167,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.579540+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:48:54.579540+0000","last_peered":"2024-08-17T17:48:54.579540+0000","last_clean":"2024-08-17T17:48:54.579540+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:48:54.579540+0000","last_undegraded":"2024-08-17T17:48:54.579540+0000","last_fullsized":"2024-08-17T17:48:54.579540+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":18,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.578409+0000","last_change":"2024-08-17T17:48:13.475561+0000","last_active":"2024-08-17T17:48:54.578409+0000","last_peered":"2024-08-17T17:48:54.578409+0000","last_clean":"2024-08-17T17:48:54.578409+0000","last_became_active":"2024-08-17T17:48:13.475084+0000","last_became_peered":"2024-08-17T17:48:13.475084+0000","last_unstale":"2024-08-17T17:48:54.578409+0000","last_undegraded":"2024-08-17T17:48:54.578409+0000","last_fullsized":"2024-08-17T17:48:54.578409+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:23.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.d","version":"431'8113","reported_seq":8140,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:17.132619+0000","last_change":"2024-08-17T17:47:33.473230+0000","last_active":"2024-08-17T17:49:17.132619+0000","last_peered":"2024-08-17T17:49:17.132619+0000","last_clean":"2024-08-17T17:49:17.132619+0000","last_became_active":"2024-08-17T17:47:33.472924+0000","last_became_peered":"2024-08-17T17:47:33.472924+0000","last_unstale":"2024-08-17T17:49:17.132619+0000","last_undegraded":"2024-08-17T17:49:17.132619+0000","last_fullsized":"2024-08-17T17:49:17.132619+0000","mapping_epoch":413,"log_start":"419'1800","ondisk_log_start":"419'1800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:58.863358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":505544728,"num_objects":7715,"num_object_clones":0,"num_object_copies":23145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7715,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8113,"num_write_kb":506433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":543,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603140+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:49:03.603140+0000","last_peered":"2024-08-17T17:49:03.603140+0000","last_clean":"2024-08-17T17:49:03.603140+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:49:03.603140+0000","last_undegraded":"2024-08-17T17:49:03.603140+0000","last_fullsized":"2024-08-17T17:49:03.603140+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":406,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.578116+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:48:54.578116+0000","last_peered":"2024-08-17T17:48:54.578116+0000","last_clean":"2024-08-17T17:48:54.578116+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:48:54.578116+0000","last_undegraded":"2024-08-17T17:48:54.578116+0000","last_fullsized":"2024-08-17T17:48:54.578116+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":338,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606727+0000","last_change":"2024-08-17T17:33:20.287641+0000","last_active":"2024-08-17T17:49:03.606727+0000","last_peered":"2024-08-17T17:49:03.606727+0000","last_clean":"2024-08-17T17:49:03.606727+0000","last_became_active":"2024-08-17T17:33:20.287110+0000","last_became_peered":"2024-08-17T17:33:20.287110+0000","last_unstale":"2024-08-17T17:49:03.606727+0000","last_undegraded":"2024-08-17T17:49:03.606727+0000","last_fullsized":"2024-08-17T17:49:03.606727+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:38.827575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"431'8098","reported_seq":8145,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:18.074788+0000","last_change":"2024-08-17T17:47:33.471160+0000","last_active":"2024-08-17T17:49:18.074788+0000","last_peered":"2024-08-17T17:49:18.074788+0000","last_clean":"2024-08-17T17:49:18.074788+0000","last_became_active":"2024-08-17T17:47:33.470939+0000","last_became_peered":"2024-08-17T17:47:33.470939+0000","last_unstale":"2024-08-17T17:49:18.074788+0000","last_undegraded":"2024-08-17T17:49:18.074788+0000","last_fullsized":"2024-08-17T17:49:18.074788+0000","mapping_epoch":413,"log_start":"419'1200","ondisk_log_start":"419'1200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:09.156891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":498597888,"num_objects":7608,"num_object_clones":0,"num_object_copies":22824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8098,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":345,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.570363+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:48:54.570363+0000","last_peered":"2024-08-17T17:48:54.570363+0000","last_clean":"2024-08-17T17:48:54.570363+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:48:54.570363+0000","last_undegraded":"2024-08-17T17:48:54.570363+0000","last_fullsized":"2024-08-17T17:48:54.570363+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":207,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606628+0000","last_change":"2024-08-17T17:33:40.699715+0000","last_active":"2024-08-17T17:49:03.606628+0000","last_peered":"2024-08-17T17:49:03.606628+0000","last_clean":"2024-08-17T17:49:03.606628+0000","last_became_active":"2024-08-17T17:33:40.698968+0000","last_became_peered":"2024-08-17T17:33:40.698968+0000","last_unstale":"2024-08-17T17:49:03.606628+0000","last_undegraded":"2024-08-17T17:49:03.606628+0000","last_fullsized":"2024-08-17T17:49:03.606628+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:05.521770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":702,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603382+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:49:03.603382+0000","last_peered":"2024-08-17T17:49:03.603382+0000","last_clean":"2024-08-17T17:49:03.603382+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:49:03.603382+0000","last_undegraded":"2024-08-17T17:49:03.603382+0000","last_fullsized":"2024-08-17T17:49:03.603382+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"431'7913","reported_seq":7960,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:14.141429+0000","last_change":"2024-08-17T17:47:33.468618+0000","last_active":"2024-08-17T17:49:14.141429+0000","last_peered":"2024-08-17T17:49:14.141429+0000","last_clean":"2024-08-17T17:49:14.141429+0000","last_became_active":"2024-08-17T17:47:33.468343+0000","last_became_peered":"2024-08-17T17:47:33.468343+0000","last_unstale":"2024-08-17T17:49:14.141429+0000","last_undegraded":"2024-08-17T17:49:14.141429+0000","last_fullsized":"2024-08-17T17:49:14.141429+0000","mapping_epoch":413,"log_start":"419'2000","ondisk_log_start":"419'2000","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5913,"log_dups_size":0,"ondisk_log_size":5913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:35:12.974392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":518586368,"num_objects":7913,"num_object_clones":0,"num_object_copies":23739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7913,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":341,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.573348+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:48:54.573348+0000","last_peered":"2024-08-17T17:48:54.573348+0000","last_clean":"2024-08-17T17:48:54.573348+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:48:54.573348+0000","last_undegraded":"2024-08-17T17:48:54.573348+0000","last_fullsized":"2024-08-17T17:48:54.573348+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":458,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.621395+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:49:03.621395+0000","last_peered":"2024-08-17T17:49:03.621395+0000","last_clean":"2024-08-17T17:49:03.621395+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:49:03.621395+0000","last_undegraded":"2024-08-17T17:49:03.621395+0000","last_fullsized":"2024-08-17T17:49:03.621395+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":483,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.570512+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:48:54.570512+0000","last_peered":"2024-08-17T17:48:54.570512+0000","last_clean":"2024-08-17T17:48:54.570512+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:48:54.570512+0000","last_undegraded":"2024-08-17T17:48:54.570512+0000","last_fullsized":"2024-08-17T17:48:54.570512+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"431'7969","reported_seq":7995,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:14.988477+0000","last_change":"2024-08-17T17:47:33.473836+0000","last_active":"2024-08-17T17:49:14.988477+0000","last_peered":"2024-08-17T17:49:14.988477+0000","last_clean":"2024-08-17T17:49:14.988477+0000","last_became_active":"2024-08-17T17:47:33.473133+0000","last_became_peered":"2024-08-17T17:47:33.473133+0000","last_unstale":"2024-08-17T17:49:14.988477+0000","last_undegraded":"2024-08-17T17:49:14.988477+0000","last_fullsized":"2024-08-17T17:49:14.988477+0000","mapping_epoch":413,"log_start":"418'200","ondisk_log_start":"418'200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7769,"log_dups_size":0,"ondisk_log_size":7769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:19.139546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":516751360,"num_objects":7885,"num_object_clones":0,"num_object_copies":23655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7969,"num_write_kb":507328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":732,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607313+0000","last_change":"2024-08-17T17:33:40.676762+0000","last_active":"2024-08-17T17:49:03.607313+0000","last_peered":"2024-08-17T17:49:03.607313+0000","last_clean":"2024-08-17T17:49:03.607313+0000","last_became_active":"2024-08-17T17:33:40.647433+0000","last_became_peered":"2024-08-17T17:33:40.647433+0000","last_unstale":"2024-08-17T17:49:03.607313+0000","last_undegraded":"2024-08-17T17:49:03.607313+0000","last_fullsized":"2024-08-17T17:49:03.607313+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:23:42.529692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":701,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603518+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:49:03.603518+0000","last_peered":"2024-08-17T17:49:03.603518+0000","last_clean":"2024-08-17T17:49:03.603518+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:49:03.603518+0000","last_undegraded":"2024-08-17T17:49:03.603518+0000","last_fullsized":"2024-08-17T17:49:03.603518+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":719,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606814+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:49:03.606814+0000","last_peered":"2024-08-17T17:49:03.606814+0000","last_clean":"2024-08-17T17:49:03.606814+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:49:03.606814+0000","last_undegraded":"2024-08-17T17:49:03.606814+0000","last_fullsized":"2024-08-17T17:49:03.606814+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"431'8204","reported_seq":8229,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:15.042039+0000","last_change":"2024-08-17T17:47:33.468582+0000","last_active":"2024-08-17T17:49:15.042039+0000","last_peered":"2024-08-17T17:49:15.042039+0000","last_clean":"2024-08-17T17:49:15.042039+0000","last_became_active":"2024-08-17T17:47:33.468018+0000","last_became_peered":"2024-08-17T17:47:33.468018+0000","last_unstale":"2024-08-17T17:49:15.042039+0000","last_undegraded":"2024-08-17T17:49:15.042039+0000","last_fullsized":"2024-08-17T17:49:15.042039+0000","mapping_epoch":413,"log_start":"418'500","ondisk_log_start":"418'500","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:55:27.900073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":532545536,"num_objects":8126,"num_object_clones":0,"num_object_copies":24378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8204,"num_write_kb":522560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":391,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.624397+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:49:03.624397+0000","last_peered":"2024-08-17T17:49:03.624397+0000","last_clean":"2024-08-17T17:49:03.624397+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:49:03.624397+0000","last_undegraded":"2024-08-17T17:49:03.624397+0000","last_fullsized":"2024-08-17T17:49:03.624397+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":435,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.587886+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:48:54.587886+0000","last_peered":"2024-08-17T17:48:54.587886+0000","last_clean":"2024-08-17T17:48:54.587886+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:48:54.587886+0000","last_undegraded":"2024-08-17T17:48:54.587886+0000","last_fullsized":"2024-08-17T17:48:54.587886+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":385,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.580302+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:48:54.580302+0000","last_peered":"2024-08-17T17:48:54.580302+0000","last_clean":"2024-08-17T17:48:54.580302+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:48:54.580302+0000","last_undegraded":"2024-08-17T17:48:54.580302+0000","last_fullsized":"2024-08-17T17:48:54.580302+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":385,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577986+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:48:54.577986+0000","last_peered":"2024-08-17T17:48:54.577986+0000","last_clean":"2024-08-17T17:48:54.577986+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:48:54.577986+0000","last_undegraded":"2024-08-17T17:48:54.577986+0000","last_fullsized":"2024-08-17T17:48:54.577986+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":428,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577236+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:48:54.577236+0000","last_peered":"2024-08-17T17:48:54.577236+0000","last_clean":"2024-08-17T17:48:54.577236+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:48:54.577236+0000","last_undegraded":"2024-08-17T17:48:54.577236+0000","last_fullsized":"2024-08-17T17:48:54.577236+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":707,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.603256+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:49:03.603256+0000","last_peered":"2024-08-17T17:49:03.603256+0000","last_clean":"2024-08-17T17:49:03.603256+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:49:03.603256+0000","last_undegraded":"2024-08-17T17:49:03.603256+0000","last_fullsized":"2024-08-17T17:49:03.603256+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":247,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607343+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:49:03.607343+0000","last_peered":"2024-08-17T17:49:03.607343+0000","last_clean":"2024-08-17T17:49:03.607343+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:49:03.607343+0000","last_undegraded":"2024-08-17T17:49:03.607343+0000","last_fullsized":"2024-08-17T17:49:03.607343+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":335,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.570569+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:48:54.570569+0000","last_peered":"2024-08-17T17:48:54.570569+0000","last_clean":"2024-08-17T17:48:54.570569+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:48:54.570569+0000","last_undegraded":"2024-08-17T17:48:54.570569+0000","last_fullsized":"2024-08-17T17:48:54.570569+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":665,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.608789+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:49:03.608789+0000","last_peered":"2024-08-17T17:49:03.608789+0000","last_clean":"2024-08-17T17:49:03.608789+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:49:03.608789+0000","last_undegraded":"2024-08-17T17:49:03.608789+0000","last_fullsized":"2024-08-17T17:49:03.608789+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":479,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.583732+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:48:54.583732+0000","last_peered":"2024-08-17T17:48:54.583732+0000","last_clean":"2024-08-17T17:48:54.583732+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:48:54.583732+0000","last_undegraded":"2024-08-17T17:48:54.583732+0000","last_fullsized":"2024-08-17T17:48:54.583732+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":659,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606728+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:49:03.606728+0000","last_peered":"2024-08-17T17:49:03.606728+0000","last_clean":"2024-08-17T17:49:03.606728+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:49:03.606728+0000","last_undegraded":"2024-08-17T17:49:03.606728+0000","last_fullsized":"2024-08-17T17:49:03.606728+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":402,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.587797+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:48:54.587797+0000","last_peered":"2024-08-17T17:48:54.587797+0000","last_clean":"2024-08-17T17:48:54.587797+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:48:54.587797+0000","last_undegraded":"2024-08-17T17:48:54.587797+0000","last_fullsized":"2024-08-17T17:48:54.587797+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":691,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606909+0000","last_change":"2024-08-17T17:24:36.994782+0000","last_active":"2024-08-17T17:49:03.606909+0000","last_peered":"2024-08-17T17:49:03.606909+0000","last_clean":"2024-08-17T17:49:03.606909+0000","last_became_active":"2024-08-17T17:24:36.994354+0000","last_became_peered":"2024-08-17T17:24:36.994354+0000","last_unstale":"2024-08-17T17:49:03.606909+0000","last_undegraded":"2024-08-17T17:49:03.606909+0000","last_fullsized":"2024-08-17T17:49:03.606909+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:46.154579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":407,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607573+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:49:03.607573+0000","last_peered":"2024-08-17T17:49:03.607573+0000","last_clean":"2024-08-17T17:49:03.607573+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:49:03.607573+0000","last_undegraded":"2024-08-17T17:49:03.607573+0000","last_fullsized":"2024-08-17T17:49:03.607573+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":699,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.602915+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:49:03.602915+0000","last_peered":"2024-08-17T17:49:03.602915+0000","last_clean":"2024-08-17T17:49:03.602915+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:49:03.602915+0000","last_undegraded":"2024-08-17T17:49:03.602915+0000","last_fullsized":"2024-08-17T17:49:03.602915+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":658,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607221+0000","last_change":"2024-08-17T17:12:40.338533+0000","last_active":"2024-08-17T17:49:03.607221+0000","last_peered":"2024-08-17T17:49:03.607221+0000","last_clean":"2024-08-17T17:49:03.607221+0000","last_became_active":"2024-08-17T17:12:40.338222+0000","last_became_peered":"2024-08-17T17:12:40.338222+0000","last_unstale":"2024-08-17T17:49:03.607221+0000","last_undegraded":"2024-08-17T17:49:03.607221+0000","last_fullsized":"2024-08-17T17:49:03.607221+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:59.096217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":414,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577574+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:48:54.577574+0000","last_peered":"2024-08-17T17:48:54.577574+0000","last_clean":"2024-08-17T17:48:54.577574+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:48:54.577574+0000","last_undegraded":"2024-08-17T17:48:54.577574+0000","last_fullsized":"2024-08-17T17:48:54.577574+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":347,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.606893+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:49:03.606893+0000","last_peered":"2024-08-17T17:49:03.606893+0000","last_clean":"2024-08-17T17:49:03.606893+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:49:03.606893+0000","last_undegraded":"2024-08-17T17:49:03.606893+0000","last_fullsized":"2024-08-17T17:49:03.606893+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":403,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.621333+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:49:03.621333+0000","last_peered":"2024-08-17T17:49:03.621333+0000","last_clean":"2024-08-17T17:49:03.621333+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:49:03.621333+0000","last_undegraded":"2024-08-17T17:49:03.621333+0000","last_fullsized":"2024-08-17T17:49:03.621333+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":422,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577922+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:48:54.577922+0000","last_peered":"2024-08-17T17:48:54.577922+0000","last_clean":"2024-08-17T17:48:54.577922+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:48:54.577922+0000","last_undegraded":"2024-08-17T17:48:54.577922+0000","last_fullsized":"2024-08-17T17:48:54.577922+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":285,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577157+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:48:54.577157+0000","last_peered":"2024-08-17T17:48:54.577157+0000","last_clean":"2024-08-17T17:48:54.577157+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:48:54.577157+0000","last_undegraded":"2024-08-17T17:48:54.577157+0000","last_fullsized":"2024-08-17T17:48:54.577157+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":422,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.577458+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:48:54.577458+0000","last_peered":"2024-08-17T17:48:54.577458+0000","last_clean":"2024-08-17T17:48:54.577458+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:48:54.577458+0000","last_undegraded":"2024-08-17T17:48:54.577458+0000","last_fullsized":"2024-08-17T17:48:54.577458+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":285,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.578739+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:48:54.578739+0000","last_peered":"2024-08-17T17:48:54.578739+0000","last_clean":"2024-08-17T17:48:54.578739+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:48:54.578739+0000","last_undegraded":"2024-08-17T17:48:54.578739+0000","last_fullsized":"2024-08-17T17:48:54.578739+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":711,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.608991+0000","last_change":"2024-08-17T17:33:20.286732+0000","last_active":"2024-08-17T17:49:03.608991+0000","last_peered":"2024-08-17T17:49:03.608991+0000","last_clean":"2024-08-17T17:49:03.608991+0000","last_became_active":"2024-08-17T17:33:20.286170+0000","last_became_peered":"2024-08-17T17:33:20.286170+0000","last_unstale":"2024-08-17T17:49:03.608991+0000","last_undegraded":"2024-08-17T17:49:03.608991+0000","last_fullsized":"2024-08-17T17:49:03.608991+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:13:26.487319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":685,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.607744+0000","last_change":"2024-08-17T17:33:20.285890+0000","last_active":"2024-08-17T17:49:03.607744+0000","last_peered":"2024-08-17T17:49:03.607744+0000","last_clean":"2024-08-17T17:49:03.607744+0000","last_became_active":"2024-08-17T17:33:20.283972+0000","last_became_peered":"2024-08-17T17:33:20.283972+0000","last_unstale":"2024-08-17T17:49:03.607744+0000","last_undegraded":"2024-08-17T17:49:03.607744+0000","last_fullsized":"2024-08-17T17:49:03.607744+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:37.631421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":385,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-17T17:48:54.576861+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:48:54.576861+0000","last_peered":"2024-08-17T17:48:54.576861+0000","last_clean":"2024-08-17T17:48:54.576861+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:48:54.576861+0000","last_undegraded":"2024-08-17T17:48:54.576861+0000","last_fullsized":"2024-08-17T17:48:54.576861+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":406,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-17T17:49:03.621469+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:49:03.621469+0000","last_peered":"2024-08-17T17:49:03.621469+0000","last_clean":"2024-08-17T17:49:03.621469+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:49:03.621469+0000","last_undegraded":"2024-08-17T17:49:03.621469+0000","last_fullsized":"2024-08-17T17:49:03.621469+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":8245936152,"num_objects":125824,"num_object_clones":0,"num_object_copies":377472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":125824,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129744,"num_write_kb":8178113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24740503552,"data_stored":24740495408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109044,"ondisk_log_size":109044,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170194,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463564,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5012916,"kb_used_data":3515884,"kb_used_omap":1,"kb_used_meta":1496830,"kb_avail":88732236,"statfs":{"total":95995035648,"available":90861809664,"internally_reserved":0,"allocated":3600265216,"data_stored":3597884394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1532754264},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492238,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6593624,"kb_used_data":5616336,"kb_used_omap":7,"kb_used_meta":977208,"kb_avail":87151528,"statfs":{"total":95995035648,"available":89243164672,"internally_reserved":0,"allocated":5751128064,"data_stored":5748769542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":1000661325},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986830,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7532588,"kb_used_data":6032164,"kb_used_omap":1,"kb_used_meta":1500350,"kb_avail":86212564,"statfs":{"total":95995035648,"available":88281665536,"internally_reserved":0,"allocated":6176935936,"data_stored":6174561982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1536358744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,0,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225658,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4557340,"kb_used_data":3479188,"kb_used_omap":4,"kb_used_meta":1078139,"kb_avail":89187812,"statfs":{"total":95995035648,"available":91328319488,"internally_reserved":0,"allocated":3562688512,"data_stored":3560326216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1104014784},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075982,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533875,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4277084,"kb_used_data":3058772,"kb_used_omap":5,"kb_used_meta":1218298,"kb_avail":89468068,"statfs":{"total":95995035648,"available":91615301632,"internally_reserved":0,"allocated":3132182528,"data_stored":3129819502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1247537392},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089852,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3685300,"kb_used_data":2544044,"kb_used_omap":5,"kb_used_meta":1141242,"kb_avail":90059852,"statfs":{"total":95995035648,"available":92221288448,"internally_reserved":0,"allocated":2605101056,"data_stored":2602720293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1168631867},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2592800768,"data_stored":2592800768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3139764224,"data_stored":3139764224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3545960448,"data_stored":3545956376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":6137057280,"data_stored":6137053208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":5745213440,"data_stored":5745213440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3579707392,"data_stored":3579707392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:49:22.767 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:49:22.767 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:22.767 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:49:22.767 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.558233 2024-08-17T17:49:22.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 3 0.5582331645102878 2024-08-17T17:49:22.791 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:49:22.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:22.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:22.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:22.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:23.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2072226511' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:22 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:22 smithi152 ceph-mon[93456]: pgmap v1735: 89 pgs: 89 active+clean; 7.4 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 85 B/s rd, 28 MiB/s wr, 965 op/s 2024-08-17T17:49:23.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2072226511' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:22 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:23.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:22 smithi142 ceph-mon[103614]: pgmap v1735: 89 pgs: 89 active+clean; 7.4 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 85 B/s rd, 28 MiB/s wr, 965 op/s 2024-08-17T17:49:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2072226511' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:22 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:23.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:22 smithi142 ceph-mon[106729]: pgmap v1735: 89 pgs: 89 active+clean; 7.4 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 85 B/s rd, 28 MiB/s wr, 965 op/s 2024-08-17T17:49:23.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:23.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:23.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:23.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:23.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:23.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:23.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:23.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:23.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:23.680 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:23.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.874 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:23.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:23.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.021 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:23 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:23 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:49:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:23 smithi152 ceph-mon[93456]: osdmap e432: 8 total, 8 up, 6 in 2024-08-17T17:49:24.113 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:23 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:24.113 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:23 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:49:24.113 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:23 smithi142 ceph-mon[106729]: osdmap e432: 8 total, 8 up, 6 in 2024-08-17T17:49:24.113 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:23 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:24.114 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:23 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:49:24.114 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:23 smithi142 ceph-mon[103614]: osdmap e432: 8 total, 8 up, 6 in 2024-08-17T17:49:24.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:24.166 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:24.168 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:24.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:24.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:24.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:24.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:24.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:24.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.720 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:24.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:24.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:24.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:24.986 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:24.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:24.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:24 smithi142 ceph-mon[106729]: pgmap v1737: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 968 op/s 2024-08-17T17:49:24.995 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:24 smithi142 ceph-mon[103614]: pgmap v1737: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 968 op/s 2024-08-17T17:49:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:24 smithi152 ceph-mon[93456]: pgmap v1737: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 968 op/s 2024-08-17T17:49:25.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:25.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:25.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:25.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:25.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:25.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:25.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:25.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:25.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:25.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:25.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:25.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:25.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:25.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:25.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:25.922 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:26.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:26.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:26.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:49:26.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:26.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:26.317 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:49:26.317 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:26.329 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:26.349 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:49:26.349 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:26.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:26.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:26.379 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:49:26.380 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:26.410 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:49:26.411 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:26.440 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:49:26.441 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:26.441 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:26 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1001920613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:26.441 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:26 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1001920613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:26.471 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:49:26.471 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:26.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:26.501 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:49:26.501 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:26.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:26.520 INFO:teuthology.orchestra.run.smithi142.stdout:osd.7: {} 2024-08-17T17:49:26.521 INFO:teuthology.orchestra.run.smithi142.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:49:26.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:26.662 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:26.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:26.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:26 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1001920613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:26.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:26.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:26.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:27.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:27.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:27.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:27.210 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:27.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:27.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:27.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:27.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:27.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:27.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:27.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:27.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:27.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:27.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:27.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:27 smithi152 ceph-mon[93456]: pgmap v1738: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 102 B/s rd, 6.2 MiB/s wr, 979 op/s 2024-08-17T17:49:27.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:27 smithi142 ceph-mon[106729]: pgmap v1738: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 102 B/s rd, 6.2 MiB/s wr, 979 op/s 2024-08-17T17:49:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:27 smithi142 ceph-mon[103614]: pgmap v1738: 89 pgs: 89 active+clean; 7.3 GiB data, 29 GiB used, 507 GiB / 536 GiB avail; 102 B/s rd, 6.2 MiB/s wr, 979 op/s 2024-08-17T17:49:27.898 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:28.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:28.037 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:28.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:28.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:28.278 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:28.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:28.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:28.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:28.545 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:28.574 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:28.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:28.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:28.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:28.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:28.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:28.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:28.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:28.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:29.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:29.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:29.190 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:29.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:29.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:29.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:29.462 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:29.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:29.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:29.636 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:29.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:29.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:29.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:29.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:29 smithi152 ceph-mon[93456]: pgmap v1739: 89 pgs: 89 active+clean; 7.1 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 192 KiB/s wr, 982 op/s 2024-08-17T17:49:29.860 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:29 smithi142 ceph-mon[106729]: pgmap v1739: 89 pgs: 89 active+clean; 7.1 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 192 KiB/s wr, 982 op/s 2024-08-17T17:49:29.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:29 smithi142 ceph-mon[103614]: pgmap v1739: 89 pgs: 89 active+clean; 7.1 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 192 KiB/s wr, 982 op/s 2024-08-17T17:49:29.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:29.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:29.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:30.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.036 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:30.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:30.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:30.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:30.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:30.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:49:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:49:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:30.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:30.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:30.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:30.478 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:30.495 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:30.495 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:30.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:30.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:30.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:30.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:30.752 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:30.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:30.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:31.019 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:31.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:31.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:31.156 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:31.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.300 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:31.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:31.401 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:31.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:31.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:31.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:31.559 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.3 primary-affinity to 0.558233 (8365842) 2024-08-17T17:49:31.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:31.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:31.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:49:31.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:31.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:31.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-mon[106729]: pgmap v1740: 89 pgs: 89 active+clean; 7.0 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 0 B/s wr, 986 op/s 2024-08-17T17:49:31.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2049346041' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878} : dispatch 2024-08-17T17:49:31.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878} : dispatch 2024-08-17T17:49:31.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-mon[103614]: pgmap v1740: 89 pgs: 89 active+clean; 7.0 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 0 B/s wr, 986 op/s 2024-08-17T17:49:31.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2049346041' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878} : dispatch 2024-08-17T17:49:31.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878} : dispatch 2024-08-17T17:49:31.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:31 smithi152 ceph-mon[93456]: pgmap v1740: 89 pgs: 89 active+clean; 7.0 GiB data, 28 GiB used, 508 GiB / 536 GiB avail; 0 B/s wr, 986 op/s 2024-08-17T17:49:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2049346041' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878} : dispatch 2024-08-17T17:49:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878} : dispatch 2024-08-17T17:49:31.836 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:31.938 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:31.960 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:32.057 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:49:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:49:31] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-08-17T17:49:32.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:32.084 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:32.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:32.185 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:32.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:32.308 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:32.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:32.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:32.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:32.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:49:32.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:32.581 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:32.683 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:32.713 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:32.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878}]': finished 2024-08-17T17:49:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:32 smithi152 ceph-mon[93456]: osdmap e433: 8 total, 8 up, 6 in 2024-08-17T17:49:32.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:32 smithi152 ceph-mon[93456]: pgmap v1742: 89 pgs: 5 peering, 84 active+clean; 6.8 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:32.845 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878}]': finished 2024-08-17T17:49:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:32 smithi142 ceph-mon[106729]: osdmap e433: 8 total, 8 up, 6 in 2024-08-17T17:49:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:32 smithi142 ceph-mon[106729]: pgmap v1742: 89 pgs: 5 peering, 84 active+clean; 6.8 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5582331645102878}]': finished 2024-08-17T17:49:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:32 smithi142 ceph-mon[103614]: osdmap e433: 8 total, 8 up, 6 in 2024-08-17T17:49:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:32 smithi142 ceph-mon[103614]: pgmap v1742: 89 pgs: 5 peering, 84 active+clean; 6.8 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:32.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:32.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:33.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:33.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:33.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:33.257 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:33.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:33.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:33.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:33.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:33.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:33.572 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:33.606 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:33.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:33.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:33.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:33.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:33.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.785 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:33.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:33 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:33.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:33 smithi152 ceph-mon[93456]: osdmap e434: 8 total, 8 up, 6 in 2024-08-17T17:49:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:33 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:33.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:33 smithi142 ceph-mon[106729]: osdmap e434: 8 total, 8 up, 6 in 2024-08-17T17:49:33.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:33 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:49:33.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:33 smithi142 ceph-mon[103614]: osdmap e434: 8 total, 8 up, 6 in 2024-08-17T17:49:33.887 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:33.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:34.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:34.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:34.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:34.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:34.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:34.310 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:34.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:34.439 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:34.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:34.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:34.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:34.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:34.734 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:34 smithi142 ceph-mon[106729]: pgmap v1744: 89 pgs: 5 peering, 84 active+clean; 6.8 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:49:34.734 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:34 smithi142 ceph-mon[103614]: pgmap v1744: 89 pgs: 5 peering, 84 active+clean; 6.8 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:49:34.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:34.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:34.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:34.980 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:34.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:34 smithi152 ceph-mon[93456]: pgmap v1744: 89 pgs: 5 peering, 84 active+clean; 6.8 GiB data, 28 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:49:35.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:35.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:35.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:35.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:35.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:35.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:35.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:35.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:35.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:35.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:35.594 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:49:35.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:35.658 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:35.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:35.760 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:35.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2716309976' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:35 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2716309976' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:35.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:35 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:35.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:35.931 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:36.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:36.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2716309976' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:36.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:35 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:36.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:36.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:36.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:36.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:36.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:36.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:36.482 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:36.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:36.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:36.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:36.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:36.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.760 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:36.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:36.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:36.867 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[103614]: osdmap e435: 8 total, 8 up, 6 in 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[103614]: pgmap v1746: 89 pgs: 5 peering, 84 active+clean; 6.7 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[106729]: osdmap e435: 8 total, 8 up, 6 in 2024-08-17T17:49:36.868 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:36 smithi142 ceph-mon[106729]: pgmap v1746: 89 pgs: 5 peering, 84 active+clean; 6.7 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:49:36.891 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:36.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:36.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:37.027 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:37.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:36 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:36 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:36 smithi152 ceph-mon[93456]: osdmap e435: 8 total, 8 up, 6 in 2024-08-17T17:49:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:36 smithi152 ceph-mon[93456]: pgmap v1746: 89 pgs: 5 peering, 84 active+clean; 6.7 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T17:49:37.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:37.156 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:37.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:37.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:37.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:37.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:37.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:37.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:37.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:37.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:37.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:37.775 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:37.801 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:37.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:37.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:37.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:37.954 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:49:37.954 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:37.955 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 3, 4, 6, 5, 7] 2024-08-17T17:49:37.955 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:49:37.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:49:38.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:38.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:38.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:38.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:38.302 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:38.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:38 smithi152 systemd[1]: Stopping Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:49:38.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:38.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:38.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.470 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:38.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:38.548 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:38.548 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:38.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:38.603 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:38.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:38.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:38.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:38.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:38 smithi152 ceph-mon[93456]: osd.7 marked itself down and dead 2024-08-17T17:49:38.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[149251]: 2024-08-17T17:49:38.477+0000 7f3c1c29a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:49:38.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[149251]: 2024-08-17T17:49:38.477+0000 7f3c1c29a640 -1 osd.7 435 *** Got signal Terminated *** 2024-08-17T17:49:38.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:38 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[149251]: 2024-08-17T17:49:38.477+0000 7f3c1c29a640 -1 osd.7 435 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:49:38.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:38 smithi142 ceph-mon[106729]: osd.7 marked itself down and dead 2024-08-17T17:49:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:38 smithi142 ceph-mon[103614]: osd.7 marked itself down and dead 2024-08-17T17:49:38.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:38.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:38.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:39.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:39.043 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:49:39.043 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:39.071 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:49:39.071 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:39.094 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:49:39.094 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:39.114 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:49:39.115 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:39.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:39.150 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:49:39.151 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:39.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:39.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:39.183 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:49:39.184 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:39.207 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:49:39.207 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:39.208 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:49:39.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:39.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:39.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:39.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:39.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:39.486 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:39 smithi152 podman[273381]: 2024-08-17 17:49:39.19038782 +0000 UTC m=+0.865366502 container died bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:49:39.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:39.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:39.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:39.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:39.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:39.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:39 smithi152 podman[273381]: 2024-08-17 17:49:39.684106962 +0000 UTC m=+1.359085645 container cleanup bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:49:39.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:39 smithi152 bash[273381]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:49:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:39 smithi152 ceph-mon[93456]: pgmap v1747: 89 pgs: 89 active+clean; 6.5 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 989 op/s 2024-08-17T17:49:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:39 smithi152 ceph-mon[93456]: osdmap e436: 8 total, 7 up, 6 in 2024-08-17T17:49:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:39 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-08-17T17:49:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/717605064' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:39.842 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:39.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[106729]: pgmap v1747: 89 pgs: 89 active+clean; 6.5 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 989 op/s 2024-08-17T17:49:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[106729]: osdmap e436: 8 total, 7 up, 6 in 2024-08-17T17:49:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-08-17T17:49:39.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/717605064' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:39.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[103614]: pgmap v1747: 89 pgs: 89 active+clean; 6.5 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 989 op/s 2024-08-17T17:49:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[103614]: osdmap e436: 8 total, 7 up, 6 in 2024-08-17T17:49:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-08-17T17:49:39.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/717605064' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:39.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:49:39.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:39.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:40.044 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:49:40.044 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:49:40.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:40.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:40.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:40.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:40.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:40 smithi152 podman[273467]: 2024-08-17 17:49:40.053249106 +0000 UTC m=+0.843608244 container remove bbfbc97a363d9b9b27d41e638261e1ead811509e74c550776f827a3fb18ab4a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:49:40.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:49:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:49:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:40.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:40.398 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:49:40.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:40.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:40.478 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:40.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:40.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:40.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:40.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:40.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.696 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:40.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:40.763 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:40 smithi152 podman[273661]: 2024-08-17 17:49:40.552632109 +0000 UTC m=+0.039910169 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:49:40.763 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:40 smithi152 podman[273661]: 2024-08-17 17:49:40.680311987 +0000 UTC m=+0.167590031 container create 4099a026d200d6b9eec2499c2e5a23617df426f0d04cb3c21311bd66b22fe360 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:49:40.798 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:40.844 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:40.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:40.946 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:40.989 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:41.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:41.136 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:41.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:41.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.269 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:41.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:41.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:41.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:41.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:41.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:41.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:41.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:41.746 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:41 smithi142 ceph-mon[103614]: pgmap v1749: 89 pgs: 89 active+clean; 6.4 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 872 op/s 2024-08-17T17:49:41.746 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:41.746 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:41 smithi142 ceph-mon[106729]: pgmap v1749: 89 pgs: 89 active+clean; 6.4 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 872 op/s 2024-08-17T17:49:41.747 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:41.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:41.809 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:41 smithi152 ceph-mon[93456]: pgmap v1749: 89 pgs: 89 active+clean; 6.4 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 872 op/s 2024-08-17T17:49:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:41.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:41.933 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:41.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:42.035 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:42.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:49:42.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:42.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:42.088 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:49:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:49:41] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:49:42.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:42.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:42.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:42.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:42.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:42.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:42.454 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:42.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:42.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:42.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:42 smithi152 podman[273661]: 2024-08-17 17:49:42.261132264 +0000 UTC m=+1.748410313 container init 4099a026d200d6b9eec2499c2e5a23617df426f0d04cb3c21311bd66b22fe360 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T17:49:42.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:42 smithi152 podman[273661]: 2024-08-17 17:49:42.275861642 +0000 UTC m=+1.763139685 container start 4099a026d200d6b9eec2499c2e5a23617df426f0d04cb3c21311bd66b22fe360 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:49:42.589 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:42.691 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:42.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:42.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:42.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:42.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:42.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:42.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:42.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:43.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:43.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:43.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:43.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:43.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:43.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:43.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:43.505 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:43.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:43.564 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:49:43.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:43.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:43.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:43.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:43.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:43 smithi152 ceph-mon[93456]: pgmap v1750: 89 pgs: 89 active+clean; 6.3 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2004121120' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:43.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:43 smithi152 podman[273661]: 2024-08-17 17:49:43.347297179 +0000 UTC m=+2.834575225 container attach 4099a026d200d6b9eec2499c2e5a23617df426f0d04cb3c21311bd66b22fe360 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:49:43.823 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:49:43 smithi152 podman[273661]: 2024-08-17 17:49:43.348075854 +0000 UTC m=+2.835353890 container died 4099a026d200d6b9eec2499c2e5a23617df426f0d04cb3c21311bd66b22fe360 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:49:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:43 smithi142 ceph-mon[106729]: pgmap v1750: 89 pgs: 89 active+clean; 6.3 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2004121120' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:43 smithi142 ceph-mon[103614]: pgmap v1750: 89 pgs: 89 active+clean; 6.3 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2004121120' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:43.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:43.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:44.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:44.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:44.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:44.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:44.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:44.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:44.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:44.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:44.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:44.451 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:44.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.583 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:44.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:44.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.706 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:44.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:44.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:44 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2004121120' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:44.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:44 smithi152 ceph-mon[93456]: osdmap e437: 8 total, 7 up, 6 in 2024-08-17T17:49:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:44 smithi152 ceph-mon[93456]: pgmap v1752: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:44.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2004121120' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[106729]: osdmap e437: 8 total, 7 up, 6 in 2024-08-17T17:49:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[106729]: pgmap v1752: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2004121120' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[103614]: osdmap e437: 8 total, 7 up, 6 in 2024-08-17T17:49:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:44 smithi142 ceph-mon[103614]: pgmap v1752: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:49:44.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:44.964 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:45.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.097 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:45.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:45.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:45.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:45.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:45.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:45.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:45.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:45.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:45.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:45.631 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:45.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:45.779 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:45.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:45.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:45.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:45.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:46.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.039 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:46.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:46.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:46.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:46.199 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:49:46.199 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:46.224 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:49:46.225 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:46.256 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:49:46.256 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:46.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:46.275 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:49:46.275 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:46.300 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:49:46.300 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:46.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:46.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:46.332 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:49:46.333 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:46.359 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:49:46.359 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:46.359 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:49:46.409 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:46.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:46.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:46.536 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:46.557 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:46.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:46 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4140915036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:46.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:46 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4140915036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:46.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:46 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4140915036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:46.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:46.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:46.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:46.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:46.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:46.948 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:47.050 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:47.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:47.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:47.160 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:49:47.160 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:49:47.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:49:47.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:47.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:47.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:47.238 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:49:47.238 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:49:47.246 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:47.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:47.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.356 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:47.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:47.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:47 smithi152 ceph-mon[93456]: pgmap v1753: 89 pgs: 89 active+clean; 6.1 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 819 op/s 2024-08-17T17:49:47.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:47 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:47.458 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:47.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:47.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:47.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:47.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:47 smithi142 ceph-mon[106729]: pgmap v1753: 89 pgs: 89 active+clean; 6.1 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 819 op/s 2024-08-17T17:49:47.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:47 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:47 smithi142 ceph-mon[103614]: pgmap v1753: 89 pgs: 89 active+clean; 6.1 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 819 op/s 2024-08-17T17:49:47.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:47 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:47.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:47.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:49:47.752 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:47.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:47.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:47.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:47.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:48.037 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:48.065 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:48.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:48.166 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:48.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:48.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:48.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:48.431 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:48.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:48.569 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:48.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:48.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:48.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:48.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:48.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:48.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:48.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:48.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:48.978 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:49.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:49.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:49.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:49:49.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:49.234 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:49.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:49.326 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:49.327 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:49.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:49.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:49.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:49.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:49.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:49.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:49.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.636 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:49.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:49.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:49.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:49.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:49 smithi152 ceph-mon[93456]: pgmap v1754: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 908 op/s 2024-08-17T17:49:49.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:49 smithi142 ceph-mon[106729]: pgmap v1754: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 908 op/s 2024-08-17T17:49:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:49 smithi142 ceph-mon[103614]: pgmap v1754: 89 pgs: 89 active+clean; 6.0 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 908 op/s 2024-08-17T17:49:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:49.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:49.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:50.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:50.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:49:50.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:50.156 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:50.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:50.258 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:50.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:50.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:49:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:49:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:50.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:50.436 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:50.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:50.581 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:50.585 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:50.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:50.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:50.729 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:50.831 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:50.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:50.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:51.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:51.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:51.030 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:51.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:51.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:51.259 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:51.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:51.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:51.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:51.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:51.565 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:49:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:51.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-mon[106729]: pgmap v1755: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 889 op/s 2024-08-17T17:49:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1550285790' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-mon[103614]: pgmap v1755: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 889 op/s 2024-08-17T17:49:51.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1550285790' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:51.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:51.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:51.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:51.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:51 smithi152 ceph-mon[93456]: pgmap v1755: 89 pgs: 89 active+clean; 5.9 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 889 op/s 2024-08-17T17:49:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1550285790' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:51.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:51 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:51.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:51.925 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:49:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:49:51] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:49:51.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:51.978 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:52.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:52.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:52.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:52.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:52.239 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:49:52.239 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:52.239 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:49:52.239 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 1.000000 2024-08-17T17:49:52.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 4 1 2024-08-17T17:49:52.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:52.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:52.405 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:52.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:52.545 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:52.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:52.647 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:52.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:52.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.825 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:52.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:52.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:52 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:52.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:52 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:49:52.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:52 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:52.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:52 smithi152 ceph-mon[93456]: osdmap e438: 8 total, 7 up, 6 in 2024-08-17T17:49:52.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:52 smithi152 ceph-mon[93456]: pgmap v1757: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 980 op/s 2024-08-17T17:49:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:49:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[106729]: osdmap e438: 8 total, 7 up, 6 in 2024-08-17T17:49:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[106729]: pgmap v1757: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 980 op/s 2024-08-17T17:49:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:49:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[103614]: osdmap e438: 8 total, 7 up, 6 in 2024-08-17T17:49:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:52 smithi142 ceph-mon[103614]: pgmap v1757: 89 pgs: 89 active+clean; 5.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 980 op/s 2024-08-17T17:49:52.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:52.957 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:53.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:53.084 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:53.139 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:53.139 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:53.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:53.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:53.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:53.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:53.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:53.486 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:53.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.644 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:53.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:53.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:53.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:53.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:53.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:49:53.897 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:53.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:54.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:54.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:54.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:54.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:54.320 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:54.357 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:54.426 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:54.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:54.523 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:54.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:54.594 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:54.696 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:54.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:54.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:54.821 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:54.851 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:54.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:55.002 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:55.104 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:55.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:55.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:55.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:55.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:55.394 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:55.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:55.532 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:55.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:55.667 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:55.712 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:55 smithi152 ceph-mon[93456]: pgmap v1758: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:49:55.754 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:49:55.754 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:55.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:55.775 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:49:55.775 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:55.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:55.804 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:49:55.804 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:55.804 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:55 smithi142 ceph-mon[103614]: pgmap v1758: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:49:55.805 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:55 smithi142 ceph-mon[106729]: pgmap v1758: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:49:55.825 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:49:55.825 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:55.845 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:49:55.846 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:55.875 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:49:55.875 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:55.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:55.903 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:49:55.903 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:55.905 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:49:55.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:56.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:56.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:56.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:56.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:56.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:56.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:56.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:56.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:56.454 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:49:56.454 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198347 12.7 12:edb18e09:::benchmark_data_smithi142_261410_object80150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:37.631482+0000", 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 318.818567051, 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1475268830000001, 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198347 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631482+0000", 2024-08-17T17:49:56.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631482+0000", 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631480+0000", 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631486+0000", 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6969999999999998e-06 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631499+0000", 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2653000000000001e-05 2024-08-17T17:49:56.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631504+0000", 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0919999999999997e-06 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.662521+0000", 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031017335 2024-08-17T17:49:56.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.666399+0000", 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038782230000000001 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.697112+0000", 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030712891999999999 2024-08-17T17:49:56.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.701434+0000", 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043223760000000002 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.663607+0000", 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96217242300000005 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.667006+0000", 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033988130000000001 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.673808+0000", 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068021929999999998 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.696925+0000", 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023116927999999998 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.698058+0000", 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011333669999999999 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.699204+0000", 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011459829999999999 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.767572+0000", 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.068367791999999997 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.769521+0000", 2024-08-17T17:49:56.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019486320000000001 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.779009+0000", 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094880050000000007 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.465 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198563 12.7 12:e8027f39:::benchmark_data_smithi142_261410_object80366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:38.487489+0000", 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 317.96256004899999, 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1351648779999999, 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198563 2024-08-17T17:49:56.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487489+0000", 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487489+0000", 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487487+0000", 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487494+0000", 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3210000000000001e-06 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487507+0000", 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.345e-05 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487513+0000", 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.806e-06 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.575081+0000", 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.087567542999999998 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.579769+0000", 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046882110000000003 2024-08-17T17:49:56.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.645321+0000", 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.065552113999999995 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.649276+0000", 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039547619999999997 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.569789+0000", 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92051316299999997 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.573243+0000", 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034539979999999998 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:56.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.579849+0000", 2024-08-17T17:49:56.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066063770000000001 2024-08-17T17:49:56.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.607563+0000", 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027713550999999999 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.608747+0000", 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001184428 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.611243+0000", 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024959159999999999 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.612429+0000", 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00118545 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.614361+0000", 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019328329999999999 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.622654+0000", 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082922659999999995 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.480 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198764 12.7 12:e75f9baa:::benchmark_data_smithi142_261410_object80567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:39.363457+0000", 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 317.08659129199998, 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.131761899, 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.481 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198764 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363457+0000", 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363457+0000", 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363456+0000", 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363462+0000", 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.101e-06 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363471+0000", 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7899999999999994e-06 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363477+0000", 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9519999999999999e-06 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.389870+0000", 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026392283999999998 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.393700+0000", 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038300169999999998 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.468266+0000", 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.074566553999999993 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.472389+0000", 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041231289999999997 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.423214+0000", 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95082494900000003 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.426690+0000", 2024-08-17T17:49:56.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003475803 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.433549+0000", 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068586109999999997 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.450814+0000", 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017264901999999999 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.451972+0000", 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001158145 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.466170+0000", 2024-08-17T17:49:56.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014198275 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.484940+0000", 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018770011 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.486814+0000", 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001873792 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.495219+0000", 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084055919999999999 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198915 12.7 12:e781df6c:::benchmark_data_smithi142_261410_object80718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 316.22037837900001, 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.174704889, 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198915 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229674+0000", 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8970000000000001e-06 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229681+0000", 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9389999999999997e-06 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229685+0000", 2024-08-17T17:49:56.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3730000000000001e-06 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.311692+0000", 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082006734999999997 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.315505+0000", 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038131139999999998 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.336219+0000", 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020714249000000001 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.340383+0000", 2024-08-17T17:49:56.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041637410000000003 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.342708+0000", 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0023252469999999 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.346270+0000", 2024-08-17T17:49:56.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035622259999999999 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.353861+0000", 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0075912849999999997 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.391568+0000", 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037706228000000001 2024-08-17T17:49:56.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.392701+0000", 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011330800000000001 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.393099+0000", 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039868500000000002 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.394236+0000", 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011362169999999999 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.396099+0000", 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018637079999999999 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.404375+0000", 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082759449999999998 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198918 12.7 12:ebaa8f49:::benchmark_data_smithi142_261410_object80721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.237002+0000", 2024-08-17T17:49:56.502 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 316.21304636799999, 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1822594, 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198918 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237002+0000", 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237002+0000", 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237001+0000", 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237007+0000", 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4999999999999999e-06 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237020+0000", 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.31e-05 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237026+0000", 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0360000000000003e-06 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.317063+0000", 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.080037013000000004 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.320729+0000", 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036662050000000001 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.355665+0000", 2024-08-17T17:49:56.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.034935739 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.359838+0000", 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041732590000000003 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.373092+0000", 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0132541580000001 2024-08-17T17:49:56.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.376509+0000", 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034164769999999998 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.383323+0000", 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068136610000000004 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.406004+0000", 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022681499000000001 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.406488+0000", 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000484076 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.407757+0000", 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001269318 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.409111+0000", 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001353876 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.410973+0000", 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018619859999999999 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.419262+0000", 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082884780000000002 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198919 12.7 12:e46f8c19:::benchmark_data_smithi142_261410_object80722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.239145+0000", 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 316.21090422700001, 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2519525709999999, 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198919 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239145+0000", 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239145+0000", 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239144+0000", 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239149+0000", 2024-08-17T17:49:56.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8749999999999999e-06 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239158+0000", 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9709999999999993e-06 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239161+0000", 2024-08-17T17:49:56.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7000000000000002e-06 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.360251+0000", 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.121089716 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.364026+0000", 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037748669999999999 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.404095+0000", 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040069283999999997 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.408160+0000", 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040646240000000002 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.436458+0000", 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.028298011 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441031+0000", 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004573321 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.447908+0000", 2024-08-17T17:49:56.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068769850000000004 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.475585+0000", 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027677013 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.476878+0000", 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012927679999999999 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.477259+0000", 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038065700000000001 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.479288+0000", 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020290170000000001 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.481744+0000", 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002456906 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.491097+0000", 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093526819999999993 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.522 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198936 12.7 12:e99b51fa:::benchmark_data_smithi142_261410_object80739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.522164+0000", 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 315.92788514300003, 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1296211789999999, 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198936 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.523 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522164+0000", 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522164+0000", 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522162+0000", 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522170+0000", 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6799999999999993e-06 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522182+0000", 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1952e-05 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522186+0000", 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1219999999999997e-06 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.562515+0000", 2024-08-17T17:49:56.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040329190000000001 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.566615+0000", 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040994330000000004 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.618234+0000", 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.051619273 2024-08-17T17:49:56.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.622379+0000", 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041452449999999997 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.597410+0000", 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97503114400000002 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.600747+0000", 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033365970000000002 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.607430+0000", 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066826740000000004 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.636271+0000", 2024-08-17T17:49:56.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028841201 2024-08-17T17:49:56.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.638583+0000", 2024-08-17T17:49:56.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023119550000000001 2024-08-17T17:49:56.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.639753+0000", 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011705719999999999 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.640922+0000", 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001169128 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.642772+0000", 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018494099999999999 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.651785+0000", 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090129310000000001 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198958 12.7 12:e0f11b00:::benchmark_data_smithi142_261410_object80761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.593679+0000", 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 315.85636994800001, 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1428628599999999, 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.534 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198958 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593679+0000", 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593679+0000", 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593677+0000", 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593685+0000", 2024-08-17T17:49:56.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6229999999999997e-06 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593698+0000", 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3156000000000001e-05 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593706+0000", 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7309999999999993e-06 2024-08-17T17:49:56.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.638168+0000", 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.044462116000000003 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.641965+0000", 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037972180000000002 2024-08-17T17:49:56.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.692915+0000", 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.050950286999999997 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.697078+0000", 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004162603 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.663251+0000", 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96617332499999997 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.667563+0000", 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043112419999999999 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.674289+0000", 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.006725979 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.691308+0000", 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017019863999999999 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.692448+0000", 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011392279999999999 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.693632+0000", 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001184762 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.725305+0000", 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031672968000000003 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.727252+0000", 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019466799999999999 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.736542+0000", 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092896290000000006 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198963 12.7 12:ee7826ac:::benchmark_data_smithi142_261410_object80766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.629744+0000", 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 315.82030497400001, 2024-08-17T17:49:56.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1424937150000001, 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198963 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629744+0000", 2024-08-17T17:49:56.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629744+0000", 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629742+0000", 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629751+0000", 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5709999999999998e-06 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629764+0000", 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3567000000000001e-05 2024-08-17T17:49:56.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629771+0000", 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9720000000000003e-06 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.642536+0000", 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01276499 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.646367+0000", 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038304649999999999 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.716727+0000", 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.070360012 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.720845+0000", 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041184680000000001 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.694028+0000", 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.973182938 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.697431+0000", 2024-08-17T17:49:56.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003402712 2024-08-17T17:49:56.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.704052+0000", 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066207219999999999 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.711653+0000", 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076014120000000001 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.712779+0000", 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001125544 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.713909+0000", 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001129955 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.762028+0000", 2024-08-17T17:49:56.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.048118919000000003 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.763936+0000", 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019082229999999999 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.772238+0000", 2024-08-17T17:49:56.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083017330000000004 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199178 12.7 12:e545f06d:::benchmark_data_smithi142_261410_object80981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:41.441883+0000", 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 315.00816574700002, 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.15849951, 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.555 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199178 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441883+0000", 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441883+0000", 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441881+0000", 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441892+0000", 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1060000000000001e-05 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441910+0000", 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8453000000000001e-05 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441918+0000", 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8010000000000004e-06 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.496560+0000", 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.054642124 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.501148+0000", 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045884860000000001 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.579575+0000", 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.078426483000000005 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.583636+0000", 2024-08-17T17:49:56.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040609299999999999 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.549416+0000", 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96577959199999996 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.553775+0000", 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004359145 2024-08-17T17:49:56.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.560374+0000", 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065997119999999998 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.567655+0000", 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072809179999999999 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.569162+0000", 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015071850000000001 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.571749+0000", 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025864870000000002 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.589264+0000", 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017515037000000001 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.591105+0000", 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001840639 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.600383+0000", 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092779160000000006 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199190 12.7 12:ee8d4a4a:::benchmark_data_smithi142_261410_object80993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:41.522294+0000", 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 314.92775507900001, 2024-08-17T17:49:56.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1390958, 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199190 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522294+0000", 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522294+0000", 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522293+0000", 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522297+0000", 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9470000000000002e-06 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522308+0000", 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0305999999999999e-05 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522312+0000", 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0130000000000004e-06 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.584040+0000", 2024-08-17T17:49:56.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.061727929000000001 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.587799+0000", 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037590089999999998 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.630110+0000", 2024-08-17T17:49:56.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042311720999999997 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.634210+0000", 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040994880000000001 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.623180+0000", 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98897003000000006 2024-08-17T17:49:56.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.626652+0000", 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034722490000000002 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.633375+0000", 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067232910000000002 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.645982+0000", 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012606254000000001 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.647427+0000", 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014450470000000001 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.647809+0000", 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000382526 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.649307+0000", 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014981549999999999 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.651673+0000", 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023651990000000001 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.661389+0000", 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0097168299999999992 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199206 12.7 12:ece9c3ab:::benchmark_data_smithi142_261410_object81009:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:41.750968+0000", 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 314.69908086599997, 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.144830995, 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199206 2024-08-17T17:49:56.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750968+0000", 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750968+0000", 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750967+0000", 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750973+0000", 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4420000000000003e-06 2024-08-17T17:49:56.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750984+0000", 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0604999999999999e-05 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750988+0000", 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.806549+0000", 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.055560905000000001 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.810354+0000", 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038050599999999999 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.854073+0000", 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043719081999999999 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.858133+0000", 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040597200000000002 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.835051+0000", 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97691826800000003 2024-08-17T17:49:56.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.838509+0000", 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034577039999999998 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.845207+0000", 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066979739999999998 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.881443+0000", 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036235410000000003 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.883084+0000", 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016413389999999999 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.884242+0000", 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001158266 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.885750+0000", 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001507988 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.887617+0000", 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018663670000000001 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.895799+0000", 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081824020000000001 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.585 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199449 12.7 12:e5e46031:::benchmark_data_smithi142_261410_object81252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:42.689896+0000", 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 313.76015242699998, 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.176776528, 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.586 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199449 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689896+0000", 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689896+0000", 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689895+0000", 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689902+0000", 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689917+0000", 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4659e-05 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689925+0000", 2024-08-17T17:49:56.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8620000000000003e-06 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.762228+0000", 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.072303386999999997 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.765932+0000", 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037033679999999999 2024-08-17T17:49:56.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.806005+0000", 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040072789999999997 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.809901+0000", 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038965800000000002 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.810713+0000", 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000811713 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.814169+0000", 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034557979999999999 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.821038+0000", 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068688890000000004 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.852300+0000", 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031262184999999998 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.852690+0000", 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039005499999999999 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.854680+0000", 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019902330000000001 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.856368+0000", 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016883079999999999 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.858314+0000", 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00194524 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.866673+0000", 2024-08-17T17:49:56.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083593409999999993 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199845 12.7 12:eb45e44c:::benchmark_data_smithi142_261410_object81648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:44.899208+0000", 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 311.550840666, 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1399064969999999, 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.596 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199845 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899208+0000", 2024-08-17T17:49:56.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899208+0000", 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899206+0000", 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:56.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899216+0000", 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.448e-06 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899231+0000", 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5106000000000001e-05 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899239+0000", 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2579999999999995e-06 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.904216+0000", 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049764359999999999 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908071+0000", 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038552119999999998 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.972517+0000", 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.064445767000000001 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.976669+0000", 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041523150000000002 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.979284+0000", 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0026148180000001 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.983482+0000", 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041981509999999998 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.991310+0000", 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078283929999999995 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.025355+0000", 2024-08-17T17:49:56.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.034044645999999998 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.025732+0000", 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037706399999999998 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.027293+0000", 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001561408 2024-08-17T17:49:56.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.028529+0000", 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001235508 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.030941+0000", 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024122589999999999 2024-08-17T17:49:56.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.039115+0000", 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081734019999999998 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.606 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199849 12.7 12:e0856a61:::benchmark_data_smithi142_261410_object81652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:44.908424+0000", 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 311.54162458799999, 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1458578130000001, 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199849 2024-08-17T17:49:56.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908424+0000", 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908424+0000", 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908422+0000", 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908429+0000", 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2709999999999998e-06 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908442+0000", 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3506000000000001e-05 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908448+0000", 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2329999999999997e-06 2024-08-17T17:49:56.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.977216+0000", 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.068768515000000002 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.981719+0000", 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045033110000000003 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.999592+0000", 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017873138 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.003531+0000", 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039381390000000002 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.007790+0000", 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0042596829999999 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.011070+0000", 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032793549999999999 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.017571+0000", 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065018080000000004 2024-08-17T17:49:56.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.040313+0000", 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022741259999999999 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.041548+0000", 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001235225 2024-08-17T17:49:56.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.041917+0000", 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036895999999999999 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.043112+0000", 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011954470000000001 2024-08-17T17:49:56.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.045174+0000", 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020620980000000001 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.054282+0000", 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091075559999999993 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.616 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:200246 12.7 12:e07dd4ec:::benchmark_data_smithi142_261410_object82049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:46.619725+0000", 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 309.830323894, 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.150011423, 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.617 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200246 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619725+0000", 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619725+0000", 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619723+0000", 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619731+0000", 2024-08-17T17:49:56.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9379999999999992e-06 2024-08-17T17:49:56.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:49:56.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619747+0000", 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5361000000000001e-05 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619754+0000", 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2599999999999999e-06 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.640705+0000", 2024-08-17T17:49:56.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020951588 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.644209+0000", 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035039759999999998 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.674286+0000", 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030076540999999998 2024-08-17T17:49:56.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.678496+0000", 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042095359999999998 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.714365+0000", 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0358691840000001 2024-08-17T17:49:56.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.717503+0000", 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031387659999999999 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.723757+0000", 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062535209999999997 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.739570+0000", 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015812577000000001 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.757147+0000", 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017577808 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.758276+0000", 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011289799999999999 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.759426+0000", 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001150082 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.761284+0000", 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001857612 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.769736+0000", 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084522499999999997 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.629 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:204315 12.7 12:e376318c:::benchmark_data_smithi142_261410_object86118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:45:03.139661+0000", 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 293.310387499, 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.069447558, 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 204315 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.630 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.139661+0000", 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.139661+0000", 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.139660+0000", 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.139665+0000", 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2850000000000002e-06 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.139677+0000", 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1606999999999999e-05 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.139683+0000", 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9630000000000003e-06 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.213894+0000", 2024-08-17T17:49:56.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.074210816999999998 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.217382+0000", 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003488489 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.258961+0000", 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.041578802999999998 2024-08-17T17:49:56.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:03.262681+0000", 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037196820000000002 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.161094+0000", 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89841349500000001 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.164195+0000", 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003101122 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.170030+0000", 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058344759999999999 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.196552+0000", 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026522052000000001 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.196900+0000", 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034794999999999997 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.198815+0000", 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019154650000000001 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.199813+0000", 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00099785299999999993 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.201462+0000", 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016486840000000001 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:04.209109+0000", 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076468949999999999 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:204906 12.7 12:e01465f0:::benchmark_data_smithi142_261410_object86709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:45:05.410010+0000", 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 291.04003869500002, 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.05634615, 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 204906 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.410010+0000", 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.410010+0000", 2024-08-17T17:49:56.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.410009+0000", 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.410017+0000", 2024-08-17T17:49:56.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2269999999999993e-06 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.410033+0000", 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5841e-05 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.410040+0000", 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3189999999999997e-06 2024-08-17T17:49:56.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.459800+0000", 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.049759824000000001 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.464286+0000", 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044859740000000002 2024-08-17T17:49:56.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.486037+0000", 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021750901999999999 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.490136+0000", 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040991780000000002 2024-08-17T17:49:56.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.284339+0000", 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79420296199999996 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.287362+0000", 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00302287 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.293228+0000", 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058665210000000004 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.322584+0000", 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.029355870999999999 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.323551+0000", 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096712199999999995 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.324528+0000", 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00097676300000000007 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.456792+0000", 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13226441799999999 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.458717+0000", 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001924441 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.466356+0000", 2024-08-17T17:49:56.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076394289999999997 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:204920 12.7 12:e735ec79:::benchmark_data_smithi142_261410_object86723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:45:05.458916+0000", 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 290.99113300499999, 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0189634299999999, 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.650 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 204920 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.458916+0000", 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.458916+0000", 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.458914+0000", 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.458920+0000", 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5740000000000003e-06 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.458931+0000", 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0743e-05 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.458935+0000", 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5680000000000001e-06 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.490592+0000", 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031656205999999999 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.494773+0000", 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041815869999999996 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.538447+0000", 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043673377999999999 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:05.542642+0000", 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041956609999999998 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.307856+0000", 2024-08-17T17:49:56.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76521389900000003 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.310678+0000", 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028215229999999998 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.316191+0000", 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055135319999999998 2024-08-17T17:49:56.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.325541+0000", 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093493810000000004 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.326585+0000", 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001044938 2024-08-17T17:49:56.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.327550+0000", 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096414799999999996 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.468089+0000", 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14053964299999999 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.470161+0000", 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020716979999999999 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.477879+0000", 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077182509999999998 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.660 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:205168 12.7 12:eaf9f994:::benchmark_data_smithi142_261410_object86971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:45:06.097775+0000", 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 290.35227376900002, 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.045999583, 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:49:56.661 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 205168 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.097775+0000", 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.097775+0000", 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.097774+0000", 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:49:56.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.097781+0000", 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1400000000000002e-06 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.097793+0000", 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2435e-05 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.097798+0000", 2024-08-17T17:49:56.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0379999999999999e-06 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.161580+0000", 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.063781563999999999 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.164728+0000", 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031483570000000001 2024-08-17T17:49:56.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.200477+0000", 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.035749048999999998 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:06.203940+0000", 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034627859999999998 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.095826+0000", 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89188630800000002 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.099139+0000", 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033127539999999998 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.104940+0000", 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058005319999999997 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.127884+0000", 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022944573999999999 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.130443+0000", 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025589219999999999 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.132806+0000", 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023628880000000001 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.133235+0000", 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00042879700000000002 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.135354+0000", 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021192569999999998 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:49:56.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:45:07.143775+0000", 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084205849999999995 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:49:56.671 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:49:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.673 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:56.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:56.674 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:56 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1439476065' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:56.674 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:56.674 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:56.674 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:56 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1439476065' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:56.674 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:56.675 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:56.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:56.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:56.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:56 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1439476065' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:56.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:49:56.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:56.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:56.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:57.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:57.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:57.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:57.223 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:57.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:57.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:57.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:57.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:57.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:49:57.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:57.602 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-08-17T17:49:57.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:57.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:57.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:57.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:57.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:57 smithi152 ceph-mon[93456]: pgmap v1759: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:49:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2721631405' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:49:57.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:57 smithi142 ceph-mon[106729]: pgmap v1759: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:49:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2721631405' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:49:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:57 smithi142 ceph-mon[103614]: pgmap v1759: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:49:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2721631405' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:49:57.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.897 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:57.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:57.982 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:57.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:58.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:58.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:58.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:58.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:58.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:58.275 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:58.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:58.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:49:58.464 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:49:58.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:58.547 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:58.649 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:58.676 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:58.777 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:58.806 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2721631405' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:49:58.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:58 smithi152 ceph-mon[93456]: osdmap e439: 8 total, 7 up, 6 in 2024-08-17T17:49:58.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:49:58 smithi152 ceph-mon[93456]: pgmap v1761: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 922 op/s 2024-08-17T17:49:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2721631405' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:49:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:58 smithi142 ceph-mon[106729]: osdmap e439: 8 total, 7 up, 6 in 2024-08-17T17:49:58.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:49:58 smithi142 ceph-mon[106729]: pgmap v1761: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 922 op/s 2024-08-17T17:49:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2721631405' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:49:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:58 smithi142 ceph-mon[103614]: osdmap e439: 8 total, 7 up, 6 in 2024-08-17T17:49:58.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:49:58 smithi142 ceph-mon[103614]: pgmap v1761: 89 pgs: 89 active+clean; 5.4 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 922 op/s 2024-08-17T17:49:58.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:58.938 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:59.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:59.061 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:59.074 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:59.176 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.218 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:59.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:59.319 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:59.349 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:59.450 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:59.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:59.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:59.579 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:59.606 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:59.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:59.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:59.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:49:59.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:59.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:59.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:49:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:59.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:00.022 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:170496 13.f 13:f4fa85a8:::benchmark_data_smithi142_393845_object42680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.003560+0000", 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.026490551000000001, 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026524171999999999, 2024-08-17T17:50:00.032 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 170496 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.003560+0000", 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.003560+0000", 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.003559+0000", 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.003567+0000", 2024-08-17T17:50:00.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1300000000000001e-06 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.003580+0000", 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3696000000000001e-05 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.003585+0000", 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8779999999999997e-06 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.011700+0000", 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081144149999999998 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.011822+0000", 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121975 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.011949+0000", 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012718699999999999 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.023457+0000", 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011508096000000001 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029611+0000", 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061538260000000003 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:170502 13.7 13:e20798e3:::benchmark_data_smithi142_393845_object42686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.004759+0000", 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.025291757000000002, 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025414544000000001, 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:00.038 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 170502 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.004759+0000", 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.004759+0000", 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.004758+0000", 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.004766+0000", 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9540000000000004e-06 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.004775+0000", 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7309999999999999e-06 2024-08-17T17:50:00.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.004780+0000", 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4469999999999997e-06 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.016048+0000", 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011267642 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.016302+0000", 2024-08-17T17:50:00.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025450000000000001 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.016747+0000", 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00044476900000000002 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.023487+0000", 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067397300000000002 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.042 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:170507 13.9 e439/413 13:98287891:::benchmark_data_smithi142_393845_object42691:head v 439'10531, mlcod=439'10529)", 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.021262+0000", 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0087885240000000007, 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089846270000000002, 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021262+0000", 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021262+0000", 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021261+0000", 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:00.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021271+0000", 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0353999999999999e-05 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021291+0000", 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9366000000000001e-05 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021297+0000", 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5089999999999999e-06 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021433+0000", 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000135642 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021450+0000", 2024-08-17T17:50:00.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7385e-05 2024-08-17T17:50:00.047 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.047 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.047 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.047 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:170508 13.8 e439/413 13:145f6616:::benchmark_data_smithi142_393845_object42692:head v 439'10843, mlcod=439'10841)", 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.021313+0000", 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0087381420000000008, 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089928869999999998, 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021313+0000", 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021313+0000", 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021312+0000", 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021315+0000", 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1350000000000001e-06 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021321+0000", 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3489999999999999e-06 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021324+0000", 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1860000000000001e-06 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021368+0000", 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4115e-05 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.021389+0000", 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0726000000000001e-05 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:170509 13.8 e439/413 13:14f5eb9d:::benchmark_data_smithi142_393845_object42693:head v 439'10844, mlcod=439'10841)", 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.026838+0000", 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0032130959999999999, 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035196770000000001, 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026838+0000", 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026838+0000", 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026836+0000", 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026845+0000", 2024-08-17T17:50:00.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4870000000000002e-06 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026869+0000", 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3623000000000001e-05 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026876+0000", 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0160000000000003e-06 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026977+0000", 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010191000000000001 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.026998+0000", 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0112999999999999e-05 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.055 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:170513 13.4 13:22a3c6bd:::benchmark_data_smithi142_393845_object42697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.027592+0000", 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0024591069999999999, 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028414360000000001, 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 170513 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027592+0000", 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027592+0000", 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027590+0000", 2024-08-17T17:50:00.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027598+0000", 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7100000000000007e-06 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027611+0000", 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2988e-05 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027616+0000", 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6049999999999997e-06 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027662+0000", 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5735999999999997e-05 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027772+0000", 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109867 2024-08-17T17:50:00.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.027893+0000", 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012152800000000001 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.48005.0:170496 13.f e439/413)", 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:00.029762+0000", 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00028880300000000002, 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00073243799999999997, 2024-08-17T17:50:00.060 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029762+0000", 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029762+0000", 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029762+0000", 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029763+0000", 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.542e-06 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029768+0000", 2024-08-17T17:50:00.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5979999999999999e-06 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.029771+0000", 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7549999999999999e-06 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.030487+0000", 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000715914 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:00.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:00.030491+0000", 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3440000000000003e-06 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 7 2024-08-17T17:50:00.064 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:50:00.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:00.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:00.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:00.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:00.291 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:00.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:00 smithi152 ceph-mon[93456]: overall HEALTH_OK 2024-08-17T17:50:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:50:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:50:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:00.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:00 smithi142 ceph-mon[106729]: overall HEALTH_OK 2024-08-17T17:50:00.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:00 smithi142 ceph-mon[103614]: overall HEALTH_OK 2024-08-17T17:50:00.394 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:00.433 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:00.466 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:50:00.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:00.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:00.665 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:00.694 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:00.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:00.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:00.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:00.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:00.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:01.058 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:01.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:01.118 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:01.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:50:01.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:01.227 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:01.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:01 smithi152 ceph-mon[93456]: pgmap v1762: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:50:01.329 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:01.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:01.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:01 smithi142 ceph-mon[106729]: pgmap v1762: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:50:01.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:01 smithi142 ceph-mon[103614]: pgmap v1762: 89 pgs: 89 active+clean; 5.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:50:01.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:01.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:01.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:01.639 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.671 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:01.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:01.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:01.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:01.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:01.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:01.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:02.056 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:50:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:50:01] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.51.0" 2024-08-17T17:50:02.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:02.081 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:50:02.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:02.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:02.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:02.319 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1597182418' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:02.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:02.355 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:02.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:02.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1597182418' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:02.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:02.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1597182418' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:02.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:02.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:02.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:02.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:02.619 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:02.721 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:02.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:02.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:02.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:02.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:02.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:03.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:03.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:03.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[106729]: osdmap e440: 8 total, 7 up, 6 in 2024-08-17T17:50:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[106729]: pgmap v1764: 89 pgs: 89 active+clean; 5.2 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 871 op/s 2024-08-17T17:50:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[103614]: osdmap e440: 8 total, 7 up, 6 in 2024-08-17T17:50:03.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:03 smithi142 ceph-mon[103614]: pgmap v1764: 89 pgs: 89 active+clean; 5.2 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 871 op/s 2024-08-17T17:50:03.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:03.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:03.247 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:03.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:03 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:03.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:03 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:03.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:03 smithi152 ceph-mon[93456]: osdmap e440: 8 total, 7 up, 6 in 2024-08-17T17:50:03.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:03 smithi152 ceph-mon[93456]: pgmap v1764: 89 pgs: 89 active+clean; 5.2 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 871 op/s 2024-08-17T17:50:03.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:03.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:03.321 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:50:03.321 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:03.346 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:50:03.346 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:03.370 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:50:03.370 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:03.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:03.395 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:50:03.396 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:03.401 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:03.419 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:50:03.420 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:03.444 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:50:03.445 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:03.470 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:50:03.470 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:03.471 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:50:03.504 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:03.531 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:03.633 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:03.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:03.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:03.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:03.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:03.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:03.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:03.990 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:50:03.990 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:50:03.990 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:50:03.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:50:04.028 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:04.029 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:04.029 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:04.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:04.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.212 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:04.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:04.313 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:04.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1515569445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:04.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:04.356 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1515569445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:04.356 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1515569445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:04.442 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:04.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:04.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:04.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:04.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:04.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:50:04.729 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:50:04.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:50:04.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:04.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:04.888 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:04.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:04.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:05.091 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:05.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:05.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:05.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:05.229 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:05.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:05.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:05.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:05.353 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:05 smithi142 ceph-mon[106729]: pgmap v1765: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-17T17:50:05.353 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:05 smithi142 ceph-mon[103614]: pgmap v1765: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-17T17:50:05.381 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:05 smithi152 ceph-mon[93456]: pgmap v1765: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-17T17:50:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:05.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:05.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:05.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:05.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:05.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:05.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:05.785 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:05.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:05.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.918 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:05.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:06.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:50:06.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:06.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:06.184 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:06.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:06.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:06.347 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:06.449 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:06.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:06.578 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:06.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:06.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:06.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:50:06.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:06.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:06.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:06.873 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:06.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:06.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:07.034 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:07.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:07.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:07.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:07.220 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:50:07.225 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:50:07.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:07.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.304 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:07.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:07.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:07.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:07.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:07.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:07.655 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:07 smithi152 ceph-mon[93456]: pgmap v1766: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 754 op/s 2024-08-17T17:50:07.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:07.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:07.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:07.840 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:07 smithi142 ceph-mon[106729]: pgmap v1766: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 754 op/s 2024-08-17T17:50:07.840 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:07 smithi142 ceph-mon[103614]: pgmap v1766: 89 pgs: 89 active+clean; 5.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 754 op/s 2024-08-17T17:50:07.855 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:07.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:07.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:08.085 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:08.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:08.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:08.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:08.239 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:08.341 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:08.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:08.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:08.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:08.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:08.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:08.636 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:08.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:08.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:08.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:08.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:08 smithi152 ceph-mon[93456]: from='client.48170 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:50:08.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:08 smithi142 ceph-mon[106729]: from='client.48170 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:50:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:08 smithi142 ceph-mon[103614]: from='client.48170 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:50:08.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:09.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:09.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:09.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:09.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:09.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:09.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:09.308 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1766,"stamp":"2024-08-17T17:50:06.499018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5382871013,"num_objects":82154,"num_object_clones":0,"num_object_copies":246462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82154,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":173778,"num_write_kb":8182480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2958510,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108459,"ondisk_log_size":108459,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":23363740,"kb_used_data":15893160,"kb_used_omap":27,"kb_used_meta":7470372,"kb_avail":539107172,"statfs":{"total":575970213888,"available":552045744128,"internally_reserved":0,"allocated":16274595840,"data_stored":16260161481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27928,"internal_metadata":7649661672},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,6,5,6,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-442171392,"num_objects":-6747,"num_object_clones":0,"num_object_copies":-20241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6747,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.945155"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":523,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.583960+0000","last_change":"2024-08-17T17:49:32.579092+0000","last_active":"2024-08-17T17:49:57.583960+0000","last_peered":"2024-08-17T17:49:57.583960+0000","last_clean":"2024-08-17T17:49:57.583960+0000","last_became_active":"2024-08-17T17:49:32.578298+0000","last_became_peered":"2024-08-17T17:49:32.578298+0000","last_unstale":"2024-08-17T17:49:57.583960+0000","last_undegraded":"2024-08-17T17:49:57.583960+0000","last_fullsized":"2024-08-17T17:49:57.583960+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:42.168553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":149,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571094+0000","last_change":"2024-08-17T17:33:40.653046+0000","last_active":"2024-08-17T17:49:57.571094+0000","last_peered":"2024-08-17T17:49:57.571094+0000","last_clean":"2024-08-17T17:49:57.571094+0000","last_became_active":"2024-08-17T17:33:40.644432+0000","last_became_peered":"2024-08-17T17:33:40.644432+0000","last_unstale":"2024-08-17T17:49:57.571094+0000","last_undegraded":"2024-08-17T17:49:57.571094+0000","last_fullsized":"2024-08-17T17:49:57.571094+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:21.044279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":343,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578897+0000","last_change":"2024-08-17T17:33:40.645564+0000","last_active":"2024-08-17T17:49:57.578897+0000","last_peered":"2024-08-17T17:49:57.578897+0000","last_clean":"2024-08-17T17:49:57.578897+0000","last_became_active":"2024-08-17T17:33:40.644871+0000","last_became_peered":"2024-08-17T17:33:40.644871+0000","last_unstale":"2024-08-17T17:49:57.578897+0000","last_undegraded":"2024-08-17T17:49:57.578897+0000","last_fullsized":"2024-08-17T17:49:57.578897+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:25.000754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":227,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.083523+0000","last_change":"2024-08-17T17:49:32.579854+0000","last_active":"2024-08-17T17:50:02.083523+0000","last_peered":"2024-08-17T17:50:02.083523+0000","last_clean":"2024-08-17T17:50:02.083523+0000","last_became_active":"2024-08-17T17:49:32.579543+0000","last_became_peered":"2024-08-17T17:49:32.579543+0000","last_unstale":"2024-08-17T17:50:02.083523+0000","last_undegraded":"2024-08-17T17:50:02.083523+0000","last_fullsized":"2024-08-17T17:50:02.083523+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:14:26.847540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":601,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079074+0000","last_change":"2024-08-17T17:33:40.646911+0000","last_active":"2024-08-17T17:50:02.079074+0000","last_peered":"2024-08-17T17:50:02.079074+0000","last_clean":"2024-08-17T17:50:02.079074+0000","last_became_active":"2024-08-17T17:33:40.646421+0000","last_became_peered":"2024-08-17T17:33:40.646421+0000","last_unstale":"2024-08-17T17:50:02.079074+0000","last_undegraded":"2024-08-17T17:50:02.079074+0000","last_fullsized":"2024-08-17T17:50:02.079074+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:12.666012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":292,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578021+0000","last_change":"2024-08-17T17:33:40.640533+0000","last_active":"2024-08-17T17:49:57.578021+0000","last_peered":"2024-08-17T17:49:57.578021+0000","last_clean":"2024-08-17T17:49:57.578021+0000","last_became_active":"2024-08-17T17:33:40.640256+0000","last_became_peered":"2024-08-17T17:33:40.640256+0000","last_unstale":"2024-08-17T17:49:57.578021+0000","last_undegraded":"2024-08-17T17:49:57.578021+0000","last_fullsized":"2024-08-17T17:49:57.578021+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:05.718718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":695,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079453+0000","last_change":"2024-08-17T17:33:40.653157+0000","last_active":"2024-08-17T17:50:02.079453+0000","last_peered":"2024-08-17T17:50:02.079453+0000","last_clean":"2024-08-17T17:50:02.079453+0000","last_became_active":"2024-08-17T17:33:40.652786+0000","last_became_peered":"2024-08-17T17:33:40.652786+0000","last_unstale":"2024-08-17T17:50:02.079453+0000","last_undegraded":"2024-08-17T17:50:02.079453+0000","last_fullsized":"2024-08-17T17:50:02.079453+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:23.696438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":460,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.580857+0000","last_change":"2024-08-17T17:24:37.022679+0000","last_active":"2024-08-17T17:49:57.580857+0000","last_peered":"2024-08-17T17:49:57.580857+0000","last_clean":"2024-08-17T17:49:57.580857+0000","last_became_active":"2024-08-17T17:24:36.968355+0000","last_became_peered":"2024-08-17T17:24:36.968355+0000","last_unstale":"2024-08-17T17:49:57.580857+0000","last_undegraded":"2024-08-17T17:49:57.580857+0000","last_fullsized":"2024-08-17T17:49:57.580857+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:19.846739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"440'10722","reported_seq":10757,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.477452+0000","last_change":"2024-08-17T17:47:33.474484+0000","last_active":"2024-08-17T17:50:02.477452+0000","last_peered":"2024-08-17T17:50:02.477452+0000","last_clean":"2024-08-17T17:50:02.477452+0000","last_became_active":"2024-08-17T17:47:33.474193+0000","last_became_peered":"2024-08-17T17:47:33.474193+0000","last_unstale":"2024-08-17T17:50:02.477452+0000","last_undegraded":"2024-08-17T17:50:02.477452+0000","last_fullsized":"2024-08-17T17:50:02.477452+0000","mapping_epoch":413,"log_start":"421'3700","ondisk_log_start":"421'3700","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:23.071916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336723968,"num_objects":5138,"num_object_clones":0,"num_object_copies":15414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10722,"num_write_kb":507520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":145,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.568578+0000","last_change":"2024-08-17T17:33:40.657593+0000","last_active":"2024-08-17T17:49:57.568578+0000","last_peered":"2024-08-17T17:49:57.568578+0000","last_clean":"2024-08-17T17:49:57.568578+0000","last_became_active":"2024-08-17T17:33:40.648756+0000","last_became_peered":"2024-08-17T17:33:40.648756+0000","last_unstale":"2024-08-17T17:49:57.568578+0000","last_undegraded":"2024-08-17T17:49:57.568578+0000","last_fullsized":"2024-08-17T17:49:57.568578+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:43.348576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":326,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.576122+0000","last_change":"2024-08-17T17:33:40.638442+0000","last_active":"2024-08-17T17:49:57.576122+0000","last_peered":"2024-08-17T17:49:57.576122+0000","last_clean":"2024-08-17T17:49:57.576122+0000","last_became_active":"2024-08-17T17:33:40.638149+0000","last_became_peered":"2024-08-17T17:33:40.638149+0000","last_unstale":"2024-08-17T17:49:57.576122+0000","last_undegraded":"2024-08-17T17:49:57.576122+0000","last_fullsized":"2024-08-17T17:49:57.576122+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:11.674887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"440'10786","reported_seq":10852,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:04.194929+0000","last_change":"2024-08-17T17:47:33.469969+0000","last_active":"2024-08-17T17:50:04.194929+0000","last_peered":"2024-08-17T17:50:04.194929+0000","last_clean":"2024-08-17T17:50:04.194929+0000","last_became_active":"2024-08-17T17:47:33.469675+0000","last_became_peered":"2024-08-17T17:47:33.469675+0000","last_unstale":"2024-08-17T17:50:04.194929+0000","last_undegraded":"2024-08-17T17:50:04.194929+0000","last_fullsized":"2024-08-17T17:50:04.194929+0000","mapping_epoch":413,"log_start":"425'4900","ondisk_log_start":"425'4900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:24.678241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332660736,"num_objects":5076,"num_object_clones":0,"num_object_copies":15228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10786,"num_write_kb":507584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":147,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.568583+0000","last_change":"2024-08-17T17:33:40.661373+0000","last_active":"2024-08-17T17:49:57.568583+0000","last_peered":"2024-08-17T17:49:57.568583+0000","last_clean":"2024-08-17T17:49:57.568583+0000","last_became_active":"2024-08-17T17:33:40.651427+0000","last_became_peered":"2024-08-17T17:33:40.651427+0000","last_unstale":"2024-08-17T17:49:57.568583+0000","last_undegraded":"2024-08-17T17:49:57.568583+0000","last_fullsized":"2024-08-17T17:49:57.568583+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:04.106535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":411,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.577938+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:49:57.577938+0000","last_peered":"2024-08-17T17:49:57.577938+0000","last_clean":"2024-08-17T17:49:57.577938+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:49:57.577938+0000","last_undegraded":"2024-08-17T17:49:57.577938+0000","last_fullsized":"2024-08-17T17:49:57.577938+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"440'10927","reported_seq":10993,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:05.080556+0000","last_change":"2024-08-17T17:47:33.474669+0000","last_active":"2024-08-17T17:50:05.080556+0000","last_peered":"2024-08-17T17:50:05.080556+0000","last_clean":"2024-08-17T17:50:05.080556+0000","last_became_active":"2024-08-17T17:47:33.474373+0000","last_became_peered":"2024-08-17T17:47:33.474373+0000","last_unstale":"2024-08-17T17:50:05.080556+0000","last_undegraded":"2024-08-17T17:50:05.080556+0000","last_fullsized":"2024-08-17T17:50:05.080556+0000","mapping_epoch":413,"log_start":"421'3700","ondisk_log_start":"421'3700","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7227,"log_dups_size":0,"ondisk_log_size":7227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:34:23.147112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327483392,"num_objects":4997,"num_object_clones":0,"num_object_copies":14991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10927,"num_write_kb":509568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":143,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578888+0000","last_change":"2024-08-17T17:33:40.671482+0000","last_active":"2024-08-17T17:49:57.578888+0000","last_peered":"2024-08-17T17:49:57.578888+0000","last_clean":"2024-08-17T17:49:57.578888+0000","last_became_active":"2024-08-17T17:33:40.645269+0000","last_became_peered":"2024-08-17T17:33:40.645269+0000","last_unstale":"2024-08-17T17:49:57.578888+0000","last_undegraded":"2024-08-17T17:49:57.578888+0000","last_fullsized":"2024-08-17T17:49:57.578888+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:02.083569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":421,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.570703+0000","last_change":"2024-08-17T17:33:40.655239+0000","last_active":"2024-08-17T17:49:57.570703+0000","last_peered":"2024-08-17T17:49:57.570703+0000","last_clean":"2024-08-17T17:49:57.570703+0000","last_became_active":"2024-08-17T17:33:40.654725+0000","last_became_peered":"2024-08-17T17:33:40.654725+0000","last_unstale":"2024-08-17T17:49:57.570703+0000","last_undegraded":"2024-08-17T17:49:57.570703+0000","last_fullsized":"2024-08-17T17:49:57.570703+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:34.172184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"440'10981","reported_seq":11047,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:03.011159+0000","last_change":"2024-08-17T17:47:33.471249+0000","last_active":"2024-08-17T17:50:03.011159+0000","last_peered":"2024-08-17T17:50:03.011159+0000","last_clean":"2024-08-17T17:50:03.011159+0000","last_became_active":"2024-08-17T17:47:33.470968+0000","last_became_peered":"2024-08-17T17:47:33.470968+0000","last_unstale":"2024-08-17T17:50:03.011159+0000","last_undegraded":"2024-08-17T17:50:03.011159+0000","last_fullsized":"2024-08-17T17:50:03.011159+0000","mapping_epoch":413,"log_start":"424'4100","ondisk_log_start":"424'4100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6881,"log_dups_size":0,"ondisk_log_size":6881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:45.656151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341245952,"num_objects":5207,"num_object_clones":0,"num_object_copies":15621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10981,"num_write_kb":518016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":639,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079096+0000","last_change":"2024-08-17T17:21:31.870130+0000","last_active":"2024-08-17T17:50:02.079096+0000","last_peered":"2024-08-17T17:50:02.079096+0000","last_clean":"2024-08-17T17:50:02.079096+0000","last_became_active":"2024-08-17T17:21:31.869788+0000","last_became_peered":"2024-08-17T17:21:31.869788+0000","last_unstale":"2024-08-17T17:50:02.079096+0000","last_undegraded":"2024-08-17T17:50:02.079096+0000","last_fullsized":"2024-08-17T17:50:02.079096+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:06.800656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":405,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.083918+0000","last_change":"2024-08-17T17:24:37.039575+0000","last_active":"2024-08-17T17:50:02.083918+0000","last_peered":"2024-08-17T17:50:02.083918+0000","last_clean":"2024-08-17T17:50:02.083918+0000","last_became_active":"2024-08-17T17:24:37.039208+0000","last_became_peered":"2024-08-17T17:24:37.039208+0000","last_unstale":"2024-08-17T17:50:02.083918+0000","last_undegraded":"2024-08-17T17:50:02.083918+0000","last_fullsized":"2024-08-17T17:50:02.083918+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:46.835913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"440'10741","reported_seq":10776,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.211302+0000","last_change":"2024-08-17T17:47:33.472177+0000","last_active":"2024-08-17T17:50:02.211302+0000","last_peered":"2024-08-17T17:50:02.211302+0000","last_clean":"2024-08-17T17:50:02.211302+0000","last_became_active":"2024-08-17T17:47:33.471903+0000","last_became_peered":"2024-08-17T17:47:33.471903+0000","last_unstale":"2024-08-17T17:50:02.211302+0000","last_undegraded":"2024-08-17T17:50:02.211302+0000","last_fullsized":"2024-08-17T17:50:02.211302+0000","mapping_epoch":413,"log_start":"424'4400","ondisk_log_start":"424'4400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:39:03.482291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345571328,"num_objects":5273,"num_object_clones":0,"num_object_copies":15819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10741,"num_write_kb":512448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":30,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571007+0000","last_change":"2024-08-17T17:49:32.596457+0000","last_active":"2024-08-17T17:49:57.571007+0000","last_peered":"2024-08-17T17:49:57.571007+0000","last_clean":"2024-08-17T17:49:57.571007+0000","last_became_active":"2024-08-17T17:49:32.579278+0000","last_became_peered":"2024-08-17T17:49:32.579278+0000","last_unstale":"2024-08-17T17:49:57.571007+0000","last_undegraded":"2024-08-17T17:49:57.571007+0000","last_fullsized":"2024-08-17T17:49:57.571007+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:24.903271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":394,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.574358+0000","last_change":"2024-08-17T17:12:40.335572+0000","last_active":"2024-08-17T17:49:57.574358+0000","last_peered":"2024-08-17T17:49:57.574358+0000","last_clean":"2024-08-17T17:49:57.574358+0000","last_became_active":"2024-08-17T17:12:40.334564+0000","last_became_peered":"2024-08-17T17:12:40.334564+0000","last_unstale":"2024-08-17T17:49:57.574358+0000","last_undegraded":"2024-08-17T17:49:57.574358+0000","last_fullsized":"2024-08-17T17:49:57.574358+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:48.678926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"440'10696","reported_seq":10731,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.429386+0000","last_change":"2024-08-17T17:47:33.472785+0000","last_active":"2024-08-17T17:50:02.429386+0000","last_peered":"2024-08-17T17:50:02.429386+0000","last_clean":"2024-08-17T17:50:02.429386+0000","last_became_active":"2024-08-17T17:47:33.472497+0000","last_became_peered":"2024-08-17T17:47:33.472497+0000","last_unstale":"2024-08-17T17:50:02.429386+0000","last_undegraded":"2024-08-17T17:50:02.429386+0000","last_fullsized":"2024-08-17T17:50:02.429386+0000","mapping_epoch":413,"log_start":"421'3700","ondisk_log_start":"421'3700","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:39:33.059184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10696,"num_write_kb":510848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":395,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.570846+0000","last_change":"2024-08-17T17:07:51.482742+0000","last_active":"2024-08-17T17:49:57.570846+0000","last_peered":"2024-08-17T17:49:57.570846+0000","last_clean":"2024-08-17T17:49:57.570846+0000","last_became_active":"2024-08-17T17:07:51.482298+0000","last_became_peered":"2024-08-17T17:07:51.482298+0000","last_unstale":"2024-08-17T17:49:57.570846+0000","last_undegraded":"2024-08-17T17:49:57.570846+0000","last_fullsized":"2024-08-17T17:49:57.570846+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:57.818516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":421,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.569189+0000","last_change":"2024-08-17T17:33:20.283627+0000","last_active":"2024-08-17T17:49:57.569189+0000","last_peered":"2024-08-17T17:49:57.569189+0000","last_clean":"2024-08-17T17:49:57.569189+0000","last_became_active":"2024-08-17T17:33:20.283344+0000","last_became_peered":"2024-08-17T17:33:20.283344+0000","last_unstale":"2024-08-17T17:49:57.569189+0000","last_undegraded":"2024-08-17T17:49:57.569189+0000","last_fullsized":"2024-08-17T17:49:57.569189+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:39.567974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"440'10907","reported_seq":10942,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.466696+0000","last_change":"2024-08-17T17:47:33.473418+0000","last_active":"2024-08-17T17:50:02.466696+0000","last_peered":"2024-08-17T17:50:02.466696+0000","last_clean":"2024-08-17T17:50:02.466696+0000","last_became_active":"2024-08-17T17:47:33.473144+0000","last_became_peered":"2024-08-17T17:47:33.473144+0000","last_unstale":"2024-08-17T17:50:02.466696+0000","last_undegraded":"2024-08-17T17:50:02.466696+0000","last_fullsized":"2024-08-17T17:50:02.466696+0000","mapping_epoch":413,"log_start":"421'3900","ondisk_log_start":"421'3900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:19:35.223379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343080960,"num_objects":5235,"num_object_clones":0,"num_object_copies":15705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10907,"num_write_kb":516544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":432,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578106+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:49:57.578106+0000","last_peered":"2024-08-17T17:49:57.578106+0000","last_clean":"2024-08-17T17:49:57.578106+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:49:57.578106+0000","last_undegraded":"2024-08-17T17:49:57.578106+0000","last_fullsized":"2024-08-17T17:49:57.578106+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":394,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.568701+0000","last_change":"2024-08-17T17:07:51.482544+0000","last_active":"2024-08-17T17:49:57.568701+0000","last_peered":"2024-08-17T17:49:57.568701+0000","last_clean":"2024-08-17T17:49:57.568701+0000","last_became_active":"2024-08-17T17:07:51.482114+0000","last_became_peered":"2024-08-17T17:07:51.482114+0000","last_unstale":"2024-08-17T17:49:57.568701+0000","last_undegraded":"2024-08-17T17:49:57.568701+0000","last_fullsized":"2024-08-17T17:49:57.568701+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:46:56.175749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"440'10923","reported_seq":10958,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.273031+0000","last_change":"2024-08-17T17:47:33.473396+0000","last_active":"2024-08-17T17:50:02.273031+0000","last_peered":"2024-08-17T17:50:02.273031+0000","last_clean":"2024-08-17T17:50:02.273031+0000","last_became_active":"2024-08-17T17:47:33.472959+0000","last_became_peered":"2024-08-17T17:47:33.472959+0000","last_unstale":"2024-08-17T17:50:02.273031+0000","last_undegraded":"2024-08-17T17:50:02.273031+0000","last_fullsized":"2024-08-17T17:50:02.273031+0000","mapping_epoch":413,"log_start":"425'4600","ondisk_log_start":"425'4600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6323,"log_dups_size":0,"ondisk_log_size":6323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:26.187848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341639168,"num_objects":5213,"num_object_clones":0,"num_object_copies":15639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10923,"num_write_kb":516352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":698,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.078890+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:50:02.078890+0000","last_peered":"2024-08-17T17:50:02.078890+0000","last_clean":"2024-08-17T17:50:02.078890+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:50:02.078890+0000","last_undegraded":"2024-08-17T17:50:02.078890+0000","last_fullsized":"2024-08-17T17:50:02.078890+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":152,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.568478+0000","last_change":"2024-08-17T17:49:32.580175+0000","last_active":"2024-08-17T17:49:57.568478+0000","last_peered":"2024-08-17T17:49:57.568478+0000","last_clean":"2024-08-17T17:49:57.568478+0000","last_became_active":"2024-08-17T17:49:32.579415+0000","last_became_peered":"2024-08-17T17:49:32.579415+0000","last_unstale":"2024-08-17T17:49:57.568478+0000","last_undegraded":"2024-08-17T17:49:57.568478+0000","last_fullsized":"2024-08-17T17:49:57.568478+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:49:08.716633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.c","version":"440'10999","reported_seq":11064,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:04.170267+0000","last_change":"2024-08-17T17:47:33.468618+0000","last_active":"2024-08-17T17:50:04.170267+0000","last_peered":"2024-08-17T17:50:04.170267+0000","last_clean":"2024-08-17T17:50:04.170267+0000","last_became_active":"2024-08-17T17:47:33.468320+0000","last_became_peered":"2024-08-17T17:47:33.468320+0000","last_unstale":"2024-08-17T17:50:04.170267+0000","last_undegraded":"2024-08-17T17:50:04.170267+0000","last_fullsized":"2024-08-17T17:50:04.170267+0000","mapping_epoch":413,"log_start":"425'5100","ondisk_log_start":"425'5100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:12.338465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334430208,"num_objects":5103,"num_object_clones":0,"num_object_copies":15309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10999,"num_write_kb":515264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":842,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.085388+0000","last_change":"2024-08-17T17:47:14.531821+0000","last_active":"2024-08-17T17:50:02.085388+0000","last_peered":"2024-08-17T17:50:02.085388+0000","last_clean":"2024-08-17T17:50:02.085388+0000","last_became_active":"2024-08-17T17:47:14.531354+0000","last_became_peered":"2024-08-17T17:47:14.531354+0000","last_unstale":"2024-08-17T17:50:02.085388+0000","last_undegraded":"2024-08-17T17:50:02.085388+0000","last_fullsized":"2024-08-17T17:50:02.085388+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:54.544598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":375,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578702+0000","last_change":"2024-08-17T17:46:26.863739+0000","last_active":"2024-08-17T17:49:57.578702+0000","last_peered":"2024-08-17T17:49:57.578702+0000","last_clean":"2024-08-17T17:49:57.578702+0000","last_became_active":"2024-08-17T17:33:40.659933+0000","last_became_peered":"2024-08-17T17:33:40.659933+0000","last_unstale":"2024-08-17T17:49:57.578702+0000","last_undegraded":"2024-08-17T17:49:57.578702+0000","last_fullsized":"2024-08-17T17:49:57.578702+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:37.526602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":194,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.579716+0000","last_change":"2024-08-17T17:49:32.578475+0000","last_active":"2024-08-17T17:49:57.579716+0000","last_peered":"2024-08-17T17:49:57.579716+0000","last_clean":"2024-08-17T17:49:57.579716+0000","last_became_active":"2024-08-17T17:49:32.578140+0000","last_became_peered":"2024-08-17T17:49:32.578140+0000","last_unstale":"2024-08-17T17:49:57.579716+0000","last_undegraded":"2024-08-17T17:49:57.579716+0000","last_fullsized":"2024-08-17T17:49:57.579716+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:42.574161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":404,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571412+0000","last_change":"2024-08-17T17:33:40.641719+0000","last_active":"2024-08-17T17:49:57.571412+0000","last_peered":"2024-08-17T17:49:57.571412+0000","last_clean":"2024-08-17T17:49:57.571412+0000","last_became_active":"2024-08-17T17:33:40.641473+0000","last_became_peered":"2024-08-17T17:33:40.641473+0000","last_unstale":"2024-08-17T17:49:57.571412+0000","last_undegraded":"2024-08-17T17:49:57.571412+0000","last_fullsized":"2024-08-17T17:49:57.571412+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:05.931871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"440'10806","reported_seq":10841,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.487366+0000","last_change":"2024-08-17T17:47:33.474825+0000","last_active":"2024-08-17T17:50:02.487366+0000","last_peered":"2024-08-17T17:50:02.487366+0000","last_clean":"2024-08-17T17:50:02.487366+0000","last_became_active":"2024-08-17T17:47:33.474247+0000","last_became_peered":"2024-08-17T17:47:33.474247+0000","last_unstale":"2024-08-17T17:50:02.487366+0000","last_undegraded":"2024-08-17T17:50:02.487366+0000","last_fullsized":"2024-08-17T17:50:02.487366+0000","mapping_epoch":413,"log_start":"421'3800","ondisk_log_start":"421'3800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:46:34.619479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337117184,"num_objects":5144,"num_object_clones":0,"num_object_copies":15432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10806,"num_write_kb":510400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":494,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080062+0000","last_change":"2024-08-17T17:33:40.650926+0000","last_active":"2024-08-17T17:50:02.080062+0000","last_peered":"2024-08-17T17:50:02.080062+0000","last_clean":"2024-08-17T17:50:02.080062+0000","last_became_active":"2024-08-17T17:33:40.650582+0000","last_became_peered":"2024-08-17T17:33:40.650582+0000","last_unstale":"2024-08-17T17:50:02.080062+0000","last_undegraded":"2024-08-17T17:50:02.080062+0000","last_fullsized":"2024-08-17T17:50:02.080062+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:21.668694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":376,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.577801+0000","last_change":"2024-08-17T17:24:36.990968+0000","last_active":"2024-08-17T17:49:57.577801+0000","last_peered":"2024-08-17T17:49:57.577801+0000","last_clean":"2024-08-17T17:49:57.577801+0000","last_became_active":"2024-08-17T17:24:36.990665+0000","last_became_peered":"2024-08-17T17:24:36.990665+0000","last_unstale":"2024-08-17T17:49:57.577801+0000","last_undegraded":"2024-08-17T17:49:57.577801+0000","last_fullsized":"2024-08-17T17:49:57.577801+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:08.530308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":419,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.084109+0000","last_change":"2024-08-17T17:33:40.641947+0000","last_active":"2024-08-17T17:50:02.084109+0000","last_peered":"2024-08-17T17:50:02.084109+0000","last_clean":"2024-08-17T17:50:02.084109+0000","last_became_active":"2024-08-17T17:33:40.641416+0000","last_became_peered":"2024-08-17T17:33:40.641416+0000","last_unstale":"2024-08-17T17:50:02.084109+0000","last_undegraded":"2024-08-17T17:50:02.084109+0000","last_fullsized":"2024-08-17T17:50:02.084109+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:58.921986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"440'10896","reported_seq":10930,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:05.181158+0000","last_change":"2024-08-17T17:47:33.471881+0000","last_active":"2024-08-17T17:50:05.181158+0000","last_peered":"2024-08-17T17:50:05.181158+0000","last_clean":"2024-08-17T17:50:05.181158+0000","last_became_active":"2024-08-17T17:47:33.471614+0000","last_became_peered":"2024-08-17T17:47:33.471614+0000","last_unstale":"2024-08-17T17:50:05.181158+0000","last_undegraded":"2024-08-17T17:50:05.181158+0000","last_fullsized":"2024-08-17T17:50:05.181158+0000","mapping_epoch":413,"log_start":"421'3200","ondisk_log_start":"421'3200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7696,"log_dups_size":0,"ondisk_log_size":7696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:40.873776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328990720,"num_objects":5020,"num_object_clones":0,"num_object_copies":15060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10896,"num_write_kb":509312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":377,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.090280+0000","last_change":"2024-08-17T17:49:32.576461+0000","last_active":"2024-08-17T17:50:02.090280+0000","last_peered":"2024-08-17T17:50:02.090280+0000","last_clean":"2024-08-17T17:50:02.090280+0000","last_became_active":"2024-08-17T17:49:32.576024+0000","last_became_peered":"2024-08-17T17:49:32.576024+0000","last_unstale":"2024-08-17T17:50:02.090280+0000","last_undegraded":"2024-08-17T17:50:02.090280+0000","last_fullsized":"2024-08-17T17:50:02.090280+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:54.701755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":176,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.580051+0000","last_change":"2024-08-17T17:33:20.285085+0000","last_active":"2024-08-17T17:49:57.580051+0000","last_peered":"2024-08-17T17:49:57.580051+0000","last_clean":"2024-08-17T17:49:57.580051+0000","last_became_active":"2024-08-17T17:33:20.284594+0000","last_became_peered":"2024-08-17T17:33:20.284594+0000","last_unstale":"2024-08-17T17:49:57.580051+0000","last_undegraded":"2024-08-17T17:49:57.580051+0000","last_fullsized":"2024-08-17T17:49:57.580051+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:40.229421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":27,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.568266+0000","last_change":"2024-08-17T17:48:13.475561+0000","last_active":"2024-08-17T17:49:57.568266+0000","last_peered":"2024-08-17T17:49:57.568266+0000","last_clean":"2024-08-17T17:49:57.568266+0000","last_became_active":"2024-08-17T17:48:13.475084+0000","last_became_peered":"2024-08-17T17:48:13.475084+0000","last_unstale":"2024-08-17T17:49:57.568266+0000","last_undegraded":"2024-08-17T17:49:57.568266+0000","last_fullsized":"2024-08-17T17:49:57.568266+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:23.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.d","version":"440'10741","reported_seq":10777,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.240902+0000","last_change":"2024-08-17T17:47:33.473230+0000","last_active":"2024-08-17T17:50:02.240902+0000","last_peered":"2024-08-17T17:50:02.240902+0000","last_clean":"2024-08-17T17:50:02.240902+0000","last_became_active":"2024-08-17T17:47:33.472924+0000","last_became_peered":"2024-08-17T17:47:33.472924+0000","last_unstale":"2024-08-17T17:50:02.240902+0000","last_undegraded":"2024-08-17T17:50:02.240902+0000","last_fullsized":"2024-08-17T17:50:02.240902+0000","mapping_epoch":413,"log_start":"424'4400","ondisk_log_start":"424'4400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:47:58.863358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316120,"num_objects":5087,"num_object_clones":0,"num_object_copies":15261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5087,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10741,"num_write_kb":506433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":562,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080240+0000","last_change":"2024-08-17T17:21:31.910529+0000","last_active":"2024-08-17T17:50:02.080240+0000","last_peered":"2024-08-17T17:50:02.080240+0000","last_clean":"2024-08-17T17:50:02.080240+0000","last_became_active":"2024-08-17T17:21:31.910006+0000","last_became_peered":"2024-08-17T17:21:31.910006+0000","last_unstale":"2024-08-17T17:50:02.080240+0000","last_undegraded":"2024-08-17T17:50:02.080240+0000","last_fullsized":"2024-08-17T17:50:02.080240+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:38.416021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":415,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571388+0000","last_change":"2024-08-17T17:24:37.045029+0000","last_active":"2024-08-17T17:49:57.571388+0000","last_peered":"2024-08-17T17:49:57.571388+0000","last_clean":"2024-08-17T17:49:57.571388+0000","last_became_active":"2024-08-17T17:24:37.043717+0000","last_became_peered":"2024-08-17T17:24:37.043717+0000","last_unstale":"2024-08-17T17:49:57.571388+0000","last_undegraded":"2024-08-17T17:49:57.571388+0000","last_fullsized":"2024-08-17T17:49:57.571388+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:17.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571031+0000","last_change":"2024-08-17T17:49:32.578600+0000","last_active":"2024-08-17T17:49:57.571031+0000","last_peered":"2024-08-17T17:49:57.571031+0000","last_clean":"2024-08-17T17:49:57.571031+0000","last_became_active":"2024-08-17T17:49:32.578330+0000","last_became_peered":"2024-08-17T17:49:32.578330+0000","last_unstale":"2024-08-17T17:49:57.571031+0000","last_undegraded":"2024-08-17T17:49:57.571031+0000","last_fullsized":"2024-08-17T17:49:57.571031+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:00:52.051839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.b","version":"440'10623","reported_seq":10689,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.277723+0000","last_change":"2024-08-17T17:49:32.596729+0000","last_active":"2024-08-17T17:50:02.277723+0000","last_peered":"2024-08-17T17:50:02.277723+0000","last_clean":"2024-08-17T17:50:02.277723+0000","last_became_active":"2024-08-17T17:49:32.578209+0000","last_became_peered":"2024-08-17T17:49:32.578209+0000","last_unstale":"2024-08-17T17:50:02.277723+0000","last_undegraded":"2024-08-17T17:50:02.277723+0000","last_fullsized":"2024-08-17T17:50:02.277723+0000","mapping_epoch":433,"log_start":"424'4348","ondisk_log_start":"424'4348","created":413,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:48:15.715714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333119488,"num_objects":5083,"num_object_clones":0,"num_object_copies":15249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10623,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":354,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578813+0000","last_change":"2024-08-17T17:33:20.282341+0000","last_active":"2024-08-17T17:49:57.578813+0000","last_peered":"2024-08-17T17:49:57.578813+0000","last_clean":"2024-08-17T17:49:57.578813+0000","last_became_active":"2024-08-17T17:33:20.282010+0000","last_became_peered":"2024-08-17T17:33:20.282010+0000","last_unstale":"2024-08-17T17:49:57.578813+0000","last_undegraded":"2024-08-17T17:49:57.578813+0000","last_fullsized":"2024-08-17T17:49:57.578813+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:26.566039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":465,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080859+0000","last_change":"2024-08-17T17:49:32.580420+0000","last_active":"2024-08-17T17:50:02.080859+0000","last_peered":"2024-08-17T17:50:02.080859+0000","last_clean":"2024-08-17T17:50:02.080859+0000","last_became_active":"2024-08-17T17:49:32.580022+0000","last_became_peered":"2024-08-17T17:49:32.580022+0000","last_unstale":"2024-08-17T17:50:02.080859+0000","last_undegraded":"2024-08-17T17:50:02.080859+0000","last_fullsized":"2024-08-17T17:50:02.080859+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:11.347554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":721,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080473+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:50:02.080473+0000","last_peered":"2024-08-17T17:50:02.080473+0000","last_clean":"2024-08-17T17:50:02.080473+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:50:02.080473+0000","last_undegraded":"2024-08-17T17:50:02.080473+0000","last_fullsized":"2024-08-17T17:50:02.080473+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"440'10792","reported_seq":10858,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:04.182347+0000","last_change":"2024-08-17T17:47:33.468618+0000","last_active":"2024-08-17T17:50:04.182347+0000","last_peered":"2024-08-17T17:50:04.182347+0000","last_clean":"2024-08-17T17:50:04.182347+0000","last_became_active":"2024-08-17T17:47:33.468343+0000","last_became_peered":"2024-08-17T17:47:33.468343+0000","last_unstale":"2024-08-17T17:50:04.182347+0000","last_undegraded":"2024-08-17T17:50:04.182347+0000","last_fullsized":"2024-08-17T17:50:04.182347+0000","mapping_epoch":413,"log_start":"425'4900","ondisk_log_start":"425'4900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:35:12.974392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331612160,"num_objects":5060,"num_object_clones":0,"num_object_copies":15180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10792,"num_write_kb":507264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":350,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.584837+0000","last_change":"2024-08-17T17:33:20.280522+0000","last_active":"2024-08-17T17:49:57.584837+0000","last_peered":"2024-08-17T17:49:57.584837+0000","last_clean":"2024-08-17T17:49:57.584837+0000","last_became_active":"2024-08-17T17:33:20.280022+0000","last_became_peered":"2024-08-17T17:33:20.280022+0000","last_unstale":"2024-08-17T17:49:57.584837+0000","last_undegraded":"2024-08-17T17:49:57.584837+0000","last_fullsized":"2024-08-17T17:49:57.584837+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:22.305544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":476,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.083644+0000","last_change":"2024-08-17T17:33:40.634976+0000","last_active":"2024-08-17T17:50:02.083644+0000","last_peered":"2024-08-17T17:50:02.083644+0000","last_clean":"2024-08-17T17:50:02.083644+0000","last_became_active":"2024-08-17T17:33:40.634738+0000","last_became_peered":"2024-08-17T17:33:40.634738+0000","last_unstale":"2024-08-17T17:50:02.083644+0000","last_undegraded":"2024-08-17T17:50:02.083644+0000","last_fullsized":"2024-08-17T17:50:02.083644+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:18.161902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":492,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578618+0000","last_change":"2024-08-17T17:33:40.650305+0000","last_active":"2024-08-17T17:49:57.578618+0000","last_peered":"2024-08-17T17:49:57.578618+0000","last_clean":"2024-08-17T17:49:57.578618+0000","last_became_active":"2024-08-17T17:33:40.650002+0000","last_became_peered":"2024-08-17T17:33:40.650002+0000","last_unstale":"2024-08-17T17:49:57.578618+0000","last_undegraded":"2024-08-17T17:49:57.578618+0000","last_fullsized":"2024-08-17T17:49:57.578618+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:11.512097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"440'10816","reported_seq":10851,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:05.173380+0000","last_change":"2024-08-17T17:47:33.473836+0000","last_active":"2024-08-17T17:50:05.173380+0000","last_peered":"2024-08-17T17:50:05.173380+0000","last_clean":"2024-08-17T17:50:05.173380+0000","last_became_active":"2024-08-17T17:47:33.473133+0000","last_became_peered":"2024-08-17T17:47:33.473133+0000","last_unstale":"2024-08-17T17:50:05.173380+0000","last_undegraded":"2024-08-17T17:50:05.173380+0000","last_fullsized":"2024-08-17T17:50:05.173380+0000","mapping_epoch":413,"log_start":"421'3100","ondisk_log_start":"421'3100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7716,"log_dups_size":0,"ondisk_log_size":7716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:19.139546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10816,"num_write_kb":507328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":346,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.083680+0000","last_change":"2024-08-17T17:49:32.576289+0000","last_active":"2024-08-17T17:50:02.083680+0000","last_peered":"2024-08-17T17:50:02.083680+0000","last_clean":"2024-08-17T17:50:02.083680+0000","last_became_active":"2024-08-17T17:49:32.575928+0000","last_became_peered":"2024-08-17T17:49:32.575928+0000","last_unstale":"2024-08-17T17:50:02.083680+0000","last_undegraded":"2024-08-17T17:50:02.083680+0000","last_fullsized":"2024-08-17T17:50:02.083680+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:57.303438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":719,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080683+0000","last_change":"2024-08-17T17:07:51.483301+0000","last_active":"2024-08-17T17:50:02.080683+0000","last_peered":"2024-08-17T17:50:02.080683+0000","last_clean":"2024-08-17T17:50:02.080683+0000","last_became_active":"2024-08-17T17:07:51.482998+0000","last_became_peered":"2024-08-17T17:07:51.482998+0000","last_unstale":"2024-08-17T17:50:02.080683+0000","last_undegraded":"2024-08-17T17:50:02.080683+0000","last_fullsized":"2024-08-17T17:50:02.080683+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.942766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":737,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.078732+0000","last_change":"2024-08-17T17:33:40.658844+0000","last_active":"2024-08-17T17:50:02.078732+0000","last_peered":"2024-08-17T17:50:02.078732+0000","last_clean":"2024-08-17T17:50:02.078732+0000","last_became_active":"2024-08-17T17:33:40.658583+0000","last_became_peered":"2024-08-17T17:33:40.658583+0000","last_unstale":"2024-08-17T17:50:02.078732+0000","last_undegraded":"2024-08-17T17:50:02.078732+0000","last_fullsized":"2024-08-17T17:50:02.078732+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:57.298913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"440'11144","reported_seq":11179,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:05.114398+0000","last_change":"2024-08-17T17:47:33.468582+0000","last_active":"2024-08-17T17:50:05.114398+0000","last_peered":"2024-08-17T17:50:05.114398+0000","last_clean":"2024-08-17T17:50:05.114398+0000","last_became_active":"2024-08-17T17:47:33.468018+0000","last_became_peered":"2024-08-17T17:47:33.468018+0000","last_unstale":"2024-08-17T17:50:05.114398+0000","last_undegraded":"2024-08-17T17:50:05.114398+0000","last_fullsized":"2024-08-17T17:50:05.114398+0000","mapping_epoch":413,"log_start":"421'3400","ondisk_log_start":"421'3400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:32.453242+0000","last_clean_scrub_stamp":"2024-08-17T17:47:32.453242+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:55:27.900073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339869696,"num_objects":5186,"num_object_clones":0,"num_object_copies":15558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11144,"num_write_kb":522560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":409,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.084023+0000","last_change":"2024-08-17T17:24:37.098708+0000","last_active":"2024-08-17T17:50:02.084023+0000","last_peered":"2024-08-17T17:50:02.084023+0000","last_clean":"2024-08-17T17:50:02.084023+0000","last_became_active":"2024-08-17T17:24:37.076343+0000","last_became_peered":"2024-08-17T17:24:37.076343+0000","last_unstale":"2024-08-17T17:50:02.084023+0000","last_undegraded":"2024-08-17T17:50:02.084023+0000","last_fullsized":"2024-08-17T17:50:02.084023+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:22.770146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":444,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.584310+0000","last_change":"2024-08-17T17:33:40.655360+0000","last_active":"2024-08-17T17:49:57.584310+0000","last_peered":"2024-08-17T17:49:57.584310+0000","last_clean":"2024-08-17T17:49:57.584310+0000","last_became_active":"2024-08-17T17:33:40.654816+0000","last_became_peered":"2024-08-17T17:33:40.654816+0000","last_unstale":"2024-08-17T17:49:57.584310+0000","last_undegraded":"2024-08-17T17:49:57.584310+0000","last_fullsized":"2024-08-17T17:49:57.584310+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:27.022048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":394,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.578058+0000","last_change":"2024-08-17T17:21:31.935115+0000","last_active":"2024-08-17T17:49:57.578058+0000","last_peered":"2024-08-17T17:49:57.578058+0000","last_clean":"2024-08-17T17:49:57.578058+0000","last_became_active":"2024-08-17T17:21:31.934774+0000","last_became_peered":"2024-08-17T17:21:31.934774+0000","last_unstale":"2024-08-17T17:49:57.578058+0000","last_undegraded":"2024-08-17T17:49:57.578058+0000","last_fullsized":"2024-08-17T17:49:57.578058+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:19.679887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":394,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571268+0000","last_change":"2024-08-17T17:07:51.483940+0000","last_active":"2024-08-17T17:49:57.571268+0000","last_peered":"2024-08-17T17:49:57.571268+0000","last_clean":"2024-08-17T17:49:57.571268+0000","last_became_active":"2024-08-17T17:07:51.482820+0000","last_became_peered":"2024-08-17T17:07:51.482820+0000","last_unstale":"2024-08-17T17:49:57.571268+0000","last_undegraded":"2024-08-17T17:49:57.571268+0000","last_fullsized":"2024-08-17T17:49:57.571268+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:44.621106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":437,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.574582+0000","last_change":"2024-08-17T17:33:40.876803+0000","last_active":"2024-08-17T17:49:57.574582+0000","last_peered":"2024-08-17T17:49:57.574582+0000","last_clean":"2024-08-17T17:49:57.574582+0000","last_became_active":"2024-08-17T17:33:40.870380+0000","last_became_peered":"2024-08-17T17:33:40.870380+0000","last_unstale":"2024-08-17T17:49:57.574582+0000","last_undegraded":"2024-08-17T17:49:57.574582+0000","last_fullsized":"2024-08-17T17:49:57.574582+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:29.340365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":726,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080393+0000","last_change":"2024-08-17T17:33:20.281764+0000","last_active":"2024-08-17T17:50:02.080393+0000","last_peered":"2024-08-17T17:50:02.080393+0000","last_clean":"2024-08-17T17:50:02.080393+0000","last_became_active":"2024-08-17T17:33:20.281443+0000","last_became_peered":"2024-08-17T17:33:20.281443+0000","last_unstale":"2024-08-17T17:50:02.080393+0000","last_undegraded":"2024-08-17T17:50:02.080393+0000","last_fullsized":"2024-08-17T17:50:02.080393+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:29.051378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":265,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079684+0000","last_change":"2024-08-17T17:33:40.673207+0000","last_active":"2024-08-17T17:50:02.079684+0000","last_peered":"2024-08-17T17:50:02.079684+0000","last_clean":"2024-08-17T17:50:02.079684+0000","last_became_active":"2024-08-17T17:33:40.645490+0000","last_became_peered":"2024-08-17T17:33:40.645490+0000","last_unstale":"2024-08-17T17:50:02.079684+0000","last_undegraded":"2024-08-17T17:50:02.079684+0000","last_fullsized":"2024-08-17T17:50:02.079684+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:17.913464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":344,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.579308+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:49:57.579308+0000","last_peered":"2024-08-17T17:49:57.579308+0000","last_clean":"2024-08-17T17:49:57.579308+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:49:57.579308+0000","last_undegraded":"2024-08-17T17:49:57.579308+0000","last_fullsized":"2024-08-17T17:49:57.579308+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":684,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079441+0000","last_change":"2024-08-17T17:33:40.657957+0000","last_active":"2024-08-17T17:50:02.079441+0000","last_peered":"2024-08-17T17:50:02.079441+0000","last_clean":"2024-08-17T17:50:02.079441+0000","last_became_active":"2024-08-17T17:33:40.657134+0000","last_became_peered":"2024-08-17T17:33:40.657134+0000","last_unstale":"2024-08-17T17:50:02.079441+0000","last_undegraded":"2024-08-17T17:50:02.079441+0000","last_fullsized":"2024-08-17T17:50:02.079441+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:47.318089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":488,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.582315+0000","last_change":"2024-08-17T17:25:46.632118+0000","last_active":"2024-08-17T17:49:57.582315+0000","last_peered":"2024-08-17T17:49:57.582315+0000","last_clean":"2024-08-17T17:49:57.582315+0000","last_became_active":"2024-08-17T17:25:46.631826+0000","last_became_peered":"2024-08-17T17:25:46.631826+0000","last_unstale":"2024-08-17T17:49:57.582315+0000","last_undegraded":"2024-08-17T17:49:57.582315+0000","last_fullsized":"2024-08-17T17:49:57.582315+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:33.742417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":678,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.078647+0000","last_change":"2024-08-17T17:12:40.349755+0000","last_active":"2024-08-17T17:50:02.078647+0000","last_peered":"2024-08-17T17:50:02.078647+0000","last_clean":"2024-08-17T17:50:02.078647+0000","last_became_active":"2024-08-17T17:12:40.347556+0000","last_became_peered":"2024-08-17T17:12:40.347556+0000","last_unstale":"2024-08-17T17:50:02.078647+0000","last_undegraded":"2024-08-17T17:50:02.078647+0000","last_fullsized":"2024-08-17T17:50:02.078647+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:59:11.802614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":411,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.582902+0000","last_change":"2024-08-17T17:21:31.868654+0000","last_active":"2024-08-17T17:49:57.582902+0000","last_peered":"2024-08-17T17:49:57.582902+0000","last_clean":"2024-08-17T17:49:57.582902+0000","last_became_active":"2024-08-17T17:21:31.868382+0000","last_became_peered":"2024-08-17T17:21:31.868382+0000","last_unstale":"2024-08-17T17:49:57.582902+0000","last_undegraded":"2024-08-17T17:49:57.582902+0000","last_fullsized":"2024-08-17T17:49:57.582902+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:59.133609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":331,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.085399+0000","last_change":"2024-08-17T17:49:32.576497+0000","last_active":"2024-08-17T17:50:02.085399+0000","last_peered":"2024-08-17T17:50:02.085399+0000","last_clean":"2024-08-17T17:50:02.085399+0000","last_became_active":"2024-08-17T17:49:32.576128+0000","last_became_peered":"2024-08-17T17:49:32.576128+0000","last_unstale":"2024-08-17T17:50:02.085399+0000","last_undegraded":"2024-08-17T17:50:02.085399+0000","last_fullsized":"2024-08-17T17:50:02.085399+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:38.681960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":426,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.078988+0000","last_change":"2024-08-17T17:33:40.652511+0000","last_active":"2024-08-17T17:50:02.078988+0000","last_peered":"2024-08-17T17:50:02.078988+0000","last_clean":"2024-08-17T17:50:02.078988+0000","last_became_active":"2024-08-17T17:33:40.652232+0000","last_became_peered":"2024-08-17T17:33:40.652232+0000","last_unstale":"2024-08-17T17:50:02.078988+0000","last_undegraded":"2024-08-17T17:50:02.078988+0000","last_fullsized":"2024-08-17T17:50:02.078988+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:38.007356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":717,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079983+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:50:02.079983+0000","last_peered":"2024-08-17T17:50:02.079983+0000","last_clean":"2024-08-17T17:50:02.079983+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:50:02.079983+0000","last_undegraded":"2024-08-17T17:50:02.079983+0000","last_fullsized":"2024-08-17T17:50:02.079983+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.080359+0000","last_change":"2024-08-17T17:49:32.576719+0000","last_active":"2024-08-17T17:50:02.080359+0000","last_peered":"2024-08-17T17:50:02.080359+0000","last_clean":"2024-08-17T17:50:02.080359+0000","last_became_active":"2024-08-17T17:49:32.576409+0000","last_became_peered":"2024-08-17T17:49:32.576409+0000","last_unstale":"2024-08-17T17:50:02.080359+0000","last_undegraded":"2024-08-17T17:50:02.080359+0000","last_fullsized":"2024-08-17T17:50:02.080359+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:40:16.038780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":423,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.563577+0000","last_change":"2024-08-17T17:33:20.284576+0000","last_active":"2024-08-17T17:49:57.563577+0000","last_peered":"2024-08-17T17:49:57.563577+0000","last_clean":"2024-08-17T17:49:57.563577+0000","last_became_active":"2024-08-17T17:33:20.284108+0000","last_became_peered":"2024-08-17T17:33:20.284108+0000","last_unstale":"2024-08-17T17:49:57.563577+0000","last_undegraded":"2024-08-17T17:49:57.563577+0000","last_fullsized":"2024-08-17T17:49:57.563577+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:44.134623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":365,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.079043+0000","last_change":"2024-08-17T17:33:20.284474+0000","last_active":"2024-08-17T17:50:02.079043+0000","last_peered":"2024-08-17T17:50:02.079043+0000","last_clean":"2024-08-17T17:50:02.079043+0000","last_became_active":"2024-08-17T17:33:20.284156+0000","last_became_peered":"2024-08-17T17:33:20.284156+0000","last_unstale":"2024-08-17T17:50:02.079043+0000","last_undegraded":"2024-08-17T17:50:02.079043+0000","last_fullsized":"2024-08-17T17:50:02.079043+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:32.028972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":422,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.083824+0000","last_change":"2024-08-17T17:33:40.642949+0000","last_active":"2024-08-17T17:50:02.083824+0000","last_peered":"2024-08-17T17:50:02.083824+0000","last_clean":"2024-08-17T17:50:02.083824+0000","last_became_active":"2024-08-17T17:33:40.642023+0000","last_became_peered":"2024-08-17T17:33:40.642023+0000","last_unstale":"2024-08-17T17:50:02.083824+0000","last_undegraded":"2024-08-17T17:50:02.083824+0000","last_fullsized":"2024-08-17T17:50:02.083824+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:21:42.947968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":431,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571206+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:49:57.571206+0000","last_peered":"2024-08-17T17:49:57.571206+0000","last_clean":"2024-08-17T17:49:57.571206+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:49:57.571206+0000","last_undegraded":"2024-08-17T17:49:57.571206+0000","last_fullsized":"2024-08-17T17:49:57.571206+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":294,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.574458+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:49:57.574458+0000","last_peered":"2024-08-17T17:49:57.574458+0000","last_clean":"2024-08-17T17:49:57.574458+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:49:57.574458+0000","last_undegraded":"2024-08-17T17:49:57.574458+0000","last_fullsized":"2024-08-17T17:49:57.574458+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":431,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.563421+0000","last_change":"2024-08-17T17:24:37.094334+0000","last_active":"2024-08-17T17:49:57.563421+0000","last_peered":"2024-08-17T17:49:57.563421+0000","last_clean":"2024-08-17T17:49:57.563421+0000","last_became_active":"2024-08-17T17:24:37.094016+0000","last_became_peered":"2024-08-17T17:24:37.094016+0000","last_unstale":"2024-08-17T17:49:57.563421+0000","last_undegraded":"2024-08-17T17:49:57.563421+0000","last_fullsized":"2024-08-17T17:49:57.563421+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:52.116644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":294,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.569072+0000","last_change":"2024-08-17T17:21:31.874477+0000","last_active":"2024-08-17T17:49:57.569072+0000","last_peered":"2024-08-17T17:49:57.569072+0000","last_clean":"2024-08-17T17:49:57.569072+0000","last_became_active":"2024-08-17T17:21:31.874166+0000","last_became_peered":"2024-08-17T17:21:31.874166+0000","last_unstale":"2024-08-17T17:49:57.569072+0000","last_undegraded":"2024-08-17T17:49:57.569072+0000","last_fullsized":"2024-08-17T17:49:57.569072+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:52:48.627930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":521,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.090220+0000","last_change":"2024-08-17T17:49:32.577625+0000","last_active":"2024-08-17T17:50:02.090220+0000","last_peered":"2024-08-17T17:50:02.090220+0000","last_clean":"2024-08-17T17:50:02.090220+0000","last_became_active":"2024-08-17T17:49:32.577269+0000","last_became_peered":"2024-08-17T17:49:32.577269+0000","last_unstale":"2024-08-17T17:50:02.090220+0000","last_undegraded":"2024-08-17T17:50:02.090220+0000","last_fullsized":"2024-08-17T17:50:02.090220+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:46.618522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":29,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.571546+0000","last_change":"2024-08-17T17:49:32.579485+0000","last_active":"2024-08-17T17:49:57.571546+0000","last_peered":"2024-08-17T17:49:57.571546+0000","last_clean":"2024-08-17T17:49:57.571546+0000","last_became_active":"2024-08-17T17:49:32.579092+0000","last_became_peered":"2024-08-17T17:49:32.579092+0000","last_unstale":"2024-08-17T17:49:57.571546+0000","last_undegraded":"2024-08-17T17:49:57.571546+0000","last_fullsized":"2024-08-17T17:49:57.571546+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:17.053873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":394,"reported_epoch":439,"state":"active+clean","last_fresh":"2024-08-17T17:49:57.563142+0000","last_change":"2024-08-17T17:07:51.482354+0000","last_active":"2024-08-17T17:49:57.563142+0000","last_peered":"2024-08-17T17:49:57.563142+0000","last_clean":"2024-08-17T17:49:57.563142+0000","last_became_active":"2024-08-17T17:07:51.481641+0000","last_became_peered":"2024-08-17T17:07:51.481641+0000","last_unstale":"2024-08-17T17:49:57.563142+0000","last_undegraded":"2024-08-17T17:49:57.563142+0000","last_fullsized":"2024-08-17T17:49:57.563142+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:54.757719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":425,"reported_epoch":440,"state":"active+clean","last_fresh":"2024-08-17T17:50:02.083826+0000","last_change":"2024-08-17T17:33:20.305566+0000","last_active":"2024-08-17T17:50:02.083826+0000","last_peered":"2024-08-17T17:50:02.083826+0000","last_clean":"2024-08-17T17:50:02.083826+0000","last_became_active":"2024-08-17T17:33:20.290155+0000","last_became_peered":"2024-08-17T17:33:20.290155+0000","last_unstale":"2024-08-17T17:50:02.083826+0000","last_undegraded":"2024-08-17T17:50:02.083826+0000","last_fullsized":"2024-08-17T17:50:02.083826+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:30.320590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5382275096,"num_objects":82128,"num_object_clones":0,"num_object_copies":246384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82128,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173500,"num_write_kb":8180033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16112824320,"data_stored":16112812104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108252,"ondisk_log_size":108252,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":6670,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":135168,"data_stored":22154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29419,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170198,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463573,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3848852,"kb_used_data":2356108,"kb_used_omap":1,"kb_used_meta":1492670,"kb_avail":89896300,"statfs":{"total":95995035648,"available":92053811200,"internally_reserved":0,"allocated":2412654592,"data_stored":2410241332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1528494424},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492248,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4566852,"kb_used_data":3602108,"kb_used_omap":7,"kb_used_meta":964728,"kb_avail":89178300,"statfs":{"total":95995035648,"available":91318579200,"internally_reserved":0,"allocated":3688558592,"data_stored":3686163563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":987881805},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986839,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5524108,"kb_used_data":4019588,"kb_used_omap":1,"kb_used_meta":1504510,"kb_avail":88221044,"statfs":{"total":95995035648,"available":90338349056,"internally_reserved":0,"allocated":4116058112,"data_stored":4113651720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1540618584},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225667,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3426440,"kb_used_data":2307968,"kb_used_omap":4,"kb_used_meta":1118395,"kb_avail":90318712,"statfs":{"total":95995035648,"available":92486361088,"internally_reserved":0,"allocated":2363359232,"data_stored":2360960429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1145236928},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396075992,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533885,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3202700,"kb_used_data":1980228,"kb_used_omap":5,"kb_used_meta":1222458,"kb_avail":90542452,"statfs":{"total":95995035648,"available":92715470848,"internally_reserved":0,"allocated":2027753472,"data_stored":2025349867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1251797232},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089862,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2794788,"kb_used_data":1627160,"kb_used_omap":5,"kb_used_meta":1167610,"kb_avail":90950364,"statfs":{"total":95995035648,"available":93133172736,"internally_reserved":0,"allocated":1666211840,"data_stored":1663794570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1195632699},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1649213440,"data_stored":1649213440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2007306240,"data_stored":2007302168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2342326272,"data_stored":2342322200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4076867584,"data_stored":4076863512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3643539456,"data_stored":3643539456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2393571328,"data_stored":2393571328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:50:09.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-17T17:50:09.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:09.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:09.525 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:09.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:09.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:09.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:09.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:09.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:09 smithi152 ceph-mon[93456]: pgmap v1767: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 908 op/s 2024-08-17T17:50:09.828 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:09 smithi142 ceph-mon[106729]: pgmap v1767: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 908 op/s 2024-08-17T17:50:09.829 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:09 smithi142 ceph-mon[103614]: pgmap v1767: 89 pgs: 89 active+clean; 4.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 908 op/s 2024-08-17T17:50:09.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:09.837 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:09.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:09.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:10.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:10.078 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:10.078 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:10.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:10.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:10.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:10.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:10.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:10.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:10.505 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:50:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:50:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:10.576 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:50:10.607 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:10.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:10.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:10.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:10.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:50:10.855 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/483087551' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:10.855 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:10.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/483087551' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/483087551' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:10.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:10.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:10.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:11.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:11.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:11.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:11.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:11.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:11.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:11.296 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:11.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:11.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:11.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:11.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:11.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.599 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:11.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:11.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:11.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:11.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[106729]: pgmap v1768: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s 2024-08-17T17:50:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[106729]: osdmap e441: 8 total, 7 up, 6 in 2024-08-17T17:50:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[103614]: pgmap v1768: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s 2024-08-17T17:50:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[103614]: osdmap e441: 8 total, 7 up, 6 in 2024-08-17T17:50:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:11.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:11 smithi152 ceph-mon[93456]: pgmap v1768: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s 2024-08-17T17:50:11.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:11 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:11.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:11 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:11.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:11 smithi152 ceph-mon[93456]: osdmap e441: 8 total, 7 up, 6 in 2024-08-17T17:50:11.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:11.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:11.839 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:11.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:11.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:11.892 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:11.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:12.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:12.033 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:50:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:50:11] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:50:12.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:12.142 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:12.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:12.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:12.279 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:12.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:12.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:12.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:12.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:12.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:12.673 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:12.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.818 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:12.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:12.870 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:12 smithi152 ceph-mon[93456]: pgmap v1770: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 915 op/s 2024-08-17T17:50:12.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:12 smithi142 ceph-mon[106729]: pgmap v1770: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 915 op/s 2024-08-17T17:50:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:12 smithi142 ceph-mon[103614]: pgmap v1770: 89 pgs: 89 active+clean; 4.7 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 915 op/s 2024-08-17T17:50:12.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:12.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:12.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:13.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:13.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:13.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:13.167 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:50:13.167 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:13.189 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:50:13.189 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:13.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:13.217 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:50:13.218 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:13.239 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:50:13.239 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:13.260 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:13.293 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:50:13.293 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:13.325 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:50:13.326 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:13.353 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:50:13.354 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:13.354 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:50:13.361 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:182635 13.9 13:92a55f39:::benchmark_data_smithi142_393845_object54819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.370108+0000", 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.015825459, 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015856559999999999, 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:13.388 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 182635 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370108+0000", 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370108+0000", 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370107+0000", 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370114+0000", 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4960000000000001e-06 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370130+0000", 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6194000000000001e-05 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370135+0000", 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7819999999999999e-06 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370179+0000", 2024-08-17T17:50:13.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3927999999999997e-05 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370353+0000", 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017382000000000001 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370452+0000", 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9080999999999995e-05 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384298+0000", 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013845302 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384304+0000", 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4200000000000004e-06 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:182636 13.9 13:94099f88:::benchmark_data_smithi142_393845_object54820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.370590+0000", 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.01534423, 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015464505999999999, 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 182636 2024-08-17T17:50:13.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370590+0000", 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370590+0000", 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370589+0000", 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370593+0000", 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5950000000000001e-06 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370602+0000", 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3380000000000005e-06 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.370607+0000", 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3900000000000001e-06 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.374530+0000", 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003922966 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.374628+0000", 2024-08-17T17:50:13.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8097000000000006e-05 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.374729+0000", 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101061 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384966+0000", 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010236492999999999 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.398 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:182641 13.8 13:19c75ed9:::benchmark_data_smithi142_393845_object54825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.376748+0000", 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0091860199999999996, 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093798730000000004, 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:13.399 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 182641 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376748+0000", 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376748+0000", 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376746+0000", 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376753+0000", 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5640000000000002e-06 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376764+0000", 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1136e-05 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376770+0000", 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0589999999999999e-06 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376839+0000", 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8738999999999994e-05 2024-08-17T17:50:13.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.376949+0000", 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110392 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377054+0000", 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105107 2024-08-17T17:50:13.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385082+0000", 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0080276500000000008 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:182644 13.8 13:13e60927:::benchmark_data_smithi142_393845_object54828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.377258+0000", 2024-08-17T17:50:13.405 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0086758140000000004, 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089416370000000005, 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 182644 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377258+0000", 2024-08-17T17:50:13.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377258+0000", 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377257+0000", 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377265+0000", 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6559999999999992e-06 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377276+0000", 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1051e-05 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.377282+0000", 2024-08-17T17:50:13.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.947e-06 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380994+0000", 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037115389999999998 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.381106+0000", 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112724 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.381214+0000", 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010825 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:182642 13.c e441/413 13:3bfdb98e:::benchmark_data_smithi142_393845_object54826:head v 441'11502, mlcod=441'11499)", 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.378462+0000", 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0074721379999999997, 2024-08-17T17:50:13.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078038730000000002, 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.378462+0000", 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.378462+0000", 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.378461+0000", 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.378471+0000", 2024-08-17T17:50:13.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0168e-05 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.378482+0000", 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0956e-05 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.378487+0000", 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2220000000000001e-06 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380940+0000", 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024526550000000002 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380957+0000", 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7560000000000001e-05 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:182634 13.f e441/413 13:f5aa82d4:::benchmark_data_smithi142_393845_object54818:head v 441'11435, mlcod=441'11430)", 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.380870+0000", 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0050634069999999998, 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054488180000000002, 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380870+0000", 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380870+0000", 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380869+0000", 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380877+0000", 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7759999999999997e-06 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380890+0000", 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2408e-05 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380896+0000", 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1639999999999998e-06 2024-08-17T17:50:13.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.381346+0000", 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000450362 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.381370+0000", 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3529000000000001e-05 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:182633 13.4 e441/413)", 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.380920+0000", 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0050142149999999998, 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054487379999999998, 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380920+0000", 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380920+0000", 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380919+0000", 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380923+0000", 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.721e-06 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380931+0000", 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4619999999999999e-06 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.380934+0000", 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.681e-06 2024-08-17T17:50:13.421 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:182639 13.6 e441/413 13:616eccdd:::benchmark_data_smithi142_393845_object54823:head v 441'11574, mlcod=441'11572)", 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.383883+0000", 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0020510989999999998, 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002532973, 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.383883+0000", 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.383883+0000", 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.383882+0000", 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:13.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.383889+0000", 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1759999999999998e-06 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.383896+0000", 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4000000000000003e-06 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.383900+0000", 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8130000000000002e-06 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384105+0000", 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000204754 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384120+0000", 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5260999999999998e-05 2024-08-17T17:50:13.425 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:182645 13.5 e441/413 13:a0568cd9:::benchmark_data_smithi142_393845_object54829:head v 441'11416, mlcod=441'11414)", 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.384522+0000", 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001411821, 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001947666, 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384522+0000", 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384522+0000", 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384522+0000", 2024-08-17T17:50:13.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384528+0000", 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3099999999999997e-06 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384541+0000", 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3003000000000001e-05 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384546+0000", 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4260000000000002e-06 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384614+0000", 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7299999999999996e-05 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.384631+0000", 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7028999999999999e-05 2024-08-17T17:50:13.429 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:182646 13.9 13.16126ef9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.385232+0000", 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00070160199999999995, 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001289344, 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:13.430 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 182646 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385232+0000", 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385232+0000", 2024-08-17T17:50:13.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385231+0000", 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385237+0000", 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.755e-06 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385249+0000", 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2486e-05 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385255+0000", 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1859999999999997e-06 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.433 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:182647 13.8 13:190a8e43:::benchmark_data_smithi142_393845_object54831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.385264+0000", 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00067005099999999996, 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013102439999999999, 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 182647 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385264+0000", 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385264+0000", 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385263+0000", 2024-08-17T17:50:13.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385266+0000", 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0850000000000001e-06 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385274+0000", 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1559999999999998e-06 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385278+0000", 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.544e-06 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385310+0000", 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2268999999999999e-05 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385424+0000", 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011325299999999999 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385534+0000", 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110709 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.48005.0:182641 13.8 e441/413)", 2024-08-17T17:50:13.438 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:13.385780+0000", 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.000153814, 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00085589699999999999, 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385780+0000", 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385780+0000", 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385779+0000", 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385785+0000", 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6930000000000004e-06 2024-08-17T17:50:13.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385795+0000", 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0366999999999999e-05 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:13.385800+0000", 2024-08-17T17:50:13.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7979999999999996e-06 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 12 2024-08-17T17:50:13.442 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:50:13.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:13.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:13.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:50:13.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:13.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:13.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:13.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:13.724 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:13.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:13.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:13.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1868201211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1868201211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:13.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:13.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1868201211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:14.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:14.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:14.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:14.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:14.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:14.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:14.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:14.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:50:14.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:14.561 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:14.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:14.702 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:14.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:14.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:14.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:14 smithi152 ceph-mon[93456]: pgmap v1771: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 919 op/s 2024-08-17T17:50:14.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:14.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:14 smithi142 ceph-mon[106729]: pgmap v1771: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 919 op/s 2024-08-17T17:50:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:14 smithi142 ceph-mon[103614]: pgmap v1771: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 919 op/s 2024-08-17T17:50:14.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:14.967 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:14.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:50:15.007 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:50:15.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:15.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:15.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:15.255 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:15.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:15.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:15.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:15.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:15.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:15.624 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:15.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:15.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:15.800 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:15.834 INFO:teuthology.orchestra.run.smithi142.stderr:set pool 1 pgp_num to 1 2024-08-17T17:50:15.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:15.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:15.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:16.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:16.072 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1048238949' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:50:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:15 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:50:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1048238949' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:50:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:15 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:50:16.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:16.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:16.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:16.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:16.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1048238949' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:50:16.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:15 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:50:16.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:16.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.490 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:16.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:16.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:16.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:16.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:16.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:16.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:16.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:16.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:16.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:17.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:50:17.033 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:16 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:50:17.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:16 smithi142 ceph-mon[106729]: osdmap e442: 8 total, 7 up, 6 in 2024-08-17T17:50:17.034 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:16 smithi142 ceph-mon[106729]: pgmap v1773: 89 pgs: 89 active+clean; 4.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 825 op/s 2024-08-17T17:50:17.034 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:16 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:50:17.034 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:16 smithi142 ceph-mon[103614]: osdmap e442: 8 total, 7 up, 6 in 2024-08-17T17:50:17.034 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:16 smithi142 ceph-mon[103614]: pgmap v1773: 89 pgs: 89 active+clean; 4.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 825 op/s 2024-08-17T17:50:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.037 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:17.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:17.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:16 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:50:17.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:16 smithi152 ceph-mon[93456]: osdmap e442: 8 total, 7 up, 6 in 2024-08-17T17:50:17.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:16 smithi152 ceph-mon[93456]: pgmap v1773: 89 pgs: 89 active+clean; 4.4 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 825 op/s 2024-08-17T17:50:17.091 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:50:17.091 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:50:17.091 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:50:17.091 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:50:17.091 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:50:17.138 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:17.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:17.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:17.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:17.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:17.476 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:17.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:17.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:17.618 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:17.720 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:17.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:17.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:17.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:50:17.846 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:17.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:17.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:18.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:18.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:18.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:18.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:18.225 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:18.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:18.294 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:50:18.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:18.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:18.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.446 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:18.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:18.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:18.564 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:50:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.584 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:18.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:18.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:18.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:18.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:18.782 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/135564008' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:50:18.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:18.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:18.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/135564008' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:50:18.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/135564008' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:50:18.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:18.978 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:18.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:19.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:19.117 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:19.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:19.263 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:19.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:19.391 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:19.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:19.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:19.533 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[106729]: pgmap v1774: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:50:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/135564008' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:50:19.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[106729]: osdmap e443: 8 total, 7 up, 6 in 2024-08-17T17:50:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[103614]: pgmap v1774: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:50:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/135564008' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:50:19.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:19 smithi142 ceph-mon[103614]: osdmap e443: 8 total, 7 up, 6 in 2024-08-17T17:50:19.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:19.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:19.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:19.837 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:19.845 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:19 smithi152 ceph-mon[93456]: pgmap v1774: 89 pgs: 89 active+clean; 4.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:50:19.845 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:19 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:19.845 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/135564008' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:50:19.845 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:19 smithi152 ceph-mon[93456]: osdmap e443: 8 total, 7 up, 6 in 2024-08-17T17:50:19.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:19.965 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:19.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:20.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.107 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:20.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:20.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:20.233 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:20.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:20.335 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:20.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:20.464 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:50:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:50:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:20.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:20.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:20.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:20.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:20.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:20.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:20.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:20 smithi152 ceph-mon[93456]: pgmap v1776: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:50:20.866 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:20 smithi142 ceph-mon[103614]: pgmap v1776: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:50:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:20 smithi142 ceph-mon[106729]: pgmap v1776: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T17:50:20.898 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:21.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:21.034 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:21.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:21.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:21.141 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:50:21.141 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:50:21.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:50:21.141 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:50:21.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.141 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:79060 13.e 13:7e814bd9:::benchmark_data_smithi142_393845_object79059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:48:25.512017+0000", 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 114.934945099, 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23115730300000001, 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 79060 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512017+0000", 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512017+0000", 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512016+0000", 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512065+0000", 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8327e-05 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512074+0000", 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.55e-06 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512079+0000", 2024-08-17T17:50:21.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1189999999999996e-06 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.512982+0000", 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00090253299999999996 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.513094+0000", 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112458 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.514063+0000", 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096906 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.523254+0000", 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091909390000000004 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.529060+0000", 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058053590000000004 2024-08-17T17:50:21.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.743026+0000", 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21396659100000001 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.743134+0000", 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108171 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.743175+0000", 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0355999999999998e-05 2024-08-17T17:50:21.147 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:79088 13.9 13:9e73f15f:::benchmark_data_smithi142_393845_object79087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:48:25.536962+0000", 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 114.910000723, 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20612617, 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 79088 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.536962+0000", 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.536962+0000", 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.536961+0000", 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.536990+0000", 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8404000000000001e-05 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.536997+0000", 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2960000000000001e-06 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537001+0000", 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4449999999999998e-06 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537047+0000", 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6133e-05 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537158+0000", 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110603 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537280+0000", 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012208100000000001 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.540947+0000", 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036672470000000002 2024-08-17T17:50:21.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.541471+0000", 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052403900000000002 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.742972+0000", 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20150082999999999 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.743038+0000", 2024-08-17T17:50:21.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5717999999999997e-05 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.743088+0000", 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9990000000000001e-05 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:79090 13.e 13:7baa8c95:::benchmark_data_smithi142_393845_object79089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:21.154 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:48:25.537315+0000", 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 114.90964752399999, 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20838538500000001, 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 79090 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537315+0000", 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537315+0000", 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537314+0000", 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537395+0000", 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0544000000000007e-05 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537404+0000", 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4939999999999992e-06 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537409+0000", 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1250000000000001e-06 2024-08-17T17:50:21.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.537934+0000", 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052515899999999998 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.538036+0000", 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101646 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.538149+0000", 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113306 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.542989+0000", 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048395690000000002 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.543117+0000", 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012794799999999999 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745571+0000", 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20245396500000001 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745671+0000", 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010030999999999999 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745700+0000", 2024-08-17T17:50:21.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9261999999999999e-05 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:79096 13.9 13:9d49b9fd:::benchmark_data_smithi142_393845_object79095:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:48:25.545960+0000", 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 114.90100208200001, 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.1998095, 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.161 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 79096 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.545960+0000", 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.545960+0000", 2024-08-17T17:50:21.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.545959+0000", 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.546007+0000", 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8458000000000003e-05 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.546020+0000", 2024-08-17T17:50:21.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2524000000000001e-05 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.546026+0000", 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6849999999999998e-06 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.546075+0000", 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9472000000000003e-05 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.546195+0000", 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120035 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.546323+0000", 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012770899999999999 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.548979+0000", 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002655939 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.549374+0000", 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039543400000000001 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745686+0000", 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.19631164600000001 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745738+0000", 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2169999999999997e-05 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745770+0000", 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1948e-05 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:79101 13.e 13:7e17b3a2:::benchmark_data_smithi142_393845_object79100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:48:25.550641+0000", 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 114.896321732, 2024-08-17T17:50:21.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.19519239199999999, 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 79101 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550641+0000", 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550641+0000", 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550640+0000", 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550692+0000", 2024-08-17T17:50:21.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1663999999999999e-05 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550701+0000", 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1509999999999993e-06 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550705+0000", 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9620000000000004e-06 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550743+0000", 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8271e-05 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550828+0000", 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5427999999999996e-05 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:50:21.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.550934+0000", 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010587200000000001 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.553386+0000", 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024513320000000001 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.553571+0000", 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000185649 2024-08-17T17:50:21.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745764+0000", 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.192192526 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745803+0000", 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9360999999999998e-05 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745833+0000", 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0080999999999999e-05 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:79103 13.9 13:9a2be839:::benchmark_data_smithi142_393845_object79102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:48:25.554825+0000", 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 114.892137914, 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.19114695900000001, 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.174 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 79103 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554825+0000", 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554825+0000", 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554824+0000", 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554884+0000", 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0863000000000003e-05 2024-08-17T17:50:21.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554895+0000", 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0547000000000001e-05 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554899+0000", 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4179999999999999e-06 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.554935+0000", 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5601e-05 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.555021+0000", 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5832000000000001e-05 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.555110+0000", 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8979000000000003e-05 2024-08-17T17:50:21.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.557057+0000", 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019470780000000001 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.557513+0000", 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045602400000000002 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745804+0000", 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18829104399999999 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745937+0000", 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013292800000000001 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:48:25.745972+0000", 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4731999999999999e-05 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.180 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:126803 13.9 13:9da4a8c0:::benchmark_data_smithi142_393845_object126802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:13.178234+0000", 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.268728937999995, 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.124271902, 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.181 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 126803 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178234+0000", 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178234+0000", 2024-08-17T17:50:21.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178232+0000", 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178278+0000", 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6149999999999997e-05 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178292+0000", 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3879999999999999e-05 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178298+0000", 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2330000000000003e-06 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178345+0000", 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7071999999999999e-05 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178472+0000", 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012654800000000001 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.178615+0000", 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143545 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.190398+0000", 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011782709000000001 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.190676+0000", 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027795700000000001 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302424+0000", 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11174833200000001 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302471+0000", 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6613999999999998e-05 2024-08-17T17:50:21.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302505+0000", 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4700999999999999e-05 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:126804 13.9 13:9816a159:::benchmark_data_smithi142_393845_object126803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:13.179542+0000", 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.267420627999996, 2024-08-17T17:50:21.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.12303865999999999, 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 126804 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.179542+0000", 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.179542+0000", 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.179541+0000", 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.179573+0000", 2024-08-17T17:50:21.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2512000000000003e-05 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.179586+0000", 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2312e-05 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.179592+0000", 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9490000000000001e-06 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.182771+0000", 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031793540000000001 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.182859+0000", 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8181000000000002e-05 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.182947+0000", 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7573999999999999e-05 2024-08-17T17:50:21.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.190653+0000", 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.007706582 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.190697+0000", 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3856000000000002e-05 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302517+0000", 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11181978200000001 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302553+0000", 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6028999999999997e-05 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302581+0000", 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7582000000000001e-05 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.193 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:126806 13.9 13:97db9104:::benchmark_data_smithi142_393845_object126805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:13.181208+0000", 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.265754713999996, 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.12630566300000001, 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 126806 2024-08-17T17:50:21.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181208+0000", 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181208+0000", 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181207+0000", 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181241+0000", 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4269e-05 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181253+0000", 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2072000000000001e-05 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181259+0000", 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5849999999999999e-06 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.190721+0000", 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094621970000000003 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.190831+0000", 2024-08-17T17:50:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110667 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.191029+0000", 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000197154 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.197917+0000", 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068887560000000002 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.197949+0000", 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1225e-05 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.307437+0000", 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.109488078 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.307477+0000", 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0571000000000002e-05 2024-08-17T17:50:21.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.307513+0000", 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6309000000000001e-05 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:126807 13.9 13:9816b40a:::benchmark_data_smithi142_393845_object126806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:13.181266+0000", 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.265696626999997, 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.126311486, 2024-08-17T17:50:21.200 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 126807 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181266+0000", 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181266+0000", 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181265+0000", 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181291+0000", 2024-08-17T17:50:21.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5185e-05 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181297+0000", 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7889999999999997e-06 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.181301+0000", 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8190000000000002e-06 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.197976+0000", 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016674505999999999 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.198087+0000", 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011160200000000001 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.198235+0000", 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148159 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.298001+0000", 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.099765577999999994 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.298026+0000", 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4946000000000001e-05 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.307520+0000", 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094943159999999992 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.307551+0000", 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1010999999999998e-05 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.307577+0000", 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6106e-05 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.206 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:126815 13.e 13:70bc2e4c:::benchmark_data_smithi142_393845_object126814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:13.192057+0000", 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.254905606999998, 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.110143424, 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 126815 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.192057+0000", 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.192057+0000", 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.192056+0000", 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.192096+0000", 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0290000000000002e-05 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.192106+0000", 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0295000000000001e-05 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.192111+0000", 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1610000000000002e-06 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.196454+0000", 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043423059999999998 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.196569+0000", 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115056 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.196702+0000", 2024-08-17T17:50:21.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013373500000000001 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.200906+0000", 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042031710000000003 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.204348+0000", 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034424780000000001 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302124+0000", 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.097775985999999995 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302168+0000", 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4056e-05 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:13.302200+0000", 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2149e-05 2024-08-17T17:50:21.212 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:153828 13.e 13:7aa50a58:::benchmark_data_smithi142_393845_object26012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.838874+0000", 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.608088791999997, 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.130685466, 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.213 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 153828 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.838874+0000", 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.838874+0000", 2024-08-17T17:50:21.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.838872+0000", 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.838879+0000", 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2890000000000003e-06 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.838891+0000", 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1952e-05 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.838897+0000", 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0669999999999997e-06 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.841871+0000", 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029743109999999999 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.841980+0000", 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010898700000000001 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.842085+0000", 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104604 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.851321+0000", 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092365109999999993 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.851365+0000", 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3464000000000003e-05 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.969265+0000", 2024-08-17T17:50:21.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.117900458 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.969440+0000", 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017504900000000001 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.969559+0000", 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119122 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:153835 13.e 13:75881ccd:::benchmark_data_smithi142_393845_object26019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.844020+0000", 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.602942378999998, 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.12600287099999999, 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.220 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 153835 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844020+0000", 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844020+0000", 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844019+0000", 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844024+0000", 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4410000000000001e-06 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844035+0000", 2024-08-17T17:50:21.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0744000000000001e-05 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844041+0000", 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6529999999999999e-06 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847270+0000", 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003229937 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847364+0000", 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3547000000000004e-05 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847463+0000", 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8492000000000005e-05 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.851510+0000", 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040478700000000003 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.854651+0000", 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031404789999999998 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.969689+0000", 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.115038162 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.969801+0000", 2024-08-17T17:50:21.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111507 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.970023+0000", 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022247099999999999 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:153834 13.c e436/413 13:34ce0f5f:::benchmark_data_smithi142_393845_object26018:head v 436'9715, mlcod=436'9712)", 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.844533+0000", 2024-08-17T17:50:21.226 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.602429626999999, 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.115628098, 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844533+0000", 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844533+0000", 2024-08-17T17:50:21.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844532+0000", 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844539+0000", 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6200000000000001e-06 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844549+0000", 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0417999999999999e-05 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844555+0000", 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2610000000000001e-06 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844619+0000", 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4096000000000004e-05 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.844636+0000", 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7042e-05 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.856091+0000", 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011454944 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960161+0000", 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.104070344 2024-08-17T17:50:21.230 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:153836 13.8 13:1d36a29c:::benchmark_data_smithi142_393845_object26020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.845968+0000", 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.600994812000003, 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.114471831, 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.231 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 153836 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.845968+0000", 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.845968+0000", 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.845966+0000", 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.845972+0000", 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2999999999999998e-06 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.845985+0000", 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2218999999999999e-05 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.845991+0000", 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0870000000000003e-06 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.846039+0000", 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8788000000000001e-05 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.846152+0000", 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112818 2024-08-17T17:50:21.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.846257+0000", 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010479 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960329+0000", 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11407205500000001 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960335+0000", 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8529999999999997e-06 2024-08-17T17:50:21.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960366+0000", 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1581000000000001e-05 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960404+0000", 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7849999999999998e-05 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960440+0000", 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5197000000000002e-05 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:153833 13.f e436/413 13:fab056d8:::benchmark_data_smithi142_393845_object26017:head v 436'9626, mlcod=436'9624)", 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.847952+0000", 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.599010733999997, 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.112669197, 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.237 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847952+0000", 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847952+0000", 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847951+0000", 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847958+0000", 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4599999999999997e-06 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847969+0000", 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.031e-05 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.847974+0000", 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.989e-06 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848145+0000", 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017096 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848162+0000", 2024-08-17T17:50:21.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7322000000000001e-05 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960588+0000", 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.112425893 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960621+0000", 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3043999999999997e-05 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.241 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:153840 13.8 13:1e072f17:::benchmark_data_smithi142_393845_object26024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e436)", 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.848938+0000", 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.598024543000001, 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.111812918, 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 153840 2024-08-17T17:50:21.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848938+0000", 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848938+0000", 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848937+0000", 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848942+0000", 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8200000000000002e-06 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848954+0000", 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1156e-05 2024-08-17T17:50:21.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.848959+0000", 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6799999999999998e-06 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850501+0000", 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001541258 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850603+0000", 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010192900000000001 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850719+0000", 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116728 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960340+0000", 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10962039799999999 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960479+0000", 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000139689 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960669+0000", 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018938200000000001 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960708+0000", 2024-08-17T17:50:21.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8939000000000001e-05 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960751+0000", 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3237e-05 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48005.0:153837 13.4 e436/413 13:2bc797b0:::benchmark_data_smithi142_393845_object26021:head v 436'9594, mlcod=436'9592)", 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:41.850326+0000", 2024-08-17T17:50:21.248 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 38.596636294, 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.10990910199999999, 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850326+0000", 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850326+0000", 2024-08-17T17:50:21.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850326+0000", 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850333+0000", 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1829999999999997e-06 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850342+0000", 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.713e-06 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.850348+0000", 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8019999999999997e-06 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.852363+0000", 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020152070000000002 2024-08-17T17:50:21.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.852381+0000", 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8012999999999998e-05 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960167+0000", 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.107786005 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:41.960235+0000", 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7886999999999996e-05 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:156358 13.8 13:1941a6c6:::benchmark_data_smithi142_393845_object28542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:44.665675+0000", 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 35.781287024999997, 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.225168691, 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.253 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 156358 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665675+0000", 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665675+0000", 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665674+0000", 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665680+0000", 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8900000000000004e-06 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665692+0000", 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.223e-05 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665697+0000", 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4089999999999996e-06 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665740+0000", 2024-08-17T17:50:21.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2777000000000001e-05 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665844+0000", 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104323 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665962+0000", 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117909 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.669458+0000", 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034964359999999999 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.670221+0000", 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00076286699999999997 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.890761+0000", 2024-08-17T17:50:21.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22054037100000001 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.890806+0000", 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4996000000000001e-05 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.890844+0000", 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7693e-05 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.259 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:156360 13.9 13:9d1bfbc5:::benchmark_data_smithi142_393845_object28544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:49:44.665841+0000", 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 35.781121888000001, 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23069682599999999, 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:21.260 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 156360 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665841+0000", 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665841+0000", 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665840+0000", 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665845+0000", 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7489999999999998e-06 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:50:21.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665862+0000", 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7124000000000001e-05 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.665868+0000", 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8390000000000003e-06 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.666424+0000", 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00055688000000000005 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.666554+0000", 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130063 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.666701+0000", 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014667900000000001 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.670601+0000", 2024-08-17T17:50:21.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038995800000000001 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.671888+0000", 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001287659 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.896423+0000", 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.224534554 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.896499+0000", 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6327000000000003e-05 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:49:44.896537+0000", 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8161999999999998e-05 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.266 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:50:21.267 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:50:21.267 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:50:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:21.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:21.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:21.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:21.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:21.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:21.537 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:21.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.671 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:21.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:21.713 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:50:21.713 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:50:21.713 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-08-17T17:50:21.713 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:50:21.713 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:50:21.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-08-17T17:50:21.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:21.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:21.849 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:21 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:50:21.850 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:50:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:50:21] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:50:21.850 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:21 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:50:21.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:21.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:22.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:22.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:21 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:50:22.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:22.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:22.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:22.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:50:22.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:22.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:22.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:22.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:22.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:22.545 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:22.582 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:50:22.583 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:22.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:22.630 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:50:22.631 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:22.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:22.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:22.660 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:50:22.660 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:22.679 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:50:22.679 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:22.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:22 smithi142 ceph-mon[103614]: pgmap v1777: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 867 op/s 2024-08-17T17:50:22.680 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1859657562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:22.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:22 smithi142 ceph-mon[106729]: pgmap v1777: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 867 op/s 2024-08-17T17:50:22.680 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1859657562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:22.705 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:50:22.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:22.740 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:50:22.740 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:22.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:22.774 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:50:22.774 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:22.774 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:50:22.804 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:22.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:22.932 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:23.005 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:22 smithi152 ceph-mon[93456]: pgmap v1777: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 867 op/s 2024-08-17T17:50:23.005 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1859657562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:23.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:23.061 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:23.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:23.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:23.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:23.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:23.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:23.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:23.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:23.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:23.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:23.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.766 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:23.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:23.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:23.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:23.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:50:24.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:24.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.035 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:24.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:24.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:24.181 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:24.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:24.325 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:24.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:24.468 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:24.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:24.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:24.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:24.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:24.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:24.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:24.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:24.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:24.976 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:25.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:25.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:50:25.111 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:50:25.133 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:25.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:25.288 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:25.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:25.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:25.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:25.461 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:25.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:25.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:25.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:25 smithi142 ceph-mon[103614]: pgmap v1778: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 928 op/s 2024-08-17T17:50:25.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:25 smithi142 ceph-mon[106729]: pgmap v1778: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 928 op/s 2024-08-17T17:50:25.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:25.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:25.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:25 smithi152 ceph-mon[93456]: pgmap v1778: 89 pgs: 89 active+clean; 4.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 928 op/s 2024-08-17T17:50:25.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:25.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:25.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:25.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:26.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:26.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:26.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:26.228 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:26.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:26.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:26.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:26.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:26.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:26.581 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:26.603 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:26.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:26.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:26.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:26.833 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:26.863 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:26.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:26.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:26.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:27.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:27.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:50:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.123 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:27.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:27.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:27.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:27.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:27.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:27.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:27.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.539 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:27.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:27.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:27.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:27.722 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:50:27.722 INFO:teuthology.orchestra.run.smithi142.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:50:27.722 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:50:27.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:27 smithi142 ceph-mon[106729]: pgmap v1779: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 918 op/s 2024-08-17T17:50:27.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:27 smithi142 ceph-mon[103614]: pgmap v1779: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 918 op/s 2024-08-17T17:50:27.734 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:27 smithi152 ceph-mon[93456]: pgmap v1779: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 918 op/s 2024-08-17T17:50:27.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:27.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:27.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:27.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:28.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:28.090 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:28.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:28.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:28.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:28.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:28.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:28.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.503 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:28.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:28.565 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:28.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:28.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:28.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:28.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:28.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:28 smithi152 ceph-mon[93456]: pgmap v1780: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 901 op/s 2024-08-17T17:50:28.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:28 smithi142 ceph-mon[106729]: pgmap v1780: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 901 op/s 2024-08-17T17:50:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:28 smithi142 ceph-mon[103614]: pgmap v1780: 89 pgs: 89 active+clean; 3.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 901 op/s 2024-08-17T17:50:28.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:28.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.906 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:28.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:29.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:29.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:29.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:29.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:29.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:29.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:29.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:29.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:29.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:29.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:29.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:29.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:29.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:29.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:29.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:29.962 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:29.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:30.087 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:30.113 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:30.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:30.257 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:30.317 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:30.317 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:30 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:30.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:30.318 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:30 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:30.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:30.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:30 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:30.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:50:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:50:30.219Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:30.342 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:30.342 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:30.359 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:30.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:30.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:30.534 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:30.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:30.672 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:30.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:30.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:30.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:30.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:31.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:31.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:31.135 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:31.143 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:50:31.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:31.206 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:31.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:31.345 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:31.362 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:31 smithi142 ceph-mon[106729]: pgmap v1781: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 919 op/s 2024-08-17T17:50:31.362 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:31 smithi142 ceph-mon[103614]: pgmap v1781: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 919 op/s 2024-08-17T17:50:31.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:31.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:31.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:31 smithi152 ceph-mon[93456]: pgmap v1781: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 919 op/s 2024-08-17T17:50:31.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:31.605 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:31.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:31.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:31.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:31.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:31.952 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:50:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:50:31] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:50:31.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:32.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:32.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:32.154 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:32.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:32.295 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:32.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:32.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:32.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:32.573 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:32.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:32.675 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:32.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:32.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:32.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:32.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:32.940 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:32.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:33.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:33.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:33.203 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:33.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:33.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:33.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:33.370 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:33.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:33.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:33.602 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:50:33.602 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:50:33.603 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 6, 5] 2024-08-17T17:50:33.603 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T17:50:33.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:50:33.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:33.625 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:33.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:33.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:33.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:33.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:33 smithi152 ceph-mon[93456]: pgmap v1782: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:50:33.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:33 smithi152 systemd[1]: Stopping Ceph osd.6 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:50:33.851 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:33 smithi142 ceph-mon[106729]: pgmap v1782: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:50:33.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:33 smithi142 ceph-mon[103614]: pgmap v1782: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:50:33.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:33.929 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:33.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:34.030 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:34.056 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:34.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:34.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:34.188 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:34.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:34.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:34.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:34.408 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:50:34.408 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:50:34.408 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:50:34.408 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:50:34.408 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:34.409 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:34.409 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:34.409 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:34.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:34.430 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:50:34.430 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:34.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:34.463 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:50:34.463 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:34.490 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:50:34.490 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:34.490 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:50:34.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:34.569 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[52686]: 2024-08-17T17:50:34.223+0000 7f9cccf82640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:50:34.570 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[52686]: 2024-08-17T17:50:34.223+0000 7f9cccf82640 -1 osd.6 443 *** Got signal Terminated *** 2024-08-17T17:50:34.570 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:34 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[52686]: 2024-08-17T17:50:34.223+0000 7f9cccf82640 -1 osd.6 443 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:50:34.585 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:34.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:34.687 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:34.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:34.722 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:34 smithi142 ceph-mon[106729]: osd.6 marked itself down and dead 2024-08-17T17:50:34.723 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:34 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3439357562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:34.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:34 smithi142 ceph-mon[103614]: osd.6 marked itself down and dead 2024-08-17T17:50:34.723 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:34 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3439357562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:34.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:34 smithi152 ceph-mon[93456]: osd.6 marked itself down and dead 2024-08-17T17:50:34.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:34 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3439357562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:34.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:34.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:34.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:35.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:35.081 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:35.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:35.205 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:35.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:35.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:35.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:35.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:35.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:35.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:35.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:35.566 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:35 smithi152 podman[278834]: 2024-08-17 17:50:35.264034873 +0000 UTC m=+1.261979607 container died 81327b9b9e787f8318fea4f9ed64601b403fec4ed9e19b351149b6981ce8293e (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:50:35.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:35.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:35 smithi142 ceph-mon[103614]: pgmap v1783: 89 pgs: 89 active+clean; 3.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 924 op/s 2024-08-17T17:50:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:35 smithi142 ceph-mon[103614]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:50:35.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:35 smithi142 ceph-mon[103614]: osdmap e444: 8 total, 6 up, 6 in 2024-08-17T17:50:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:35 smithi142 ceph-mon[106729]: pgmap v1783: 89 pgs: 89 active+clean; 3.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 924 op/s 2024-08-17T17:50:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:35 smithi142 ceph-mon[106729]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:50:35.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:35 smithi142 ceph-mon[106729]: osdmap e444: 8 total, 6 up, 6 in 2024-08-17T17:50:35.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:35.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:35.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:35 smithi152 ceph-mon[93456]: pgmap v1783: 89 pgs: 89 active+clean; 3.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 924 op/s 2024-08-17T17:50:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:35 smithi152 ceph-mon[93456]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:50:35.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:35 smithi152 ceph-mon[93456]: osdmap e444: 8 total, 6 up, 6 in 2024-08-17T17:50:35.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:35 smithi152 podman[278834]: 2024-08-17 17:50:35.672902324 +0000 UTC m=+1.670847062 container cleanup 81327b9b9e787f8318fea4f9ed64601b403fec4ed9e19b351149b6981ce8293e (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:50:35.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:35 smithi152 bash[278834]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6 2024-08-17T17:50:35.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.847 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:35.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:35.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:35.981 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:36.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:36.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:50:36.152 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:50:36.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:36.258 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:36.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:35 smithi152 podman[278902]: 2024-08-17 17:50:35.932992426 +0000 UTC m=+0.615864619 container remove 81327b9b9e787f8318fea4f9ed64601b403fec4ed9e19b351149b6981ce8293e (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4) 2024-08-17T17:50:36.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:36.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:36.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:36.502 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:36.527 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:36.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:36.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:36.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:36.802 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:36.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:36 smithi142 ceph-mon[106729]: osdmap e445: 8 total, 6 up, 6 in 2024-08-17T17:50:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:36 smithi142 ceph-mon[106729]: pgmap v1786: 89 pgs: 13 peering, 10 stale+active+clean, 66 active+clean; 3.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:50:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:36 smithi142 ceph-mon[103614]: osdmap e445: 8 total, 6 up, 6 in 2024-08-17T17:50:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:36 smithi142 ceph-mon[103614]: pgmap v1786: 89 pgs: 13 peering, 10 stale+active+clean, 66 active+clean; 3.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:50:36.899 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:36 smithi152 ceph-mon[93456]: osdmap e445: 8 total, 6 up, 6 in 2024-08-17T17:50:36.899 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:36 smithi152 ceph-mon[93456]: pgmap v1786: 89 pgs: 13 peering, 10 stale+active+clean, 66 active+clean; 3.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T17:50:36.899 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:36 smithi152 podman[279086]: 2024-08-17 17:50:36.624401549 +0000 UTC m=+0.036112369 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:50:36.900 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:36 smithi152 podman[279086]: 2024-08-17 17:50:36.759258559 +0000 UTC m=+0.170969379 container create b9bc922f3f4ffcd069bed4e9b2bd95e7d3b443c6d57c0426f653c6b29acc8226 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.36.0) 2024-08-17T17:50:36.904 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:36.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:37.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:37.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:37.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:37.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:37.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:37.311 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:37 smithi152 podman[279086]: 2024-08-17 17:50:37.03743692 +0000 UTC m=+0.449147740 container init b9bc922f3f4ffcd069bed4e9b2bd95e7d3b443c6d57c0426f653c6b29acc8226 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:50:37.311 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:37 smithi152 podman[279086]: 2024-08-17 17:50:37.048033705 +0000 UTC m=+0.459744519 container start b9bc922f3f4ffcd069bed4e9b2bd95e7d3b443c6d57c0426f653c6b29acc8226 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:50:37.311 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:37 smithi152 podman[279086]: 2024-08-17 17:50:37.137189656 +0000 UTC m=+0.548900472 container attach b9bc922f3f4ffcd069bed4e9b2bd95e7d3b443c6d57c0426f653c6b29acc8226 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:50:37.311 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:50:37 smithi152 podman[279086]: 2024-08-17 17:50:37.311220809 +0000 UTC m=+0.722931627 container died b9bc922f3f4ffcd069bed4e9b2bd95e7d3b443c6d57c0426f653c6b29acc8226 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4) 2024-08-17T17:50:37.332 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:37.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:37.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:37.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:37.568 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:37.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:37.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:37.602 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:50:37.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:37.733 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:37.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:37.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:37.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:37 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:50:37.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/397379936' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:37.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:37 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:50:37.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/397379936' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:37.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:37.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:37 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:50:37.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/397379936' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:37.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:37.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T17:50:38.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.038 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:50:38.038 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T17:50:38.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:38.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:50:38.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:38.262 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:38.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.297 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.398 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:38.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:38.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:38.564 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:38.666 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:38.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:38.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:38.832 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:50:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/397379936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[103614]: osdmap e446: 8 total, 6 up, 6 in 2024-08-17T17:50:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[103614]: pgmap v1788: 89 pgs: 16 active+undersized, 13 peering, 5 stale+active+clean, 5 active+undersized+degraded, 50 active+clean; 3.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 864 op/s; 6414/159228 objects degraded (4.028%) 2024-08-17T17:50:38.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:50:38.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/397379936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:38.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[106729]: osdmap e446: 8 total, 6 up, 6 in 2024-08-17T17:50:38.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:38 smithi142 ceph-mon[106729]: pgmap v1788: 89 pgs: 16 active+undersized, 13 peering, 5 stale+active+clean, 5 active+undersized+degraded, 50 active+clean; 3.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 864 op/s; 6414/159228 objects degraded (4.028%) 2024-08-17T17:50:38.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:38.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:39.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:38 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:50:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/397379936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:38 smithi152 ceph-mon[93456]: osdmap e446: 8 total, 6 up, 6 in 2024-08-17T17:50:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:38 smithi152 ceph-mon[93456]: pgmap v1788: 89 pgs: 16 active+undersized, 13 peering, 5 stale+active+clean, 5 active+undersized+degraded, 50 active+clean; 3.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 864 op/s; 6414/159228 objects degraded (4.028%) 2024-08-17T17:50:39.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:39.205 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:39.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:39.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:39.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:39.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:39.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:39.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:39.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:39.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:39.697 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:39 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 6414/159228 objects degraded (4.028%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:39.697 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:39 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 6414/159228 objects degraded (4.028%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:39.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:39.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:39.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:39.850 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:50:39.850 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:50:39.850 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:50:39.850 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:50:39.850 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:50:39.874 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:39.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:39.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:40.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:40.048 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:40.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:39 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 6414/159228 objects degraded (4.028%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:40.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:40.188 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:40.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:40.322 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:40.424 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:40.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:40.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:40.484 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:50:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:50:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:40.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:50:40.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:40.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:40.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:40.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:40.758 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:40 smithi142 ceph-mon[106729]: pgmap v1789: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 3.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 988 op/s; 21761/150228 objects degraded (14.485%) 2024-08-17T17:50:40.759 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:40 smithi142 ceph-mon[103614]: pgmap v1789: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 3.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 988 op/s; 21761/150228 objects degraded (14.485%) 2024-08-17T17:50:40.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:40 smithi152 ceph-mon[93456]: pgmap v1789: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 3.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 988 op/s; 21761/150228 objects degraded (14.485%) 2024-08-17T17:50:40.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:40.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:40.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:41.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:41.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:41.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:41.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:41.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:41.335 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:41.339 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:41.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:41.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:41.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:50:41.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:41.543 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:41.645 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:41.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.700 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:41.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:41.753 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:50:41.753 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.754 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:41.802 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:41 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:41.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:41.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:41.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:41.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:42.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.107 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:42.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:50:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:50:41] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-17T17:50:42.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:42.237 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:42.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:42.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:42.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:42.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:42.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:42.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:42.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:42.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:42.774 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:42.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:42 smithi142 ceph-mon[106729]: pgmap v1790: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 21021/143247 objects degraded (14.675%) 2024-08-17T17:50:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:42 smithi142 ceph-mon[103614]: pgmap v1790: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 21021/143247 objects degraded (14.675%) 2024-08-17T17:50:42.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:42.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:43.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:43.039 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7, 6] live_osds: [0, 1, 2, 3, 4, 5] 2024-08-17T17:50:43.039 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:50:43.039 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:50:43.039 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.106881 2024-08-17T17:50:43.039 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 0.10688128232773308 2024-08-17T17:50:43.054 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:43.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:42 smithi152 ceph-mon[93456]: pgmap v1790: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s; 21021/143247 objects degraded (14.675%) 2024-08-17T17:50:43.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:43.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:43.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.377 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:43.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:43.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:43.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:43.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:43.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:43.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:43.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:44.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:44.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:44.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:44.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:44.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.091 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:44.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:44.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:44.223 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:44.325 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:44.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:44.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:44.483 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:44.579 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:44.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:44.640 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:44.741 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:44.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:44.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:44 smithi142 ceph-mon[106729]: pgmap v1791: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 20213/137280 objects degraded (14.724%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:44.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:44.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:44 smithi142 ceph-mon[103614]: pgmap v1791: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 20213/137280 objects degraded (14.724%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:44.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:45.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:45.035 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:45.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:44 smithi152 ceph-mon[93456]: pgmap v1791: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 20213/137280 objects degraded (14.724%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:45.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:45.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.174 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:45.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:45.275 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:45.306 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:45.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:45.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:45.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:45.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.578 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:45.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:45.679 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:45.712 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:45.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:45.844 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:45 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 20213/137280 objects degraded (14.724%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:45 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 20213/137280 objects degraded (14.724%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:45.946 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:45.980 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:45.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:46.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:45 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 20213/137280 objects degraded (14.724%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:46.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:46.119 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:46.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:46.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:46.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:46.373 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:46.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:46.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:46.393 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:50:46.393 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:50:46.393 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:176493 12.3 12:c78d0f5f:::benchmark_data_smithi142_261410_object58296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:28.171836+0000", 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 498.21660132, 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.16865802199999999, 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.394 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 176493 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171836+0000", 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.395 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171836+0000", 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171839+0000", 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3410000000000001e-06 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171849+0000", 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0067e-05 2024-08-17T17:50:46.396 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171871+0000", 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2272999999999998e-05 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171886+0000", 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4829e-05 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.397 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.398 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.171965+0000", 2024-08-17T17:50:46.398 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8887000000000006e-05 2024-08-17T17:50:46.398 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.398 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.398 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.398 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172151+0000", 2024-08-17T17:50:46.399 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000186606 2024-08-17T17:50:46.399 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.399 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.399 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.399 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.172338+0000", 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018691700000000001 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.175265+0000", 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002926802 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.179774+0000", 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0045086869999999999 2024-08-17T17:50:46.400 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.336390+0000", 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.15661634799999999 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.336433+0000", 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3053999999999998e-05 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.401 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:28.340495+0000", 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040612110000000003 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230609 12.3 12:c39a7471:::benchmark_data_smithi142_261410_object112412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.063520+0000", 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 266.32491825300002, 2024-08-17T17:50:46.402 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.116453664, 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230609 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.403 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063520+0000", 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063520+0000", 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063521+0000", 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.75e-06 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.404 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063527+0000", 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9359999999999996e-06 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063541+0000", 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4239e-05 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063549+0000", 2024-08-17T17:50:46.405 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9240000000000007e-06 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063690+0000", 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014042899999999999 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063935+0000", 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024514600000000001 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.406 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.064101+0000", 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016561200000000001 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.067394+0000", 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032931660000000001 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.069908+0000", 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025139479999999998 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.177193+0000", 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10728549900000001 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.177249+0000", 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5362000000000003e-05 2024-08-17T17:50:46.408 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.179973+0000", 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027246530000000001 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230612 12.3 12:c19a6bfb:::benchmark_data_smithi142_261410_object112415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.072285+0000", 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 266.31615279900001, 2024-08-17T17:50:46.409 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11038160299999999, 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230612 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.410 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072285+0000", 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072285+0000", 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072286+0000", 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2270000000000001e-06 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.411 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072291+0000", 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0209999999999999e-06 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072302+0000", 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0749e-05 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072307+0000", 2024-08-17T17:50:46.412 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4330000000000003e-06 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072352+0000", 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4472999999999999e-05 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072470+0000", 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117989 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.413 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072577+0000", 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000106969 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.080526+0000", 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079486669999999995 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.414 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.080568+0000", 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2459000000000001e-05 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.180059+0000", 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.099491065000000004 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.180127+0000", 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7873999999999994e-05 2024-08-17T17:50:46.415 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.182667+0000", 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025396770000000002 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230613 12.3 12:ccaeed19:::benchmark_data_smithi142_261410_object112416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.075193+0000", 2024-08-17T17:50:46.416 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 266.31324458300003, 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.111719124, 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230613 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.417 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075193+0000", 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075193+0000", 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.418 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075195+0000", 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.426e-06 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075200+0000", 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7479999999999999e-06 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075215+0000", 2024-08-17T17:50:46.419 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4907e-05 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075224+0000", 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4700000000000002e-06 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075579+0000", 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035526500000000001 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.420 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075756+0000", 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017674499999999999 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.076310+0000", 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055457499999999997 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.080746+0000", 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044359009999999999 2024-08-17T17:50:46.421 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.081342+0000", 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059561900000000003 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.182729+0000", 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.101387585 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.422 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.182790+0000", 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0915999999999997e-05 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.186912+0000", 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041219669999999998 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.423 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230624 12.3 12:c1d2b2f4:::benchmark_data_smithi142_261410_object112427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.089050+0000", 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 266.29938770699999, 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10048264599999999, 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230624 2024-08-17T17:50:46.424 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089050+0000", 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089050+0000", 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.425 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.426 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089051+0000", 2024-08-17T17:50:46.426 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.296e-06 2024-08-17T17:50:46.426 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.426 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.426 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089056+0000", 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4780000000000002e-06 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089067+0000", 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1053000000000001e-05 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.427 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089073+0000", 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5829999999999997e-06 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089123+0000", 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0204000000000002e-05 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089226+0000", 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010301700000000001 2024-08-17T17:50:46.428 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089323+0000", 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6947000000000005e-05 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.092609+0000", 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032867740000000001 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.429 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.093048+0000", 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00043857299999999999 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.186994+0000", 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.093945852999999996 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.187055+0000", 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0851999999999999e-05 2024-08-17T17:50:46.430 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.189533+0000", 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002478016 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230634 12.3 12:ca46e77e:::benchmark_data_smithi142_261410_object112437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.431 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.107856+0000", 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 266.28058189000001, 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.086927434999999997, 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230634 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.432 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.107856+0000", 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.107856+0000", 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.107856+0000", 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6199999999999998e-07 2024-08-17T17:50:46.433 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.107858+0000", 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1260000000000001e-06 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.107865+0000", 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2129999999999996e-06 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.434 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.107870+0000", 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.887e-06 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.112074+0000", 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0042046189999999997 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.112296+0000", 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000221556 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.435 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.112481+0000", 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000185083 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.116220+0000", 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.003738697 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.116234+0000", 2024-08-17T17:50:46.436 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4430000000000001e-05 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.192108+0000", 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.075873736999999997 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.192239+0000", 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013159700000000001 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.437 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.194783+0000", 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025440279999999998 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230961 12.3 12:c7647a8c:::benchmark_data_smithi142_261410_object112764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.687956+0000", 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 265.70048182199997, 2024-08-17T17:50:46.438 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.085181727999999998, 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230961 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.439 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.687956+0000", 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.687956+0000", 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.687957+0000", 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8000000000000004e-07 2024-08-17T17:50:46.440 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.687961+0000", 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.3499999999999999e-06 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.687973+0000", 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.145e-05 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.441 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.687984+0000", 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1090000000000001e-05 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.697726+0000", 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0097418929999999997 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.698098+0000", 2024-08-17T17:50:46.442 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00037254699999999998 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.698265+0000", 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016640599999999999 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.706662+0000", 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0083972019999999994 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.443 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.706691+0000", 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9394e-05 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.770328+0000", 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.063636759000000001 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.770596+0000", 2024-08-17T17:50:46.444 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000267982 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.773138+0000", 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002541775 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:231025 12.e 12:703931bc:::benchmark_data_smithi142_261410_object112828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:50:46.445 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.852874+0000", 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 265.53556426900002, 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.087631868000000002, 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 231025 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.446 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.852874+0000", 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.852874+0000", 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.447 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.852874+0000", 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1200000000000002e-07 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.852889+0000", 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4209999999999999e-05 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.852900+0000", 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1202999999999999e-05 2024-08-17T17:50:46.448 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.852907+0000", 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9959999999999996e-06 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.857016+0000", 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041088770000000004 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.449 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.857201+0000", 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00018527100000000001 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.857397+0000", 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000196353 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.864346+0000", 2024-08-17T17:50:46.450 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0069489959999999998 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.864422+0000", 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5558999999999999e-05 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.937756+0000", 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.073333700000000002 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.451 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.937925+0000", 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000169214 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.940505+0000", 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002580677 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.452 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232569 12.c 12:3177c3d4:::benchmark_data_smithi142_261410_object114372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.956407+0000", 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 262.43203105499998, 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.129661522, 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232569 2024-08-17T17:50:46.453 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956407+0000", 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956407+0000", 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.454 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956408+0000", 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.672e-06 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956414+0000", 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4600000000000002e-06 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.455 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956430+0000", 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6198e-05 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956438+0000", 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956504+0000", 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6289000000000002e-05 2024-08-17T17:50:46.456 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956659+0000", 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015552400000000001 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956839+0000", 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000179607 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.457 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.961488+0000", 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046491960000000004 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969337+0000", 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078488870000000006 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.082814+0000", 2024-08-17T17:50:46.458 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11347713299999999 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.082890+0000", 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5660000000000004e-05 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.086068+0000", 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031784560000000001 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.459 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232573 12.3 12:c9a7ac8e:::benchmark_data_smithi142_261410_object114376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.960989+0000", 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 262.42744901399999, 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.13106198399999999, 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.460 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232573 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960989+0000", 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960989+0000", 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.461 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960990+0000", 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2079999999999999e-06 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960995+0000", 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5299999999999998e-06 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.462 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.961007+0000", 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2485e-05 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.961015+0000", 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5390000000000002e-06 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.964990+0000", 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039751730000000002 2024-08-17T17:50:46.463 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.965151+0000", 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016096400000000001 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.965285+0000", 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013430699999999999 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969150+0000", 2024-08-17T17:50:46.464 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038647500000000001 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969445+0000", 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000295261 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089421+0000", 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11997569 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.465 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089486+0000", 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5208000000000003e-05 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.092051+0000", 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002564869 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.466 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232575 12.c 12:33878e1a:::benchmark_data_smithi142_261410_object114378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.962425+0000", 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 262.42601279000002, 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12697347, 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232575 2024-08-17T17:50:46.467 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962425+0000", 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962425+0000", 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.468 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962426+0000", 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2839999999999999e-06 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962432+0000", 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6930000000000004e-06 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962447+0000", 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4898e-05 2024-08-17T17:50:46.469 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962454+0000", 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4050000000000003e-06 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962520+0000", 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5532999999999998e-05 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.470 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962690+0000", 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016973800000000001 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962869+0000", 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017956800000000001 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969377+0000", 2024-08-17T17:50:46.471 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0065079279999999996 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969582+0000", 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020490600000000001 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.086143+0000", 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.116560921 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.472 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.473 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.473 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.086222+0000", 2024-08-17T17:50:46.473 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8812000000000004e-05 2024-08-17T17:50:46.473 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.473 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089398+0000", 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031767840000000002 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.474 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232582 12.c 12:32c22548:::benchmark_data_smithi142_261410_object114385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.978609+0000", 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 262.40982836799998, 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11411640300000001, 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232582 2024-08-17T17:50:46.475 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978609+0000", 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978609+0000", 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.476 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978611+0000", 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3799999999999999e-06 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978616+0000", 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7090000000000001e-06 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978627+0000", 2024-08-17T17:50:46.477 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1813e-05 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978633+0000", 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0900000000000001e-06 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978684+0000", 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0775000000000001e-05 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.478 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978793+0000", 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109126 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978902+0000", 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010854 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.981735+0000", 2024-08-17T17:50:46.479 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028329660000000001 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.982069+0000", 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000334562 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089453+0000", 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10738336499999999 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.480 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089503+0000", 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9768999999999997e-05 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.092726+0000", 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032233079999999998 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.481 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:234928 12.c 12:30e8dc2c:::benchmark_data_smithi142_261410_object116731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:28.151677+0000", 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 258.23676111399999, 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.099365215000000007, 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 234928 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.482 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.151677+0000", 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.151677+0000", 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.151679+0000", 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.832e-06 2024-08-17T17:50:46.483 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.151686+0000", 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1420000000000004e-06 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.151700+0000", 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4391e-05 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.484 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.151708+0000", 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8839999999999994e-06 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.167675+0000", 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015966985 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.167742+0000", 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6758e-05 2024-08-17T17:50:46.485 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.167830+0000", 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8395999999999999e-05 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.174277+0000", 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0064469660000000002 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.486 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.175560+0000", 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001282987 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.247910+0000", 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.072349714999999995 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.248014+0000", 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000104419 2024-08-17T17:50:46.487 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.251042+0000", 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030277400000000001 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:234936 12.c 12:3e968c93:::benchmark_data_smithi142_261410_object116739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:28.164896+0000", 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 258.22354215199999, 2024-08-17T17:50:46.488 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.170004136, 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 234936 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.489 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164896+0000", 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164896+0000", 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164898+0000", 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7319999999999998e-06 2024-08-17T17:50:46.490 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164907+0000", 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1780000000000003e-06 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164928+0000", 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1078999999999999e-05 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.491 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164938+0000", 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0637e-05 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.170737+0000", 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005798292 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.170824+0000", 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7584999999999996e-05 2024-08-17T17:50:46.492 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.296651+0000", 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.125827044 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.296751+0000", 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9402999999999997e-05 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:50:46.493 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.296980+0000", 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022975400000000001 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.301124+0000", 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041435020000000003 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.314087+0000", 2024-08-17T17:50:46.494 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012963638 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.331705+0000", 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017617287999999998 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.331767+0000", 2024-08-17T17:50:46.495 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2014000000000006e-05 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.334900+0000", 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00313299 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:20110 13.3 13:c7d266d8:::benchmark_data_smithi142_393845_object20109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:47:50.139943+0000", 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 176.24849451399999, 2024-08-17T17:50:46.496 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.088623615000000003, 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 20110 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139943+0000", 2024-08-17T17:50:46.497 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139943+0000", 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139945+0000", 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.578e-06 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.498 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139964+0000", 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9371000000000001e-05 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139985+0000", 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0531e-05 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.139992+0000", 2024-08-17T17:50:46.499 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1999999999999997e-06 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140054+0000", 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1909000000000006e-05 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140177+0000", 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012299500000000001 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.500 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.140311+0000", 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013438100000000001 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.141257+0000", 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00094528899999999998 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.142122+0000", 2024-08-17T17:50:46.501 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00086572999999999999 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.228452+0000", 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.086329937999999995 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.228522+0000", 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9566000000000004e-05 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.502 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:47:50.228567+0000", 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5126999999999997e-05 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:79058 13.0 e425/413 13:074b97a0:::benchmark_data_smithi142_393845_object79057:head v 425'4918, mlcod=425'4916)", 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.512369+0000", 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 140.87606845299999, 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.22531542099999999, 2024-08-17T17:50:46.503 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512369+0000", 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512369+0000", 2024-08-17T17:50:46.504 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512370+0000", 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1399999999999996e-07 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512390+0000", 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9884000000000001e-05 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.505 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512398+0000", 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2779999999999993e-06 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512403+0000", 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.746e-06 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512449+0000", 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5952000000000003e-05 2024-08-17T17:50:46.506 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512462+0000", 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3546999999999999e-05 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737630+0000", 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.22516725400000001 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.507 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737685+0000", 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5046000000000001e-05 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:79063 13.3 13:c3aa0dd5:::benchmark_data_smithi142_393845_object79062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.513029+0000", 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 140.87540925499999, 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.23871411100000001, 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.508 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 79063 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513029+0000", 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.509 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513029+0000", 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513030+0000", 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.234e-06 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.510 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513078+0000", 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8270999999999999e-05 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513089+0000", 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1062e-05 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513096+0000", 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7100000000000001e-06 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.511 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513146+0000", 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0430999999999999e-05 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513269+0000", 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012280200000000001 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.512 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513391+0000", 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121688 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740373+0000", 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.226982397 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740414+0000", 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0717000000000003e-05 2024-08-17T17:50:46.513 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.751637+0000", 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011222891 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.751687+0000", 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0217999999999999e-05 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.514 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.751743+0000", 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5689999999999997e-05 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:79060 13.e e425/413 13:7e814bd9:::benchmark_data_smithi142_393845_object79059:head v 425'4941, mlcod=425'4939)", 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.514256+0000", 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 140.874181601, 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.228641855, 2024-08-17T17:50:46.515 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514256+0000", 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514256+0000", 2024-08-17T17:50:46.516 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514257+0000", 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7400000000000002e-07 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514312+0000", 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5068999999999997e-05 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.517 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737620+0000", 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.223307486 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737636+0000", 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6827999999999999e-05 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.518 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737990+0000", 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035326200000000002 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.738007+0000", 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.766e-05 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.742843+0000", 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048356099999999997 2024-08-17T17:50:46.519 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.742898+0000", 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5065999999999998e-05 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.48005.0:79063 13.3 e425/413 ondisk, result = 0)", 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.516103+0000", 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 140.87233488999999, 2024-08-17T17:50:46.520 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.22427727, 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516103+0000", 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.521 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516103+0000", 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516104+0000", 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8000000000000005e-07 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516107+0000", 2024-08-17T17:50:46.522 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6480000000000001e-06 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737618+0000", 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.221510871 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737636+0000", 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8065000000000001e-05 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.523 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740360+0000", 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027240670000000002 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740368+0000", 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4200000000000001e-06 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740380+0000", 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2418999999999999e-05 2024-08-17T17:50:46.524 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:124293 13.3 13:c0371be0:::benchmark_data_smithi142_393845_object124292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:49:10.428733+0000", 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 95.959704443000007, 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.087472068, 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:50:46.525 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 124293 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.428733+0000", 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:50:46.526 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.428733+0000", 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.428735+0000", 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.516e-06 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.428760+0000", 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.5612000000000001e-05 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.527 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.428775+0000", 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4841000000000001e-05 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.428783+0000", 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5619999999999998e-06 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.433073+0000", 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.004289724 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.528 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.433222+0000", 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000149524 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.433384+0000", 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016176299999999999 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.436294+0000", 2024-08-17T17:50:46.529 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029100509999999999 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.436300+0000", 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8000000000000004e-06 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.516100+0000", 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.079800287999999997 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.516157+0000", 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6826000000000001e-05 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:49:10.516205+0000", 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8560999999999998e-05 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:50:46.531 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:50:46.532 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:50:46.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:46.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:46.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:46.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:46.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:46.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:46.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:46.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:46 smithi142 ceph-mon[106729]: pgmap v1792: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 19079/131559 objects degraded (14.502%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:46.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:46 smithi142 ceph-mon[103614]: pgmap v1792: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 19079/131559 objects degraded (14.502%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:46.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:46.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:46 smithi152 ceph-mon[93456]: pgmap v1792: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 19079/131559 objects degraded (14.502%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:47.073 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:47.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:47.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:47.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:47.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:47.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:47.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:47.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:47.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:47.507 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:47.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:47.644 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:47.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:47.775 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:47.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:47.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:48.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:48.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:48.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:48.184 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:48.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:48.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:48.415 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:48.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:48.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:48.575 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:48.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:48.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:48.812 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:48.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:48.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:48.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:49.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:49.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:49.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:49.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:49.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:49.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:49.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:49.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:49.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.526 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:49.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:49.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:49.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:49.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:49.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:49.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:49 smithi152 ceph-mon[93456]: pgmap v1793: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.12k op/s; 18285/123462 objects degraded (14.810%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:49 smithi142 ceph-mon[106729]: pgmap v1793: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.12k op/s; 18285/123462 objects degraded (14.810%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:49.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:49 smithi142 ceph-mon[103614]: pgmap v1793: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.12k op/s; 18285/123462 objects degraded (14.810%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:49.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:49.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:50.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.059 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:50.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:50.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:50.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:50.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:50.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.324 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:50.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:50.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:50.455 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:50.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:50.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:50:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:50:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.598 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:50.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:50.699 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:50.748 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:50.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:50 smithi142 ceph-mon[106729]: pgmap v1794: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.30k op/s; 16292/112512 objects degraded (14.480%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:50 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16292/112512 objects degraded (14.480%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:50 smithi142 ceph-mon[103614]: pgmap v1794: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.30k op/s; 16292/112512 objects degraded (14.480%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:50.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:50 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16292/112512 objects degraded (14.480%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:50.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:50.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:51.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:50 smithi152 ceph-mon[93456]: pgmap v1794: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.30k op/s; 16292/112512 objects degraded (14.480%); 0 B/s, 0 objects/s recovering 2024-08-17T17:50:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:50 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16292/112512 objects degraded (14.480%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:51.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:51.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:51.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:51.288 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:50:51.288 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:51.297 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:50:51.298 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:51.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:51.329 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:50:51.330 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:51.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.434 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:51.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:51.516 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:50:51.516 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:51.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:51.543 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:50:51.543 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:51.572 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:51.572 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:50:51.572 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:51.574 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:50:51.574 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:50:51.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:51.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:51.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3572684981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:51.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3572684981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:51.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:51.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:51.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:51.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:52.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:52.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3572684981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:52.098 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:50:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:50:51] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-17T17:50:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.107 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:52.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:52.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:52.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.240 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:52.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:52.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:52.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:52.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:52.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:52.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:52.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:52.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:52.648 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:52.682 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:52 smithi142 ceph-mon[103614]: pgmap v1795: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.24k op/s; 15563/105627 objects degraded (14.734%) 2024-08-17T17:50:52.683 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:52 smithi142 ceph-mon[106729]: pgmap v1795: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.24k op/s; 15563/105627 objects degraded (14.734%) 2024-08-17T17:50:52.750 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:52.775 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:52.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:52.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:52.910 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:53.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:53.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:52 smithi152 ceph-mon[93456]: pgmap v1795: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.24k op/s; 15563/105627 objects degraded (14.734%) 2024-08-17T17:50:53.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:53.176 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:53.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:53.312 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:53.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:53.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:53.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:53.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:53.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:50:53.650 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:50:53.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:53.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:53.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:53.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:53.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:53.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2550680245' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:53.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2550680245' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:53.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:54.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:54.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:54.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2550680245' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:54.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:54.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:54.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:50:54.228 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:50:54.246 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:54.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:54.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:54.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:54.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:54.549 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:54.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:54.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:54.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:54.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:54.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2550680245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[106729]: osdmap e447: 8 total, 6 up, 6 in 2024-08-17T17:50:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[106729]: pgmap v1797: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.24k op/s; 14808/99939 objects degraded (14.817%) 2024-08-17T17:50:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2550680245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[103614]: osdmap e447: 8 total, 6 up, 6 in 2024-08-17T17:50:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:54 smithi142 ceph-mon[103614]: pgmap v1797: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.24k op/s; 14808/99939 objects degraded (14.817%) 2024-08-17T17:50:54.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:54.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:55.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:55.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:54 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:55.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2550680245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:55.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:54 smithi152 ceph-mon[93456]: osdmap e447: 8 total, 6 up, 6 in 2024-08-17T17:50:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:54 smithi152 ceph-mon[93456]: pgmap v1797: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.24k op/s; 14808/99939 objects degraded (14.817%) 2024-08-17T17:50:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:55.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:55.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:55.209 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:55.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:55.311 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:55.344 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:55.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:55.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:55.479 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:55.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:55.510 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:55.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:55.612 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:55.714 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:55.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:55.792 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.0 primary-affinity to 0.106881 (870042) 2024-08-17T17:50:55.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:55.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:55.980 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4291820981' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308} : dispatch 2024-08-17T17:50:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:55 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308} : dispatch 2024-08-17T17:50:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:56.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4291820981' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308} : dispatch 2024-08-17T17:50:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:55 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308} : dispatch 2024-08-17T17:50:56.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4291820981' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308} : dispatch 2024-08-17T17:50:56.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:55 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308} : dispatch 2024-08-17T17:50:56.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:56.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:56.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:56.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:50:56.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:56.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:56.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:56.432 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:56.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:56.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:56.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:56.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:56.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:56.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:56.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:56.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:56.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:56.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.964 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:56.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:57.065 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:57.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308}]': finished 2024-08-17T17:50:57.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: osdmap e448: 8 total, 6 up, 6 in 2024-08-17T17:50:57.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 14808/99939 objects degraded (14.817%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:57.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:57.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: pgmap v1799: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 13659/94314 objects degraded (14.482%) 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[106729]: osdmap e449: 8 total, 6 up, 6 in 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308}]': finished 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: osdmap e448: 8 total, 6 up, 6 in 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 14808/99939 objects degraded (14.817%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: pgmap v1799: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 13659/94314 objects degraded (14.482%) 2024-08-17T17:50:57.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:56 smithi142 ceph-mon[103614]: osdmap e449: 8 total, 6 up, 6 in 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.10688128232773308}]': finished 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: osdmap e448: 8 total, 6 up, 6 in 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 14808/99939 objects degraded (14.817%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: pgmap v1799: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 13659/94314 objects degraded (14.482%) 2024-08-17T17:50:57.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:56 smithi152 ceph-mon[93456]: osdmap e449: 8 total, 6 up, 6 in 2024-08-17T17:50:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:57.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:57.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:50:57.198 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:57.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:57.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:57.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:57.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:57.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:57.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:57.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:57.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:57.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:57.788 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:57.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:57.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:58.031 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:58.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:58.171 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:58.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:58.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:58.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:58.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:58.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:58.558 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:58.578 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:58.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:58.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:58.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:58.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:58.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:58.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:59.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:59.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:59.150 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:59.164 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:59.164 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:59.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:59.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:59.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.304 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:59.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:59.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:59.437 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:59.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:59.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:59.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:59.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:59.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:59.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:59.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:59.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:59.846 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:50:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:50:59 smithi142 ceph-mon[106729]: pgmap v1801: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 12989/87123 objects degraded (14.909%) 2024-08-17T17:50:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:50:59 smithi142 ceph-mon[103614]: pgmap v1801: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 12989/87123 objects degraded (14.909%) 2024-08-17T17:50:59.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:59.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:50:59.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:50:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:00.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:50:59 smithi152 ceph-mon[93456]: pgmap v1801: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 12989/87123 objects degraded (14.909%) 2024-08-17T17:51:00.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:00.123 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:00.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:00.224 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:00.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:00.463 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:51:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:51:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:00.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:00.524 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:00.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:00.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:00.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:00.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:00 smithi152 ceph-mon[93456]: pgmap v1802: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.22k op/s; 11385/79341 objects degraded (14.349%) 2024-08-17T17:51:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:00 smithi142 ceph-mon[106729]: pgmap v1802: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.22k op/s; 11385/79341 objects degraded (14.349%) 2024-08-17T17:51:00.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:00 smithi142 ceph-mon[103614]: pgmap v1802: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.22k op/s; 11385/79341 objects degraded (14.349%) 2024-08-17T17:51:00.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:00.922 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:00.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:01.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:01.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:01.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:01.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:01.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:01.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:01.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:01.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:01.489 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7, 6] live_osds: [0, 1, 2, 3, 4, 5] 2024-08-17T17:51:01.489 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:51:01.490 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2024-08-17T17:51:01.490 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2024-08-17T17:51:01.490 DEBUG:teuthology.orchestra.run.smithi152:osd.6> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T17:51:01.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl start ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:51:01.550 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:01.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:01.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:01.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:01.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:01.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 11385/79341 objects degraded (14.349%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:01.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:01 smithi152 systemd[1]: Starting Ceph osd.6 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:51:01.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:01.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 11385/79341 objects degraded (14.349%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:01.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:51:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:51:01] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-17T17:51:01.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 11385/79341 objects degraded (14.349%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:01.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:01.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:01.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:02.080 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:02.106 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:02.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:02.208 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:02.238 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:02.303 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:01 smithi152 podman[280490]: 2024-08-17 17:51:01.846991711 +0000 UTC m=+0.036670135 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:51:02.303 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:01 smithi152 podman[280490]: 2024-08-17 17:51:01.966756384 +0000 UTC m=+0.156434801 container create 88623aaafa7fd1aa9f406a35e2eff2a41bcf5ad0f414093761357ee2e3a9d762 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:51:02.303 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:02 smithi152 podman[280490]: 2024-08-17 17:51:02.302843587 +0000 UTC m=+0.492522000 container init 88623aaafa7fd1aa9f406a35e2eff2a41bcf5ad0f414093761357ee2e3a9d762 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-08-17T17:51:02.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:02.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:02.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:02.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:02.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:02.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:02 smithi152 podman[280490]: 2024-08-17 17:51:02.312376109 +0000 UTC m=+0.502054527 container start 88623aaafa7fd1aa9f406a35e2eff2a41bcf5ad0f414093761357ee2e3a9d762 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-17T17:51:02.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:02.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:02.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:02.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.782 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:02.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:02.881 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:02.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:02.910 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:02.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:02 smithi152 ceph-mon[93456]: pgmap v1803: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.19k op/s; 10603/71328 objects degraded (14.865%) 2024-08-17T17:51:03.012 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:03.018 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:02 smithi142 ceph-mon[106729]: pgmap v1803: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.19k op/s; 10603/71328 objects degraded (14.865%) 2024-08-17T17:51:03.018 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:02 smithi142 ceph-mon[103614]: pgmap v1803: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.19k op/s; 10603/71328 objects degraded (14.865%) 2024-08-17T17:51:03.042 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:03.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:03.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:03.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:51:03.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-08-17T17:51:03.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-08-17T17:51:03.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate[280501]: --> ceph-volume raw activate successful for osd ID: 6 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 podman[280490]: 2024-08-17 17:51:03.062415713 +0000 UTC m=+1.252094124 container attach 88623aaafa7fd1aa9f406a35e2eff2a41bcf5ad0f414093761357ee2e3a9d762 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 bash[280490]: --> ceph-volume raw activate successful for osd ID: 6 2024-08-17T17:51:03.324 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:03 smithi152 podman[280490]: 2024-08-17 17:51:03.063006371 +0000 UTC m=+1.252684786 container died 88623aaafa7fd1aa9f406a35e2eff2a41bcf5ad0f414093761357ee2e3a9d762 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-08-17T17:51:03.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:03.458 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:03.560 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:03.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:03.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:03.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:03.798 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/937475306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:03.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/937475306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:03.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:03.860 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:03.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:03.996 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/937475306' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:04.099 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:04.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:04.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:04.273 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:04.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:04.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:04.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:04.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:04.523 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:04.524 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:04.525 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:04.525 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:51:04.525 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:51:04.525 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:51:04.545 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:04.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:04.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:04.680 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:51:04.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:04.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:04.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:04.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:04.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:04.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1831509569' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:04.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:04 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:04.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:04 smithi152 ceph-mon[93456]: pgmap v1804: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 9832/65706 objects degraded (14.964%) 2024-08-17T17:51:05.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:05.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1831509569' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:04 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:04 smithi142 ceph-mon[103614]: pgmap v1804: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 9832/65706 objects degraded (14.964%) 2024-08-17T17:51:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1831509569' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:04 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:04 smithi142 ceph-mon[106729]: pgmap v1804: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 9832/65706 objects degraded (14.964%) 2024-08-17T17:51:05.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:05.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:05.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:05.329 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:51:05.329 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:51:05.353 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:05.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:05.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:05.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:05 smithi152 podman[280671]: 2024-08-17 17:51:05.308654261 +0000 UTC m=+2.258067961 container remove 88623aaafa7fd1aa9f406a35e2eff2a41bcf5ad0f414093761357ee2e3a9d762 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T17:51:05.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:05.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:05.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:05.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:05.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:05.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:05.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:05.912 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:05 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:05.912 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:05 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:05.912 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:05 smithi152 ceph-mon[93456]: osdmap e450: 8 total, 6 up, 6 in 2024-08-17T17:51:05.913 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:05 smithi152 podman[280818]: 2024-08-17 17:51:05.532523482 +0000 UTC m=+0.031562759 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:51:05.913 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:05 smithi152 podman[280818]: 2024-08-17 17:51:05.643041154 +0000 UTC m=+0.142080411 container create 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4) 2024-08-17T17:51:05.913 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:05 smithi152 podman[280818]: 2024-08-17 17:51:05.912567392 +0000 UTC m=+0.411606673 container init 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:51:05.976 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:06.015 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:06.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:06.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:51:06.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:06.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:05 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:06.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:05 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:06.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:05 smithi142 ceph-mon[106729]: osdmap e450: 8 total, 6 up, 6 in 2024-08-17T17:51:06.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:05 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:06.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:05 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:06.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:05 smithi142 ceph-mon[103614]: osdmap e450: 8 total, 6 up, 6 in 2024-08-17T17:51:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.178 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:06.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:06.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:06.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:06.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:05 smithi152 podman[280818]: 2024-08-17 17:51:05.921274799 +0000 UTC m=+0.420314056 container start 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:51:06.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:06 smithi152 ceph-osd[280869]: -- 172.21.15.152:0/2083917107 <== mon.1 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55894badf340 con 0x55894b967000 2024-08-17T17:51:06.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:06 smithi152 ceph-osd[280869]: -- 172.21.15.152:0/2083917107 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55894b9941a0 con 0x55894b967000 2024-08-17T17:51:06.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:06 smithi152 bash[280818]: 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 2024-08-17T17:51:06.323 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:06 smithi152 systemd[1]: Started Ceph osd.6 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:51:06.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:06.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:06.546 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:51:06.547 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:06.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.596 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:06.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:06.697 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:06.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:06 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 9832/65706 objects degraded (14.964%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:06.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:06 smithi152 ceph-mon[93456]: pgmap v1806: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 8724/61338 objects degraded (14.223%) 2024-08-17T17:51:06.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:51:06.528+0000 7f7279fe4440 -1 Falling back to public interface 2024-08-17T17:51:06.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.861 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:06.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:06.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:07.004 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:07.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:07.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:06 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 9832/65706 objects degraded (14.964%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:07.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:06 smithi142 ceph-mon[106729]: pgmap v1806: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 8724/61338 objects degraded (14.223%) 2024-08-17T17:51:07.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:06 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 9832/65706 objects degraded (14.964%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:07.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:06 smithi142 ceph-mon[103614]: pgmap v1806: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.13k op/s; 8724/61338 objects degraded (14.223%) 2024-08-17T17:51:07.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:07.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.273 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:07.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:07.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:51:07.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.409 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:07.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:07.509 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:07.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:07.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:07.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:07.780 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:07.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:07.909 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:07.935 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:07.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:08.037 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:08.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:08.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:08.201 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:08.302 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:08.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:08.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:08.459 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:08.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:08.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:08.599 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:08.618 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2024-08-17T17:51:08.618 INFO:teuthology.orchestra.run.smithi152.stderr:0 2024-08-17T17:51:08.618 INFO:teuthology.orchestra.run.smithi152.stderr:1 2024-08-17T17:51:08.618 INFO:teuthology.orchestra.run.smithi152.stderr:2 2024-08-17T17:51:08.618 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2024-08-17T17:51:08.618 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2024-08-17T17:51:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:config diff 2024-08-17T17:51:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:config diff get 2024-08-17T17:51:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:config get 2024-08-17T17:51:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:config help [] 2024-08-17T17:51:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:config set ... 2024-08-17T17:51:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2024-08-17T17:51:08.701 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:08.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:08.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:08.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:08.891 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:51:08.891 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:51:08.892 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:51:08.892 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:51:08.892 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:51:08.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:09.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:09.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.155 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:09.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:09.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:09.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:51:09.325 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2024-08-17T17:51:09.393 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:09.424 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:09.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:09.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:09.564 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:09.666 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:09.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:09.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:51:09.790 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:09.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:09.827 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:09 smithi142 ceph-mon[106729]: pgmap v1807: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.1 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 7993/52725 objects degraded (15.160%) 2024-08-17T17:51:09.828 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:09 smithi142 ceph-mon[103614]: pgmap v1807: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.1 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 7993/52725 objects degraded (15.160%) 2024-08-17T17:51:09.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:09.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:10.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.008 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:10.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:10.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:09 smithi152 ceph-mon[93456]: pgmap v1807: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 1.1 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 7993/52725 objects degraded (15.160%) 2024-08-17T17:51:10.109 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:10.143 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:51:10.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:10.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:10.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:10.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:10.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:10.431 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:10.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:10.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:10.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:51:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:10.667 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:10.695 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:10.797 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:10.822 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:10.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:10.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:10.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:11.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:11.100 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:11.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:11.218 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:11.235 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:11.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:11.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:11.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:11.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:11.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:11.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:11.737 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:11.742 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-mon[106729]: pgmap v1808: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 913 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s; 6146/43869 objects degraded (14.010%) 2024-08-17T17:51:11.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6146/43869 objects degraded (14.010%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:11.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:11.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-mon[103614]: pgmap v1808: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 913 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s; 6146/43869 objects degraded (14.010%) 2024-08-17T17:51:11.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6146/43869 objects degraded (14.010%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:11.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:11.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:11.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:11.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:11.998 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:51:11.998 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:51:11.998 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:51:11.998 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:51:11.999 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:11.999 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:11.999 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:11.999 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:11.999 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:51:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:51:11] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:51:12.003 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:51:12.003 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:12.009 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:12.038 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:51:12.038 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:12.041 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:51:12.042 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:51:12.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:11 smithi152 ceph-mon[93456]: pgmap v1808: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 913 MiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.18k op/s; 6146/43869 objects degraded (14.010%) 2024-08-17T17:51:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:11 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6146/43869 objects degraded (14.010%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:12.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:12.190 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:12.292 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:12.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:12.421 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:12.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:12.551 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:12.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2861492497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:12 smithi142 ceph-mon[103614]: pgmap v1809: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 761 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.16k op/s; 5488/36582 objects degraded (15.002%) 2024-08-17T17:51:12.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2861492497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:12.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:12 smithi142 ceph-mon[106729]: pgmap v1809: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 761 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.16k op/s; 5488/36582 objects degraded (15.002%) 2024-08-17T17:51:12.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:12.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:12.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:12.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:12.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:12.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:13.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2861492497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:12 smithi152 ceph-mon[93456]: pgmap v1809: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 761 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.16k op/s; 5488/36582 objects degraded (15.002%) 2024-08-17T17:51:13.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:13.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:13.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:13.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:13.245 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:13.247 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:13.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.387 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:13.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:13.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:13.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:13.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:13.658 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:13.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.794 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:13.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:13.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:13.936 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:14.038 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:14.076 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:14.178 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:51:14.178 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:51:14.179 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:14.218 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:14.320 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:14.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:51:14.347 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:14.448 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:14.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:14.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:14.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:14.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:14.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:14.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:14.793 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:14.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:14.811 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:51:14.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.943 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:14.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:15.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:15.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.075 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:15.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:15.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:15.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:15.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:15.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:15.337 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:15.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:15.439 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:15.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:15.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:15.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:15.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:15.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:15.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:15.839 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:15 smithi142 ceph-mon[106729]: pgmap v1810: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 646 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 4775/31083 objects degraded (15.362%) 2024-08-17T17:51:15.840 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:15 smithi142 ceph-mon[103614]: pgmap v1810: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 646 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 4775/31083 objects degraded (15.362%) 2024-08-17T17:51:15.841 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:51:15.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:15.864 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:15 smithi152 ceph-mon[93456]: pgmap v1810: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 646 MiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.15k op/s; 4775/31083 objects degraded (15.362%) 2024-08-17T17:51:15.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:15.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:15.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:16.015 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:16.117 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:16.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:16.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:16.287 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:16.320 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:51:16.320 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:51:16.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:16.436 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:16.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:16.538 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:16.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:16.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.719 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:16.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:16.820 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:16.843 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:16 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4775/31083 objects degraded (15.362%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:16.844 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:16 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4775/31083 objects degraded (15.362%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:16.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:16.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:16.889 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:16 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4775/31083 objects degraded (15.362%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:16.907 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:16.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:16.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:16.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:0 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:1 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:2 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2024-08-17T17:51:17.029 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore allocator dump block 2024-08-17T17:51:17.030 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore allocator fragmentation block 2024-08-17T17:51:17.030 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2024-08-17T17:51:17.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:17.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:17.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:17.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:17.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:17.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:17.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:17.501 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:17.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:17.599 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:51:17.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:17.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:17.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:17 smithi152 ceph-mon[93456]: pgmap v1811: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 557 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 3708/26766 objects degraded (13.853%) 2024-08-17T17:51:17.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:17 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1314915063' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:17.711 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:51:17.606+0000 7f7279fe4440 -1 osd.6 443 log_to_monitors true 2024-08-17T17:51:17.749 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:51:17.750 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2024-08-17T17:51:17.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:17.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:17.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:17 smithi142 ceph-mon[103614]: pgmap v1811: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 557 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 3708/26766 objects degraded (13.853%) 2024-08-17T17:51:17.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:17 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1314915063' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:17.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:17 smithi142 ceph-mon[106729]: pgmap v1811: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 557 MiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 3708/26766 objects degraded (13.853%) 2024-08-17T17:51:17.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:17 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1314915063' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:17.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.933 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:17.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:18.034 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:18.064 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:18.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:18.166 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:18.198 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:18.300 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:18.321 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:51:18.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:18.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:18.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.468 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:18.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:18.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.602 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:18.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:18.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:18.615 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1314915063' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[103614]: osdmap e451: 8 total, 6 up, 6 in 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[103614]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[103614]: pgmap v1813: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 388 MiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s; 3008/18660 objects degraded (16.120%) 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1314915063' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[106729]: osdmap e451: 8 total, 6 up, 6 in 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[106729]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:51:18.616 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:18 smithi142 ceph-mon[106729]: pgmap v1813: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 388 MiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s; 3008/18660 objects degraded (16.120%) 2024-08-17T17:51:18.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:18.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:18.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:18.838 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.871 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:18.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:18.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:19.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:18 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1314915063' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:51:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:18 smithi152 ceph-mon[93456]: osdmap e451: 8 total, 6 up, 6 in 2024-08-17T17:51:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:18 smithi152 ceph-mon[93456]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:51:19.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:18 smithi152 ceph-mon[93456]: pgmap v1813: 89 pgs: 27 active+undersized, 16 active+undersized+degraded, 46 active+clean; 388 MiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s; 3008/18660 objects degraded (16.120%) 2024-08-17T17:51:19.073 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:51:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:51:18.644+0000 7f726e589640 -1 osd.6 443 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:51:19.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:19.139 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:19.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.272 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:19.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:19.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:19.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:19.515 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:19.527 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:51:19.527 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:51:19.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.554 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:19.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:19.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:19.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:19.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:19.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:19.794 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:19.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:19.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:19 smithi142 ceph-mon[106729]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:51:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:19 smithi142 ceph-mon[106729]: osdmap e452: 8 total, 6 up, 6 in 2024-08-17T17:51:19.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:19 smithi142 ceph-mon[106729]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:51:19.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:19 smithi142 ceph-mon[103614]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:51:19.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:19 smithi142 ceph-mon[103614]: osdmap e452: 8 total, 6 up, 6 in 2024-08-17T17:51:19.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:19 smithi142 ceph-mon[103614]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:51:19.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:19.924 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:19.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:20.026 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:20.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:20.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:19 smithi152 ceph-mon[93456]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:51:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:19 smithi152 ceph-mon[93456]: osdmap e452: 8 total, 6 up, 6 in 2024-08-17T17:51:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:19 smithi152 ceph-mon[93456]: from='osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:51:20.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:20.170 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:20.272 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:20.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:20.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:20.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:20.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:51:20.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:20.541 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:51:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:51:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[106729]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[106729]: osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873] boot 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[106729]: osdmap e453: 8 total, 7 up, 6 in 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[106729]: pgmap v1816: 89 pgs: 8 peering, 23 active+undersized, 12 active+undersized+degraded, 46 active+clean; 240 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 1378/11580 objects degraded (11.900%) 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[103614]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:51:20.641 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[103614]: osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873] boot 2024-08-17T17:51:20.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[103614]: osdmap e453: 8 total, 7 up, 6 in 2024-08-17T17:51:20.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:51:20.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:20 smithi142 ceph-mon[103614]: pgmap v1816: 89 pgs: 8 peering, 23 active+undersized, 12 active+undersized+degraded, 46 active+clean; 240 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 1378/11580 objects degraded (11.900%) 2024-08-17T17:51:20.642 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:20.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:20.770 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:20.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:20.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:20.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:21.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:21.040 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:21.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:51:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:20 smithi152 ceph-mon[93456]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:51:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:20 smithi152 ceph-mon[93456]: osd.6 [v2:172.21.15.152:6816/779491873,v1:172.21.15.152:6817/779491873] boot 2024-08-17T17:51:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:20 smithi152 ceph-mon[93456]: osdmap e453: 8 total, 7 up, 6 in 2024-08-17T17:51:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:20 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:51:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:20 smithi152 ceph-mon[93456]: pgmap v1816: 89 pgs: 8 peering, 23 active+undersized, 12 active+undersized+degraded, 46 active+clean; 240 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 1378/11580 objects degraded (11.900%) 2024-08-17T17:51:21.142 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:21.166 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:21.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:21.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:21.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:21.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:21.514 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:21.540 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:21.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:21.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.680 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:21.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:21.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:21.809 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:21.903 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:21 smithi142 ceph-mon[106729]: osdmap e454: 8 total, 7 up, 6 in 2024-08-17T17:51:21.903 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:21 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 1378/11580 objects degraded (11.900%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:21.903 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:21 smithi142 ceph-mon[103614]: osdmap e454: 8 total, 7 up, 6 in 2024-08-17T17:51:21.903 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:21 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 1378/11580 objects degraded (11.900%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:21.904 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:51:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:51:21] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:51:21.911 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:21.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:22.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:22.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:21 smithi152 ceph-mon[93456]: osdmap e454: 8 total, 7 up, 6 in 2024-08-17T17:51:22.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:21 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 1378/11580 objects degraded (11.900%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:22.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:22.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:22.207 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:22.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:22.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:22.433 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:22.453 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:22.485 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:22.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:22.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:22.728 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:22.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:51:22.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:22.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:22 smithi142 ceph-mon[103614]: osdmap e455: 8 total, 7 up, 6 in 2024-08-17T17:51:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:22 smithi142 ceph-mon[103614]: pgmap v1819: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 peering, 15 active+undersized, 8 active+undersized+degraded, 57 active+clean; 167 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 881 op/s; 4311/8085 objects degraded (53.321%) 2024-08-17T17:51:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:22 smithi142 ceph-mon[106729]: osdmap e455: 8 total, 7 up, 6 in 2024-08-17T17:51:22.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:22 smithi142 ceph-mon[106729]: pgmap v1819: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 peering, 15 active+undersized, 8 active+undersized+degraded, 57 active+clean; 167 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 881 op/s; 4311/8085 objects degraded (53.321%) 2024-08-17T17:51:22.897 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:22.999 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:23.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:22 smithi152 ceph-mon[93456]: osdmap e455: 8 total, 7 up, 6 in 2024-08-17T17:51:23.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:22 smithi152 ceph-mon[93456]: pgmap v1819: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 peering, 15 active+undersized, 8 active+undersized+degraded, 57 active+clean; 167 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 881 op/s; 4311/8085 objects degraded (53.321%) 2024-08-17T17:51:23.124 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:23.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:23.231 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:51:23.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:23.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.307 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:23.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:23.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:23.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:23.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:23.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:23.695 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:23.722 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:23.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:23.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:23.850 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:23.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:23.987 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:24.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:24.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:24.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:24.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:24.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:24.410 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:24.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:24.555 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:24.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2007539839' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:24.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:24.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:24.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:24.692 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 127813 objects 2024-08-17T17:51:24.692 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :130.33 2024-08-17T17:51:24.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:24.810 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:24.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:24.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2007539839' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:24.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2007539839' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:24.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:24.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:24.963 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:51:24.963 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:24.964 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:24.965 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:51:24.965 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:51:24.965 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:24.965 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:24.965 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:51:25.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.056 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:25.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:25.157 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:25.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:25.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:25.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:25.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:25.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:25.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:25.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:25.551 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:25.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:25.674 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:25.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:25.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:25.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:25 smithi152 ceph-mon[93456]: pgmap v1820: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 1 active+undersized+degraded, 76 active+clean; 106 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 762 op/s; 10191/5121 objects degraded (199.004%); 97 KiB/s, 17 objects/s recovering 2024-08-17T17:51:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:25 smithi142 ceph-mon[106729]: pgmap v1820: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 1 active+undersized+degraded, 76 active+clean; 106 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 762 op/s; 10191/5121 objects degraded (199.004%); 97 KiB/s, 17 objects/s recovering 2024-08-17T17:51:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:25 smithi142 ceph-mon[103614]: pgmap v1820: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 1 active+undersized+degraded, 76 active+clean; 106 MiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 762 op/s; 10191/5121 objects degraded (199.004%); 97 KiB/s, 17 objects/s recovering 2024-08-17T17:51:25.899 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:25.921 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:26.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:26.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:26.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:26.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:26.268 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:26.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:26.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:26.414 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:26.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:26.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:26.641 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:26.660 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:26.663 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:26.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:26.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:26.788 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:26.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:26 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 10191/5121 objects degraded (199.004%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:26 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 10191/5121 objects degraded (199.004%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:26.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:26 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 10191/5121 objects degraded (199.004%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:26.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:26.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:26.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:27.013 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:27.035 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:27.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:27.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:27.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:27.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:27.281 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:27.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:27.404 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:27.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:27.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:27.529 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:27.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:27.652 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:27.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:27.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:27.776 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:27.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:27 smithi152 ceph-mon[93456]: pgmap v1821: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 55 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 19476/2679 objects degraded (726.988%); 83 KiB/s, 15 objects/s recovering 2024-08-17T17:51:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:27 smithi142 ceph-mon[106729]: pgmap v1821: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 55 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 19476/2679 objects degraded (726.988%); 83 KiB/s, 15 objects/s recovering 2024-08-17T17:51:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:27 smithi142 ceph-mon[103614]: pgmap v1821: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 55 MiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 19476/2679 objects degraded (726.988%); 83 KiB/s, 15 objects/s recovering 2024-08-17T17:51:27.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:27.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:28.000 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:28.022 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:28.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:28.123 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:28.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:28.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:28.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:28.370 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:28.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:28.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:28.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:28.516 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:28.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:28.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:28.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:28.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:28.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.888 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:28.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:28.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:29.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:29.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:29.134 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:29.235 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:51:29.235 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:51:29.235 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:51:29.236 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:51:29.236 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:29.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:29.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:29.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:29.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:29.505 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:29.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:29.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:29.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:29.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.753 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:29.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:29.854 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:29.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:29 smithi142 ceph-mon[106729]: pgmap v1822: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 469 op/s; 19476/318 objects degraded (6124.528%); 72 KiB/s, 13 objects/s recovering 2024-08-17T17:51:29.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:29 smithi142 ceph-mon[103614]: pgmap v1822: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 469 op/s; 19476/318 objects degraded (6124.528%); 72 KiB/s, 13 objects/s recovering 2024-08-17T17:51:29.875 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:29.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:29.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:30.040 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2024-08-17T17:51:30.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2024-08-17T17:51:30.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:29 smithi152 ceph-mon[93456]: pgmap v1822: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 469 op/s; 19476/318 objects degraded (6124.528%); 72 KiB/s, 13 objects/s recovering 2024-08-17T17:51:30.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:30.122 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:30.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:30.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:30.347 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:30.369 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:30.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:30.379 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:51:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:51:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:30.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:30.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:30.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:30.618 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:30.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:30.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:30.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:30.868 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:30.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:30.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:30.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:31.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:31.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:31.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:31.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:31.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:31.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:31.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:31.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:31.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:31.586 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:31.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:31.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:31.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:31.731 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:31.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:31 smithi152 ceph-mon[93456]: pgmap v1823: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 19476/78 objects degraded (24969.231%); 58 KiB/s, 110 objects/s recovering 2024-08-17T17:51:31.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:31 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 19476/78 objects degraded (24969.231%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:31.832 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:31.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:31.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:51:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:51:31] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.51.0" 2024-08-17T17:51:31.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:31 smithi142 ceph-mon[106729]: pgmap v1823: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 19476/78 objects degraded (24969.231%); 58 KiB/s, 110 objects/s recovering 2024-08-17T17:51:31.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:31 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 19476/78 objects degraded (24969.231%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:31.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:31 smithi142 ceph-mon[103614]: pgmap v1823: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 387 op/s; 19476/78 objects degraded (24969.231%); 58 KiB/s, 110 objects/s recovering 2024-08-17T17:51:31.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:31 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 19476/78 objects degraded (24969.231%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:31.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:31.976 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:31.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:32.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:32.099 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:32.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:32.222 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:32.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:32.346 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:32.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:32.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:32.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:32.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:32.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:32.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:32.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:32.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:32.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:32.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:32.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:32.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:33.064 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:32 smithi152 ceph-mon[93456]: pgmap v1824: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 19476/78 objects degraded (24969.231%); 53 KiB/s, 100 objects/s recovering 2024-08-17T17:51:33.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:32 smithi142 ceph-mon[106729]: pgmap v1824: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 19476/78 objects degraded (24969.231%); 53 KiB/s, 100 objects/s recovering 2024-08-17T17:51:33.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:32 smithi142 ceph-mon[103614]: pgmap v1824: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 351 op/s; 19476/78 objects degraded (24969.231%); 53 KiB/s, 100 objects/s recovering 2024-08-17T17:51:33.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:33.191 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:51:33.192 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:51:33.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:33.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:33.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.334 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:33.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:33.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:33.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:33.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:33.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:33.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:33.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:33.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:33.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:33.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:33.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:33.950 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:33.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:34.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:34.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:34.176 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:34.197 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:34.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:34.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:34.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:34.445 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:34.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:34.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:34.568 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:34.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:34.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:34.692 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:34.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:34.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:34.814 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:34.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:34.916 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:34.939 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:35.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:35.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:35.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:35.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:35.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:51:35.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:35.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:35.411 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:35.433 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:35.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:35.557 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:35.659 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:35.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:35.781 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:35.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:35.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:35 smithi142 ceph-mon[106729]: pgmap v1825: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 19476/78 objects degraded (24969.231%); 48 KiB/s, 216 objects/s recovering 2024-08-17T17:51:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:35 smithi142 ceph-mon[103614]: pgmap v1825: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 19476/78 objects degraded (24969.231%); 48 KiB/s, 216 objects/s recovering 2024-08-17T17:51:35.886 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:51:35.886 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:51:35.886 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-17T17:51:35.886 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:51:35.886 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:51:35.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-17T17:51:35.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:35.952 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:36.054 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:35 smithi152 ceph-mon[93456]: pgmap v1825: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 222 op/s; 19476/78 objects degraded (24969.231%); 48 KiB/s, 216 objects/s recovering 2024-08-17T17:51:36.079 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:36.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:36.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:36.305 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:36.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:36.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:36.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:36.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:36.577 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:36.680 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:36.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:36.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:36.825 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:51:36.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:36.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:36.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:37.052 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:37.073 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:36 smithi142 ceph-mon[106729]: osdmap e456: 8 total, 7 up, 6 in 2024-08-17T17:51:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:36 smithi142 ceph-mon[106729]: pgmap v1827: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 19476/78 objects degraded (24969.231%); 0 B/s, 248 objects/s recovering 2024-08-17T17:51:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2193103313' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:36 smithi142 ceph-mon[103614]: osdmap e456: 8 total, 7 up, 6 in 2024-08-17T17:51:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:36 smithi142 ceph-mon[103614]: pgmap v1827: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 19476/78 objects degraded (24969.231%); 0 B/s, 248 objects/s recovering 2024-08-17T17:51:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2193103313' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:37.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:37.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:37.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:37.298 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:37.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:37.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:36 smithi152 ceph-mon[93456]: osdmap e456: 8 total, 7 up, 6 in 2024-08-17T17:51:37.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:36 smithi152 ceph-mon[93456]: pgmap v1827: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 86 op/s; 19476/78 objects degraded (24969.231%); 0 B/s, 248 objects/s recovering 2024-08-17T17:51:37.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2193103313' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:37.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:37.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:37.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:37.533 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:51:37.533 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:51:37.533 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:51:37.533 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:51:37.533 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:51:37.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:37.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:37.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:37.690 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:37.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:37.813 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:37.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:37.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:37.936 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:38.038 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:38.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:37 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:51:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2193103313' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:37 smithi142 ceph-mon[106729]: osdmap e457: 8 total, 7 up, 6 in 2024-08-17T17:51:38.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:37 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:51:38.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2193103313' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:37 smithi142 ceph-mon[103614]: osdmap e457: 8 total, 7 up, 6 in 2024-08-17T17:51:38.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:38.186 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:38.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:38.309 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:38.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:38.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:37 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:51:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2193103313' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:38.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:37 smithi152 ceph-mon[93456]: osdmap e457: 8 total, 7 up, 6 in 2024-08-17T17:51:38.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:38.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:38.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:38.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:38.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:38.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:38.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:38.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:38.906 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:38.928 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:38.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:38.942 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:38 smithi142 ceph-mon[106729]: pgmap v1829: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 16324/78 objects degraded (20928.205%); 0 B/s, 254 objects/s recovering 2024-08-17T17:51:38.942 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:38 smithi142 ceph-mon[103614]: pgmap v1829: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 16324/78 objects degraded (20928.205%); 0 B/s, 254 objects/s recovering 2024-08-17T17:51:39.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:39.051 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:39.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:39.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:39.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:51:39.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:39.281 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:39.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:39.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:38 smithi152 ceph-mon[93456]: pgmap v1829: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 16324/78 objects degraded (20928.205%); 0 B/s, 254 objects/s recovering 2024-08-17T17:51:39.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:39.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:39.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:39.544 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:39.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:39.668 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:39.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:39.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:39.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:39.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:39.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:39.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:40.017 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:40.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:40.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:39 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 16324/78 objects degraded (20928.205%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:40.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:39 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 16324/78 objects degraded (20928.205%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:40.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:40.161 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:40.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:39 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 16324/78 objects degraded (20928.205%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:51:40.212 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:51:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a[80187]: logger=cleanup t=2024-08-17T17:51:39.870396175Z level=info msg="Completed cleanup jobs" duration=12.661778ms 2024-08-17T17:51:40.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:40.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:40.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:40.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:40.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:40.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:40.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:51:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:51:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:40.634 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:40.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:40.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:51:40.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:40.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:40.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:40.902 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:41.004 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:41.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:41.043 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:40 smithi142 ceph-mon[106729]: pgmap v1830: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 16324/78 objects degraded (20928.205%); 0 B/s, 334 objects/s recovering 2024-08-17T17:51:41.043 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:40 smithi142 ceph-mon[103614]: pgmap v1830: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 16324/78 objects degraded (20928.205%); 0 B/s, 334 objects/s recovering 2024-08-17T17:51:41.128 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:41.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:41.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:41.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:41.273 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:41.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:41.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:40 smithi152 ceph-mon[93456]: pgmap v1830: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 16324/78 objects degraded (20928.205%); 0 B/s, 334 objects/s recovering 2024-08-17T17:51:41.374 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:41.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:41.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:41.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:41.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:41.643 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:41.708 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:41.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:41.750 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:41.751 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:41.770 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:41.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:41.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:41.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:41.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.016 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:42.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:42.027 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:51:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:51:41] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-08-17T17:51:42.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:42.140 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:42.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:42.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:42.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:42.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:42.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:42.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:42.451 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_8' removed 2024-08-17T17:51:42.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:42.519 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:42.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:42.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1255561090' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:51:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:42.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1255561090' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:51:42.642 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:42.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:42.766 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1255561090' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:51:42.868 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:42.889 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:42.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:43.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.014 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:43.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:43.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:43.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:43.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:43.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:43.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.261 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:43.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:43.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:43.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:43.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:43.485 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:43.507 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:43.609 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:43.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:43.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:43.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1255561090' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:51:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:43 smithi152 ceph-mon[93456]: osdmap e458: 8 total, 7 up, 6 in 2024-08-17T17:51:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:43 smithi152 ceph-mon[93456]: pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:43.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1255561090' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:51:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[106729]: osdmap e458: 8 total, 7 up, 6 in 2024-08-17T17:51:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[106729]: pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1255561090' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:51:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[103614]: osdmap e458: 8 total, 7 up, 6 in 2024-08-17T17:51:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[103614]: pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:43.877 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:43.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:43.978 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:44.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:44.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:44.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:44.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:44.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:44.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:44.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:44.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:44.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:44.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:44.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:44.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:44.620 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:44.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:44.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:44 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16324/78 objects degraded (20928.205%), 5 pgs degraded) 2024-08-17T17:51:44.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:44 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:51:44.844 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:44.851 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:44 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16324/78 objects degraded (20928.205%), 5 pgs degraded) 2024-08-17T17:51:44.852 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:44 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:51:44.852 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:44 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16324/78 objects degraded (20928.205%), 5 pgs degraded) 2024-08-17T17:51:44.852 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:44 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:51:44.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:44.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:44.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:44.996 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:44.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:45.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:45.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:45.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:45.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:45.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.245 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:45.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:45.346 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:45.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:45.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:45.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:45.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.618 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:45.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:45.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:45.740 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:45.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:45 smithi152 ceph-mon[93456]: pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:45.842 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:45.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:45 smithi142 ceph-mon[106729]: pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:45 smithi142 ceph-mon[103614]: pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:45.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:45.992 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:46.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:46.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:46.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:46.238 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:46.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:46.340 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:46.366 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:46.468 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:46.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:46.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:46.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:46.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.742 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:46.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:46.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:46.869 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:46.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:46 smithi152 ceph-mon[93456]: pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:46.970 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:46.991 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:47.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:47.119 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:47.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:46 smithi142 ceph-mon[106729]: pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:47.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:46 smithi142 ceph-mon[103614]: pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:47.221 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:47.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:47.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:47.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:47.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:47.502 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:47.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:47.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:47.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:47.757 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:47.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:47.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:47.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:48.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:48.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:48.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:48.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:48.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:48.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:48.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:48.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:48.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:48.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:48.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:48.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:48.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:48.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:48.735 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:48.756 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:48.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:48.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:48.879 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:48.981 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:49.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:49.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:49.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:49.128 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:49.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:49.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:49.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:49 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:49 smithi152 ceph-mon[93456]: pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:49.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:49.359 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:49.359 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:49 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:49.359 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:49 smithi142 ceph-mon[106729]: pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:49.359 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:49.360 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:49 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:49.360 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:49 smithi142 ceph-mon[103614]: pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:49.377 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:49.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:49.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:49.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:49.601 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:49.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:49.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:49.748 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:49.850 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.872 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:49.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:49.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:49.995 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:50.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:50.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:50.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:50.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:50.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:50.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:50.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:50.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:50.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:51:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:51:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:50.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:50.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:50.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:50.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:50.830 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:50 smithi142 ceph-mon[106729]: pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:50.830 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:50 smithi142 ceph-mon[103614]: pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:50.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:50.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:50.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:50.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:51.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:50 smithi152 ceph-mon[93456]: pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:51.103 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:51.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:51.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:51.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:51.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:51.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:51.376 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:51.478 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:51.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:51.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:51.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:51.626 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:51.648 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:51:51.648 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2114400258' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:51.648 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:51 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:51.648 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2114400258' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:51.648 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:51 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:51.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:51.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:51.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:51.762 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:51:51.762 INFO:teuthology.orchestra.run.smithi142.stdout: "success": "" 2024-08-17T17:51:51.762 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:51:51.855 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.878 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:51.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:51.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:52.005 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:52.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:52.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2114400258' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:51 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:52.100 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:51:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:51:51] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-08-17T17:51:52.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:52.129 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:52.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:52.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:52.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:52.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:52.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:52.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:52.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:52.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:52.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:52.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:52.744 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:51:52.745 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:51:52.745 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2024-08-17T17:51:52.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_9 16 2024-08-17T17:51:52.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:52.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:52.882 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:52.928 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:52.928 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[106729]: osdmap e459: 8 total, 7 up, 6 in 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[106729]: pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[103614]: osdmap e459: 8 total, 7 up, 6 in 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:52.929 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:52 smithi142 ceph-mon[103614]: pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:52.936 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:51:52.936 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e391/337 cost 1049576)", 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:42:45.755500+0000", 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 547.17975033499999, 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002456522, 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.937 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755500+0000", 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755500+0000", 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.938 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755501+0000", 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1310000000000001e-06 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755514+0000", 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2458e-05 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755529+0000", 2024-08-17T17:51:52.939 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5741999999999999e-05 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755536+0000", 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.058e-06 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.755592+0000", 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5347999999999999e-05 2024-08-17T17:51:52.940 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:42:45.757957+0000", 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023647849999999999 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e397/337 cost 1049576)", 2024-08-17T17:51:52.941 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:44:04.117282+0000", 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 468.81796876200002, 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002925394, 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117282+0000", 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.942 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117282+0000", 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117283+0000", 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1459999999999999e-06 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.943 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117295+0000", 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1948e-05 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117309+0000", 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4175000000000001e-05 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.944 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117325+0000", 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6030999999999999e-05 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.117399+0000", 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3910000000000002e-05 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:44:04.120207+0000", 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002808184 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.945 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e402/337 cost 1049576)", 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:10.224223+0000", 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 402.711027022, 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030112820000000001, 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224223+0000", 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224223+0000", 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.947 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224224+0000", 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2699999999999998e-07 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224246+0000", 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1263999999999998e-05 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224258+0000", 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2599000000000001e-05 2024-08-17T17:51:52.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224263+0000", 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0819999999999998e-06 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.224317+0000", 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3857000000000003e-05 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:10.227235+0000", 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0029176530000000001 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e404/337 cost 1049576)", 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:33.407392+0000", 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 379.52785807499998, 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0016614139999999999, 2024-08-17T17:51:52.950 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407392+0000", 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407392+0000", 2024-08-17T17:51:52.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407393+0000", 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.091e-06 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407407+0000", 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3905e-05 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407423+0000", 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5889e-05 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407431+0000", 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9750000000000007e-06 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.407494+0000", 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3178999999999999e-05 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.954 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.955 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:33.409054+0000", 2024-08-17T17:51:52.955 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001559375 2024-08-17T17:51:52.955 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.955 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.955 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e405/337 cost 1049576)", 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:50.010496+0000", 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 362.92475490999999, 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024622480000000002, 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.956 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010496+0000", 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010496+0000", 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010497+0000", 2024-08-17T17:51:52.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.437e-06 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010512+0000", 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5327000000000001e-05 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010535+0000", 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2408e-05 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010544+0000", 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5810000000000003e-06 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.010617+0000", 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2704000000000001e-05 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.012958+0000", 2024-08-17T17:51:52.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023407910000000001 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e405/337 cost 1049576)", 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:50.438784+0000", 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 362.49646622799997, 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013036110000000001, 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.960 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438784+0000", 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438784+0000", 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438785+0000", 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0780000000000001e-06 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438800+0000", 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4588e-05 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438811+0000", 2024-08-17T17:51:52.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1331999999999999e-05 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438818+0000", 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1119999999999998e-06 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.438874+0000", 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5789999999999999e-05 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:50.440088+0000", 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0012137109999999999 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e405/337 cost 1049576)", 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:45:51.163905+0000", 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 361.77134560600001, 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001384307, 2024-08-17T17:51:52.964 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.163905+0000", 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.163905+0000", 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.163906+0000", 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.423e-06 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.163919+0000", 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3131e-05 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.163936+0000", 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6937999999999999e-05 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.163944+0000", 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.164003+0000", 2024-08-17T17:51:52.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8906999999999997e-05 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:45:51.165289+0000", 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001286028 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e406/337 cost 1049576)", 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:00.561463+0000", 2024-08-17T17:51:52.968 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 352.37378729800002, 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035765670000000001, 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561463+0000", 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561463+0000", 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561465+0000", 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.483e-06 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561481+0000", 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6796999999999999e-05 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561504+0000", 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2816999999999999e-05 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561512+0000", 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2330000000000006e-06 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.971 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.561597+0000", 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4198999999999998e-05 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:00.565040+0000", 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0034430379999999998 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.972 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:13.150883+0000", 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 339.78436774099998, 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001313645, 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.973 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.150883+0000", 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.150883+0000", 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.150884+0000", 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.779e-06 2024-08-17T17:51:52.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.150902+0000", 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7408e-05 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.150923+0000", 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1491000000000001e-05 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.975 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.150933+0000", 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9960000000000005e-06 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.151022+0000", 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8423000000000004e-05 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.152196+0000", 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0011745480000000001 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:13.723316+0000", 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 339.21193436700003, 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002746592, 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.977 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723316+0000", 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723316+0000", 2024-08-17T17:51:52.978 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723317+0000", 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3710000000000001e-06 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723334+0000", 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6515e-05 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723356+0000", 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1721000000000001e-05 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723364+0000", 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1780000000000003e-06 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.723441+0000", 2024-08-17T17:51:52.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7007000000000003e-05 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:13.726063+0000", 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026218000000000001 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.981 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:16.101865+0000", 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 336.83338545999999, 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0017926800000000001, 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101865+0000", 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101865+0000", 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101866+0000", 2024-08-17T17:51:52.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3319999999999999e-06 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101887+0000", 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0485000000000001e-05 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101909+0000", 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2033000000000001e-05 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101917+0000", 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6759999999999996e-06 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.101993+0000", 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5347999999999998e-05 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.103658+0000", 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001664806 2024-08-17T17:51:52.985 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:16.969004+0000", 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 335.96624599900002, 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0014497729999999999, 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969004+0000", 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969004+0000", 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969006+0000", 2024-08-17T17:51:52.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1090000000000001e-06 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969019+0000", 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3749e-05 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969034+0000", 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5007e-05 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969042+0000", 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4410000000000004e-06 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.969098+0000", 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6369999999999997e-05 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:16.970454+0000", 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013560969999999999 2024-08-17T17:51:52.989 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:17.301491+0000", 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 335.63375923299998, 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0024726290000000001, 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.990 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301491+0000", 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301491+0000", 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301492+0000", 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7300000000000005e-07 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301501+0000", 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1789999999999997e-06 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301516+0000", 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4888e-05 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301523+0000", 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7989999999999996e-06 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.301596+0000", 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3285000000000001e-05 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:17.303964+0000", 2024-08-17T17:51:52.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023676050000000001 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:19.885971+0000", 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 333.04927921000001, 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0040484270000000003, 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.994 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.885971+0000", 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.885971+0000", 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.885973+0000", 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5820000000000001e-06 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.885990+0000", 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7421000000000001e-05 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.886013+0000", 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3247999999999998e-05 2024-08-17T17:51:52.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.886024+0000", 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0737000000000001e-05 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.886118+0000", 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3725000000000001e-05 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:19.890020+0000", 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039017140000000001 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.703573+0000", 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 332.23167746299998, 2024-08-17T17:51:52.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0020549140000000001, 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703573+0000", 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703573+0000", 2024-08-17T17:51:52.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703575+0000", 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8250000000000001e-06 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703593+0000", 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8569000000000001e-05 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703616+0000", 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2838999999999999e-05 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703627+0000", 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0564000000000001e-05 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.703705+0000", 2024-08-17T17:51:53.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8040999999999994e-05 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.705628+0000", 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0019230759999999999 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:53.002 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e407/337 cost 1049576)", 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:21.231709+0000", 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 331.70354140699999, 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001821736, 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231709+0000", 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231709+0000", 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231711+0000", 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0420000000000001e-06 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231725+0000", 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4309e-05 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231747+0000", 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1906e-05 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231757+0000", 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5310000000000007e-06 2024-08-17T17:51:53.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.231842+0000", 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4881000000000003e-05 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:21.233531+0000", 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001689067 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e408/337 cost 1049576)", 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:25.135906+0000", 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 327.79934416100002, 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028730600000000002, 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.135906+0000", 2024-08-17T17:51:53.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.135906+0000", 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.135908+0000", 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.279e-06 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.135923+0000", 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4970000000000001e-05 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.135943+0000", 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.031e-05 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.135951+0000", 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2530000000000004e-06 2024-08-17T17:51:53.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.136032+0000", 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0698000000000006e-05 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:25.138779+0000", 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027475500000000001 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e408/337 cost 1049576)", 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:27.658187+0000", 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 325.27706366500001, 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0013334099999999999, 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.011 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658187+0000", 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658187+0000", 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658188+0000", 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.339e-06 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658203+0000", 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4605e-05 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658223+0000", 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0275000000000002e-05 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:53.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658232+0000", 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9039999999999998e-06 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.658312+0000", 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9991e-05 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:27.659520+0000", 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001208296 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:53.014 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "MOSDPGPull(12.7 e408/337 cost 1049576)", 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:28.424082+0000", 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 324.511168107, 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001438131, 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.424082+0000", 2024-08-17T17:51:53.015 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.424082+0000", 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.424083+0000", 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.8099999999999999e-07 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.424089+0000", 2024-08-17T17:51:53.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9039999999999997e-06 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.424097+0000", 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.3480000000000004e-06 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.424101+0000", 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.9110000000000003e-06 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:51:53.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.425089+0000", 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000988575 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.425520+0000", 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00043111199999999999 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:51:53.018 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:51:53.019 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:53.040 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:52 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:52 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:51:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:52 smithi152 ceph-mon[93456]: osdmap e459: 8 total, 7 up, 6 in 2024-08-17T17:51:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:52 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:52 smithi152 ceph-mon[93456]: pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:53.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:53.168 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:53.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:53.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:53.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:53.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:53.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:53.544 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:53.646 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:53.666 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:53.759 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:51:53.759 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:51:53.759 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:53.759 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:53.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:53.773 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:51:53.774 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:53.789 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:51:53.790 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:53.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:53.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:53.805 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:51:53.806 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:53.821 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:51:53.822 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:53.837 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:51:53.838 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:53.838 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:51:53.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:53.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:53.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:54.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:54.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:54.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:54.170 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:54.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:54.297 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:54.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2189446150' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:54.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2189446150' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:54.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2189446150' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:54.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:54.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:54.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:54.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:54.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:54.676 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:54.778 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:54.800 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:54.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:54.923 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:55.025 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:55.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:55.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:51:55.051 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:51:55.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:55.171 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:55.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:55.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:55.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:55.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:54 smithi152 ceph-mon[93456]: pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:55.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:54 smithi142 ceph-mon[106729]: pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:55.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:54 smithi142 ceph-mon[103614]: pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:55.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:55.425 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:55.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:55.552 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:55.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:55.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:55.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:55.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.804 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:55.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:55.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:55.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:55.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:56.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:56.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:51:56.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:56.055 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:56.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:56.178 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:56.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:56.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:56.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:51:56.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:56.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:56.534 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:56.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:56.657 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:56.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:56.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:56.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:56.803 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:56.905 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:56.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:56.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:57.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:57.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:51:57.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:57.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:57.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:57.177 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:57.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:57.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:57.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:57.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:57.432 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:57.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:57.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:57.569 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:57.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:57.671 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:57.693 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:57.761 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:57.761 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:57.761 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:57 smithi142 ceph-mon[106729]: pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:57.762 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:57.762 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:57.762 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:57 smithi142 ceph-mon[103614]: pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:57.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:57.821 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:51:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:57 smithi152 ceph-mon[93456]: pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:57.923 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:57.945 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:58.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:58.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:58.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:58.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.198 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:58.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:58.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:58.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:58.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:58.448 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:58.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:58.549 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:58.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:58.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:58.694 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:58.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:58.808 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:58 smithi142 ceph-mon[106729]: pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:58.808 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:58 smithi142 ceph-mon[103614]: pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:58.813 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:58.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:58.852 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:58.921 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:58.942 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:59.044 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:59.067 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:59.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:58 smithi152 ceph-mon[93456]: pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:51:59.169 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:59.191 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:59.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:59.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:59.418 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:59.440 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:59.542 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.564 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:51:59.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:59.657 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_9' created 2024-08-17T17:51:59.666 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:59.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:59.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:59.818 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:59.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:59.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:59.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:51:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:59.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3819327667' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-08-17T17:51:59.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:51:59 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-08-17T17:52:00.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3819327667' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-08-17T17:52:00.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:51:59 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-08-17T17:52:00.035 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3819327667' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-08-17T17:52:00.035 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:51:59 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-08-17T17:52:00.047 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.069 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:00.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:00.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:00.197 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:00.299 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:00.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:52:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:52:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:00.343 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2024-08-17T17:52:00.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:00.446 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:00.537 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 0] out_osds: [2, 7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:52:00.537 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:52:00.537 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 3, 4, 5, 6, 0] 2024-08-17T17:52:00.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 0 2024-08-17T17:52:00.548 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:00.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:00.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:00.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:00.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:00 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-08-17T17:52:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:00 smithi152 ceph-mon[93456]: osdmap e460: 8 total, 7 up, 6 in 2024-08-17T17:52:00.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:00 smithi152 ceph-mon[93456]: pgmap v1843: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:00.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:00.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:00 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-08-17T17:52:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:00 smithi142 ceph-mon[106729]: osdmap e460: 8 total, 7 up, 6 in 2024-08-17T17:52:00.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:00 smithi142 ceph-mon[106729]: pgmap v1843: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:00.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:00 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-08-17T17:52:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:00 smithi142 ceph-mon[103614]: osdmap e460: 8 total, 7 up, 6 in 2024-08-17T17:52:00.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:00 smithi142 ceph-mon[103614]: pgmap v1843: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:00.911 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:00.953 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:00.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:01.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:01.097 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:01.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:01.199 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:01.220 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:01.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:01.344 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:01.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:01.440 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:01.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:01.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:01.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:01.592 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:01.694 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:01.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:01.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:01.839 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:01.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:01.952 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:52:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:52:01] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.51.0" 2024-08-17T17:52:01.952 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:01 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:52:01.952 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:01 smithi142 ceph-mon[106729]: osdmap e461: 8 total, 7 up, 6 in 2024-08-17T17:52:01.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:01 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:52:01.953 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:01 smithi142 ceph-mon[103614]: osdmap e461: 8 total, 7 up, 6 in 2024-08-17T17:52:01.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:01.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:02.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:01 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:52:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:01 smithi152 ceph-mon[93456]: osdmap e461: 8 total, 7 up, 6 in 2024-08-17T17:52:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:02.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:02.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:02.208 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:02.310 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:02.332 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:02.434 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:02.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:02.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:02.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:02.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:02.669 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:52:02.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:02.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:02.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:02.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:02.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:02.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:02.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:03.020 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[106729]: osdmap e462: 8 total, 7 up, 6 in 2024-08-17T17:52:03.020 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/828790702' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:03.021 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:03.021 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[106729]: pgmap v1846: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:03.021 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[103614]: osdmap e462: 8 total, 7 up, 6 in 2024-08-17T17:52:03.021 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/828790702' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:03.021 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:03.021 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:02 smithi142 ceph-mon[103614]: pgmap v1846: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:03.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:02 smithi152 ceph-mon[93456]: osdmap e462: 8 total, 7 up, 6 in 2024-08-17T17:52:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:02 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/828790702' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:02 smithi152 ceph-mon[93456]: pgmap v1846: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.077 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:03.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:03.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:03.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:03.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:03.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:03.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:03.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:03.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.576 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:03.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:03.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:03.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:03.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:03.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:03.901 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:03 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:03.901 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:03 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:03.901 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:03 smithi142 ceph-mon[103614]: osdmap e463: 8 total, 7 up, 6 in 2024-08-17T17:52:03.901 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:03 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:03.901 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:03 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:03.901 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:03 smithi142 ceph-mon[106729]: osdmap e463: 8 total, 7 up, 6 in 2024-08-17T17:52:03.924 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:03.946 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:04.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:04.068 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:03 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:03 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:03 smithi152 ceph-mon[93456]: osdmap e463: 8 total, 7 up, 6 in 2024-08-17T17:52:04.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:04.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:04.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:04.295 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:04.317 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:04.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:04.442 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:04.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:04.567 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:04.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:04.694 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:04.790 INFO:teuthology.orchestra.run.smithi142.stderr:enabled application 'rados' on pool 'unique_pool_9' 2024-08-17T17:52:04.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:04.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:04.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:04.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:04.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:05.022 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:05.041 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:05.067 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:05.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:04 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3393636096' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:52:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:04 smithi152 ceph-mon[93456]: pgmap v1848: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:04 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3393636096' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:52:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:04 smithi142 ceph-mon[106729]: pgmap v1848: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:04 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3393636096' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:52:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:04 smithi142 ceph-mon[103614]: pgmap v1848: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:05.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:05.194 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:05.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:05.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:05.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:05.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:05.423 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:05.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:05.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:05.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:05.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:05.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:05.694 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:05.796 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:05.818 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:05.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:05.946 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:06.016 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3393636096' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:52:06.016 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:05 smithi142 ceph-mon[106729]: osdmap e464: 8 total, 7 up, 6 in 2024-08-17T17:52:06.017 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3393636096' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:52:06.017 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:05 smithi142 ceph-mon[103614]: osdmap e464: 8 total, 7 up, 6 in 2024-08-17T17:52:06.048 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:06.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3393636096' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:52:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:05 smithi152 ceph-mon[93456]: osdmap e464: 8 total, 7 up, 6 in 2024-08-17T17:52:06.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:06.193 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:06.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:06.295 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:06.316 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:06.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:06.444 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:06.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:06.571 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:06.656 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:52:06.673 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:06.713 DEBUG:teuthology.orchestra.run.smithi142:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2024-08-17T17:52:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.720 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:06.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:06.721 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:52:06.721 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:52:06.767 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:hints = 1 2024-08-17T17:52:06.767 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:52:06.767 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object prefix: benchmark_data_smithi142_428506 2024-08-17T17:52:06.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:52:06.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:52:06.821 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:06.865 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:06 smithi142 ceph-mon[106729]: pgmap v1850: 89 pgs: 89 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:06.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:06 smithi142 ceph-mon[103614]: pgmap v1850: 89 pgs: 89 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:06.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:07.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:07.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:06 smithi152 ceph-mon[93456]: pgmap v1850: 89 pgs: 89 active+clean; 582 KiB data, 7.1 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:52:07.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:07.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:07.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:07.296 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:07.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:07.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:07.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.438 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:07.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:07.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:07.568 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:07.670 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:07.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:07.767 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 1 16 3818 3802 237.618 237.625 0.0039354 0.00419347 2024-08-17T17:52:07.812 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:07.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:07.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:07 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:52:07.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:07 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:52:07.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:07.979 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:08.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:07 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:52:08.081 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:08.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:08.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:08.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:08.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:08.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:08.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:08.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:08.523 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:08.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:08.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:08.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:08.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:08.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:08.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 2 16 7353 7337 229.214 220.938 0.0083966 0.00435211 2024-08-17T17:52:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:08.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:08.860 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:08 smithi142 ceph-mon[106729]: pgmap v1851: 89 pgs: 89 active+clean; 120 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 17 MiB/s wr, 277 op/s 2024-08-17T17:52:08.861 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:08 smithi142 ceph-mon[103614]: pgmap v1851: 89 pgs: 89 active+clean; 120 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 17 MiB/s wr, 277 op/s 2024-08-17T17:52:08.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:08.929 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:09.003 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:09.003 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:09.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:09.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:09.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:08 smithi152 ceph-mon[93456]: pgmap v1851: 89 pgs: 89 active+clean; 120 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 17 MiB/s wr, 277 op/s 2024-08-17T17:52:09.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:09.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:09.303 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:09.338 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:09.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:09.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:09.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:09.575 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:09.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:09.715 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:09.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:09.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 3 16 10675 10659 222.013 207.625 0.00785708 0.00449747 2024-08-17T17:52:09.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:09.887 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:09.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:10.027 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:10.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:10.157 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:10.249 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:52:10.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:10.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:10.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:10.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:10.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.428 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:10.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:10.528 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:52:10.528 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:10.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:10.536 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:52:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:52:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:10.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:10.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:10.570 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:52:10.570 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:10.605 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:52:10.605 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:10.640 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:52:10.640 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:10.670 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:10.674 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:52:10.674 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:10.705 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:52:10.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.709 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:10.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:10.735 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:52:10.736 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:10.736 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2762733290' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:10.736 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2762733290' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:10.737 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:52:10.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 4 16 13564 13548 211.647 180.562 0.00489449 0.00471792 2024-08-17T17:52:10.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2762733290' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:10.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:10.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:10.947 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:10.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:10.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:11.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:11.121 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:11.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:11.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:11.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:11.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:11.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:11.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:11.524 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:11.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:11.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:11.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:11.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 5 16 16510 16494 206.14 184.125 0.00590605 0.00484531 2024-08-17T17:52:11.795 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:11 smithi152 ceph-mon[93456]: pgmap v1852: 89 pgs: 89 active+clean; 434 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2024-08-17T17:52:11.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:11.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:11 smithi142 ceph-mon[103614]: pgmap v1852: 89 pgs: 89 active+clean; 434 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2024-08-17T17:52:11.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:11.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:52:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:52:11] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.51.0" 2024-08-17T17:52:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:11 smithi142 ceph-mon[106729]: pgmap v1852: 89 pgs: 89 active+clean; 434 MiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2024-08-17T17:52:11.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:11.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:11.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:12.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:12.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:12.181 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:12.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:12.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:12.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:12.343 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:12.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:12.474 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:12.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:12.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:12.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:12.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:12.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:12.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 6 16 18928 18912 196.969 151.125 0.00384709 0.00506998 2024-08-17T17:52:12.843 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:12.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:52:12.566+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e464 definitely_dead 0 2024-08-17T17:52:12.880 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:12.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:12.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:13.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:13.017 INFO:teuthology.orchestra.run.smithi142.stderr:marked out osd.0. 2024-08-17T17:52:13.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:13.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:13.237 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:13.259 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:13.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:13 smithi152 ceph-mon[93456]: pgmap v1853: 89 pgs: 89 active+clean; 619 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-17T17:52:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1236865494' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:52:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:13 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:52:13.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:13 smithi152 ceph-mon[93456]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:52:13.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[103614]: pgmap v1853: 89 pgs: 89 active+clean; 619 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-17T17:52:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1236865494' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:52:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:52:13.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[103614]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:52:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[106729]: pgmap v1853: 89 pgs: 89 active+clean; 619 MiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-17T17:52:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1236865494' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:52:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:52:13.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:13 smithi142 ceph-mon[106729]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:52:13.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:13.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:13.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:13.506 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:13.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:13.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:13.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:13.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:13.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:13.758 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:13.768 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 7 16 19464 19448 173.616 33.5 0.00172985 0.00514531 2024-08-17T17:52:13.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:13.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:13.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.015 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:14.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:14.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:14.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:14.267 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:14 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:52:14.267 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:14 smithi142 ceph-mon[106729]: osdmap e465: 8 total, 7 up, 5 in 2024-08-17T17:52:14.267 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:14 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:52:14.267 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:14 smithi142 ceph-mon[103614]: osdmap e465: 8 total, 7 up, 5 in 2024-08-17T17:52:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:52:14.285 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:52:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:14 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:52:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:14 smithi152 ceph-mon[93456]: osdmap e465: 8 total, 7 up, 5 in 2024-08-17T17:52:14.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:14.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:14.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.516 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:14.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:14.617 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:14.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:14.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:14.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 8 16 19464 19448 151.915 0 - 0.00514531 2024-08-17T17:52:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.771 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:14.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:14.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.895 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:14.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:14.983 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:14.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:14.997 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:15.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:15.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:15.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:15.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:15.290 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:15 smithi142 ceph-mon[103614]: osdmap e466: 8 total, 7 up, 5 in 2024-08-17T17:52:15.290 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:15 smithi142 ceph-mon[103614]: pgmap v1856: 89 pgs: 3 remapped+peering, 12 peering, 74 active+clean; 1.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-08-17T17:52:15.291 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:15 smithi142 ceph-mon[103614]: osdmap e467: 8 total, 7 up, 5 in 2024-08-17T17:52:15.291 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:15 smithi142 ceph-mon[106729]: osdmap e466: 8 total, 7 up, 5 in 2024-08-17T17:52:15.292 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:15 smithi142 ceph-mon[106729]: pgmap v1856: 89 pgs: 3 remapped+peering, 12 peering, 74 active+clean; 1.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-08-17T17:52:15.292 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:15 smithi142 ceph-mon[106729]: osdmap e467: 8 total, 7 up, 5 in 2024-08-17T17:52:15.306 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:15.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:15 smithi152 ceph-mon[93456]: osdmap e466: 8 total, 7 up, 5 in 2024-08-17T17:52:15.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:15 smithi152 ceph-mon[93456]: pgmap v1856: 89 pgs: 3 remapped+peering, 12 peering, 74 active+clean; 1.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-08-17T17:52:15.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:15 smithi152 ceph-mon[93456]: osdmap e467: 8 total, 7 up, 5 in 2024-08-17T17:52:15.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:15.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:15.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:15.580 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:15.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:15.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:15.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:15.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 9 16 20833 20817 144.542 42.7812 0.00461212 0.00691201 2024-08-17T17:52:15.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:15.862 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:15.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:15.964 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:15.997 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:16.081 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:16 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:52:16.081 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:16 smithi142 ceph-mon[106729]: osdmap e468: 8 total, 7 up, 5 in 2024-08-17T17:52:16.082 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:16 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:52:16.082 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:16 smithi142 ceph-mon[103614]: osdmap e468: 8 total, 7 up, 5 in 2024-08-17T17:52:16.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:16.136 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:16.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:16.274 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:16.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:52:16.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:16 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:52:16.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:16 smithi152 ceph-mon[93456]: osdmap e468: 8 total, 7 up, 5 in 2024-08-17T17:52:16.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:16.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:16.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.555 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:16.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:16.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:16.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.691 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:16.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:16.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 10 16 22886 22870 142.918 128.312 0.00871391 0.00699042 2024-08-17T17:52:16.792 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:16.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:16.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:16.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:17.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:17.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:17.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:17.252 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:17.320 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:17 smithi142 ceph-mon[106729]: pgmap v1859: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 12 peering, 72 active+clean; 1.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 147 MiB/s wr, 2.35k op/s; 1231/57939 objects degraded (2.125%); 1168/57939 objects misplaced (2.016%); 304 KiB/s, 4 objects/s recovering 2024-08-17T17:52:17.320 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:17 smithi142 ceph-mon[106729]: osdmap e469: 8 total, 7 up, 5 in 2024-08-17T17:52:17.320 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:17 smithi142 ceph-mon[103614]: pgmap v1859: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 12 peering, 72 active+clean; 1.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 147 MiB/s wr, 2.35k op/s; 1231/57939 objects degraded (2.125%); 1168/57939 objects misplaced (2.016%); 304 KiB/s, 4 objects/s recovering 2024-08-17T17:52:17.320 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:17 smithi142 ceph-mon[103614]: osdmap e469: 8 total, 7 up, 5 in 2024-08-17T17:52:17.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:17 smithi152 ceph-mon[93456]: pgmap v1859: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 12 peering, 72 active+clean; 1.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 147 MiB/s wr, 2.35k op/s; 1231/57939 objects degraded (2.125%); 1168/57939 objects misplaced (2.016%); 304 KiB/s, 4 objects/s recovering 2024-08-17T17:52:17.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:17 smithi152 ceph-mon[93456]: osdmap e469: 8 total, 7 up, 5 in 2024-08-17T17:52:17.354 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:17.386 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:17.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:17.518 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:17.620 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:17.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:17.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:17.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 11 16 24886 24870 141.288 125 0.010803 0.00706987 2024-08-17T17:52:17.794 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:17.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:17.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:18.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:18.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:18.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:18.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:18.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:18.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:18.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:18 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 1231/57939 objects degraded (2.125%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:52:18.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:18.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:18.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:18 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 1231/57939 objects degraded (2.125%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:52:18.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:18 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 1231/57939 objects degraded (2.125%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:52:18.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:18.475 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:18.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:18.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:18.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:18.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:18.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:18.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 12 16 26831 26815 139.644 121.562 0.00332553 0.00715523 2024-08-17T17:52:18.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:18.882 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:18.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:18.984 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:19.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:19.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:19.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:19.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:19.294 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:19.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:19 smithi152 ceph-mon[93456]: pgmap v1861: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 peering, 76 active+clean; 1.3 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s; 4930/66078 objects degraded (7.461%); 1168/66078 objects misplaced (1.768%); 269 KiB/s, 2 keys/s, 6 objects/s recovering 2024-08-17T17:52:19.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:19 smithi142 ceph-mon[106729]: pgmap v1861: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 peering, 76 active+clean; 1.3 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s; 4930/66078 objects degraded (7.461%); 1168/66078 objects misplaced (1.768%); 269 KiB/s, 2 keys/s, 6 objects/s recovering 2024-08-17T17:52:19.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:19 smithi142 ceph-mon[103614]: pgmap v1861: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 5 peering, 76 active+clean; 1.3 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s; 4930/66078 objects degraded (7.461%); 1168/66078 objects misplaced (1.768%); 269 KiB/s, 2 keys/s, 6 objects/s recovering 2024-08-17T17:52:19.396 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:19.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:19.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:19.533 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:19.563 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:19.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:19.704 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:19.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 13 16 28596 28580 137.387 110.312 0.00524839 0.00727299 2024-08-17T17:52:19.805 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:19.833 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:19.935 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:19.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:19.985 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6] out_osds: [2, 7, 0] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:52:19.985 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:52:19.986 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-17T17:52:19.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-17T17:52:20.069 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:20.108 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:20.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:20.243 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:20.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:20 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:52:20.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:52:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:52:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:20.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.368 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:20.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:20.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:20 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:52:20.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:20 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:52:20.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:20.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:20.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:20.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:20.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:20.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:20.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 14 16 30254 30238 134.975 103.625 0.00860787 0.0074029 2024-08-17T17:52:20.789 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:20.891 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:20.925 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:21.027 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:21.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:21.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:21.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:21.207 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:21.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:21.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:21 smithi152 ceph-mon[93456]: pgmap v1862: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s; 7434/79956 objects degraded (9.298%); 1482/79956 objects misplaced (1.854%); 9.3 MiB/s, 1 keys/s, 150 objects/s recovering 2024-08-17T17:52:21.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:21.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:21 smithi142 ceph-mon[106729]: pgmap v1862: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s; 7434/79956 objects degraded (9.298%); 1482/79956 objects misplaced (1.854%); 9.3 MiB/s, 1 keys/s, 150 objects/s recovering 2024-08-17T17:52:21.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:21 smithi142 ceph-mon[103614]: pgmap v1862: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s; 7434/79956 objects degraded (9.298%); 1482/79956 objects misplaced (1.854%); 9.3 MiB/s, 1 keys/s, 150 objects/s recovering 2024-08-17T17:52:21.432 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:21.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:21.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:21.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.619 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:21.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:21.720 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:21.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:21.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 15 16 31740 31724 132.167 92.875 0.0141608 0.00756027 2024-08-17T17:52:21.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:21.900 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:22.002 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:22.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:22.072 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:52:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:52:21] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.51.0" 2024-08-17T17:52:22.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:22.150 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:22.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:22.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.287 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:22.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:22.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:22.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:22.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:22.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:22.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:22.565 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:52:22.566 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:52:22.566 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:52:22.566 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:52:22.566 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:52:22.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:22.694 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:22.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 16 16 33045 33029 129.004 81.5625 0.0154825 0.00774527 2024-08-17T17:52:22.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:22.832 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:22 smithi142 ceph-mon[103614]: pgmap v1863: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 7434/84597 objects degraded (8.788%); 1482/84597 objects misplaced (1.752%); 7.4 MiB/s, 1 keys/s, 119 objects/s recovering 2024-08-17T17:52:22.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:22 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 7434/84597 objects degraded (8.788%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:22.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:22 smithi142 ceph-mon[106729]: pgmap v1863: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 7434/84597 objects degraded (8.788%); 1482/84597 objects misplaced (1.752%); 7.4 MiB/s, 1 keys/s, 119 objects/s recovering 2024-08-17T17:52:22.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:22 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 7434/84597 objects degraded (8.788%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:22.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:22.974 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:22.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:22 smithi152 ceph-mon[93456]: pgmap v1863: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 7434/84597 objects degraded (8.788%); 1482/84597 objects misplaced (1.752%); 7.4 MiB/s, 1 keys/s, 119 objects/s recovering 2024-08-17T17:52:23.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:22 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 7434/84597 objects degraded (8.788%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:23.076 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:23.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:23.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:23.246 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:23.348 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:23.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:52:23.380 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:23.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:23.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:23.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:23.662 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:23.764 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:23.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 17 16 34835 34819 127.996 111.875 0.00471832 0.0078067 2024-08-17T17:52:23.806 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:23.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:23.907 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:23.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:24.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:24.086 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:24.188 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:24.219 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:24.233 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:52:24.253 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:24.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:24.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:24.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:24.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:24.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:24.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1239317050' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:24 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:24.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:24 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:24.602 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1239317050' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:24 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:24.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:24 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:24.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1239317050' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:24.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:24 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:24.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:24 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:24.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:24.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:24.769 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 18 16 36463 36447 126.537 101.75 0.00701007 0.00789754 2024-08-17T17:52:24.779 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:24.881 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:24.916 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:24.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:25.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:25.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:25.154 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:25.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:25.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:25.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[106729]: osdmap e470: 8 total, 7 up, 5 in 2024-08-17T17:52:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:25.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[106729]: pgmap v1865: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 113 MiB/s wr, 1.81k op/s; 7434/101301 objects degraded (7.339%); 818/101301 objects misplaced (0.807%); 12 MiB/s, 1 keys/s, 193 objects/s recovering 2024-08-17T17:52:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[103614]: osdmap e470: 8 total, 7 up, 5 in 2024-08-17T17:52:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:25.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:25 smithi142 ceph-mon[103614]: pgmap v1865: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 113 MiB/s wr, 1.81k op/s; 7434/101301 objects degraded (7.339%); 818/101301 objects misplaced (0.807%); 12 MiB/s, 1 keys/s, 193 objects/s recovering 2024-08-17T17:52:25.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:25.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:25.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:25.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:25 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:25 smithi152 ceph-mon[93456]: osdmap e470: 8 total, 7 up, 5 in 2024-08-17T17:52:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:25.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:25 smithi152 ceph-mon[93456]: pgmap v1865: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 113 MiB/s wr, 1.81k op/s; 7434/101301 objects degraded (7.339%); 818/101301 objects misplaced (0.807%); 12 MiB/s, 1 keys/s, 193 objects/s recovering 2024-08-17T17:52:25.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:25.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:25.714 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:25.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 19 16 37612 37596 123.657 71.8125 0.0040098 0.00792857 2024-08-17T17:52:25.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:25.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:25.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:25.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:25.981 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:26.083 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:26.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:26.207 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:26.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:26.330 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:26.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:26.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:26.475 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:26.535 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[106729]: osdmap e471: 8 total, 7 up, 5 in 2024-08-17T17:52:26.535 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:52:26.535 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[106729]: osdmap e472: 8 total, 7 up, 5 in 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[103614]: osdmap e471: 8 total, 7 up, 5 in 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:26.536 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:26 smithi142 ceph-mon[103614]: osdmap e472: 8 total, 7 up, 5 in 2024-08-17T17:52:26.541 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:26.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:26.600 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:26.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:26.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.725 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:26.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:52:26.770193+0000 min lat: 0.00102385 max lat: 2.12417 avg lat: 0.00811699 2024-08-17T17:52:26.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:52:26.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 20 16 37916 37900 118.424 19 0.00445819 0.00811699 2024-08-17T17:52:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:26 smithi152 ceph-mon[93456]: osdmap e471: 8 total, 7 up, 5 in 2024-08-17T17:52:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:52:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:52:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:26 smithi152 ceph-mon[93456]: osdmap e472: 8 total, 7 up, 5 in 2024-08-17T17:52:26.825 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:26.855 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:26.957 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:26.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:27.093 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:27.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:27.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:27.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:52:27.230 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:27.265 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:27.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:27.367 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:27.407 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:27.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:27.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:27.580 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:52:27.580 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:27.608 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:52:27.609 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:27.633 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:52:27.633 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:27.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:27.658 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:52:27.659 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:27.687 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:52:27.687 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:27.687 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:27.687 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:27.687 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[106729]: pgmap v1868: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 84 MiB/s wr, 1.34k op/s; 7434/104088 objects degraded (7.142%); 234/104088 objects misplaced (0.225%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[106729]: osdmap e473: 8 total, 7 up, 5 in 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[103614]: pgmap v1868: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 84 MiB/s wr, 1.34k op/s; 7434/104088 objects degraded (7.142%); 234/104088 objects misplaced (0.225%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[103614]: osdmap e473: 8 total, 7 up, 5 in 2024-08-17T17:52:27.688 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.690 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:27.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:27.721 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:52:27.721 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:27.750 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:52:27.750 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:27.752 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:52:27.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 21 16 39432 39416 117.296 94.75 0.0114111 0.00852029 2024-08-17T17:52:27.791 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:27 smithi152 ceph-mon[93456]: pgmap v1868: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 84 MiB/s wr, 1.34k op/s; 7434/104088 objects degraded (7.142%); 234/104088 objects misplaced (0.225%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:27 smithi152 ceph-mon[93456]: osdmap e473: 8 total, 7 up, 5 in 2024-08-17T17:52:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:27.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:27.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:27.972 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:28.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:28.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:28.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:28.243 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:28.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:28.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:28.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:28.483 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.515 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:28.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:28.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:28.654 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:28.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:28.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 22 16 40913 40897 116.171 92.5625 0.0163033 0.00860284 2024-08-17T17:52:28.797 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:28.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4077841171' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:28.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4077841171' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:28.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4077841171' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:28.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:28.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:29.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:29.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:29.078 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:29.168 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:29.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:29.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:29.337 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:29.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:29.469 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:29.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:29.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:29.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:29.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:29.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 23 16 42508 42492 115.454 99.6875 0.00543902 0.00865526 2024-08-17T17:52:29.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:29 smithi152 ceph-mon[93456]: pgmap v1870: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 83 MiB/s wr, 1.32k op/s; 6202/112425 objects degraded (5.517%); 1024/112425 objects misplaced (0.911%); 24 MiB/s, 391 objects/s recovering 2024-08-17T17:52:29.837 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:29 smithi142 ceph-mon[106729]: pgmap v1870: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 83 MiB/s wr, 1.32k op/s; 6202/112425 objects degraded (5.517%); 1024/112425 objects misplaced (0.911%); 24 MiB/s, 391 objects/s recovering 2024-08-17T17:52:29.838 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:29 smithi142 ceph-mon[103614]: pgmap v1870: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 83 MiB/s wr, 1.32k op/s; 6202/112425 objects degraded (5.517%); 1024/112425 objects misplaced (0.911%); 24 MiB/s, 391 objects/s recovering 2024-08-17T17:52:29.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:29.880 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:29.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:29.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:30.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:30.119 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:30.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:30.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:30.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:30.385 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:30.416 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:30.481 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:52:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:52:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:30.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:30.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:30.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:30.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:30.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 24 16 44144 44128 114.904 102.25 0.0035986 0.00869688 2024-08-17T17:52:30.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:30.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:30 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 6202/112425 objects degraded (5.517%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:30 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 6202/112425 objects degraded (5.517%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:30 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 6202/112425 objects degraded (5.517%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:30.914 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:30.947 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:31.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:31.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:31.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:31.223 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:31.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:31.366 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:31.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:31.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:31.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:31.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:31.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:31.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:31.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 25 16 45657 45641 114.09 94.5625 0.00601045 0.00875879 2024-08-17T17:52:31.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:31 smithi142 ceph-mon[106729]: pgmap v1871: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 74 MiB/s wr, 1.18k op/s; 4971/122535 objects degraded (4.057%); 1517/122535 objects misplaced (1.238%); 18 MiB/s, 294 objects/s recovering 2024-08-17T17:52:31.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:31 smithi142 ceph-mon[103614]: pgmap v1871: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 74 MiB/s wr, 1.18k op/s; 4971/122535 objects degraded (4.057%); 1517/122535 objects misplaced (1.238%); 18 MiB/s, 294 objects/s recovering 2024-08-17T17:52:31.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:31.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:31.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:31 smithi152 ceph-mon[93456]: pgmap v1871: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 74 MiB/s wr, 1.18k op/s; 4971/122535 objects degraded (4.057%); 1517/122535 objects misplaced (1.238%); 18 MiB/s, 294 objects/s recovering 2024-08-17T17:52:31.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.911 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:31.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:32.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:32.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:52:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:52:31] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-08-17T17:52:32.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:32.189 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:32.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:32.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:32.431 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:32.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:32.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:32.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:32.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:32.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:32.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:32.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 26 16 47188 47172 113.382 95.6875 0.00852219 0.00881454 2024-08-17T17:52:32.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:32.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:32.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:32 smithi142 ceph-mon[106729]: pgmap v1872: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 4971/126996 objects degraded (3.914%); 1089/126996 objects misplaced (0.858%); 19 MiB/s, 304 objects/s recovering 2024-08-17T17:52:32.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:32 smithi142 ceph-mon[103614]: pgmap v1872: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 4971/126996 objects degraded (3.914%); 1089/126996 objects misplaced (0.858%); 19 MiB/s, 304 objects/s recovering 2024-08-17T17:52:32.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:33.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:33.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:32 smithi152 ceph-mon[93456]: pgmap v1872: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 4971/126996 objects degraded (3.914%); 1089/126996 objects misplaced (0.858%); 19 MiB/s, 304 objects/s recovering 2024-08-17T17:52:33.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:33.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:33.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:33.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:33.285 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:33.387 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.423 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:33.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:33.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:33.552 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:33.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:33.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:33.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 27 16 48755 48739 112.809 97.9375 0.0064213 0.00885983 2024-08-17T17:52:33.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:33.812 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:33.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:33.953 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:34.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:34.092 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:34.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:34.223 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:34.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:34.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:34.354 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:34.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:34.493 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:34.592 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:52:34.592 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:52:34.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:34.630 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:34.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:34.768 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:34.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 28 16 50256 50240 112.13 93.8125 0.00576071 0.00891246 2024-08-17T17:52:34.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:34 smithi142 ceph-mon[103614]: pgmap v1873: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 4971/143199 objects degraded (3.471%); 1089/143199 objects misplaced (0.760%); 12 MiB/s, 184 objects/s recovering 2024-08-17T17:52:34.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:34 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 4971/143199 objects degraded (3.471%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:34.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:34 smithi142 ceph-mon[106729]: pgmap v1873: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 4971/143199 objects degraded (3.471%); 1089/143199 objects misplaced (0.760%); 12 MiB/s, 184 objects/s recovering 2024-08-17T17:52:34.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:34 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 4971/143199 objects degraded (3.471%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.905 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:34.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:35.006 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:35.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:34 smithi152 ceph-mon[93456]: pgmap v1873: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 4971/143199 objects degraded (3.471%); 1089/143199 objects misplaced (0.760%); 12 MiB/s, 184 objects/s recovering 2024-08-17T17:52:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:34 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 4971/143199 objects degraded (3.471%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:35.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:35.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:35.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:35.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:35.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:35.351 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:35.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:35.459 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:35.561 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:35.593 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:moved 3 / 267 (1.1236%) 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:avg 53.4 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:stddev 2.87054 -> 1.49666 (expected baseline 6.53605) 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:min osd.6 with 51 -> 52 pgs (0.955056 -> 0.973783 * mean) 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:max osd.1 with 59 -> 56 pgs (1.10487 -> 1.04869 * mean) 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:oload 110 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:max_change 0.05 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:max_change_osds 4 2024-08-17T17:52:35.660 INFO:teuthology.orchestra.run.smithi142.stdout:average_utilization 0.0309 2024-08-17T17:52:35.661 INFO:teuthology.orchestra.run.smithi142.stdout:overload_utilization 0.0340 2024-08-17T17:52:35.661 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1 weight 1.0000 -> 0.9500 2024-08-17T17:52:35.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:35.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:35.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:35.770 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 29 16 51766 51750 111.518 94.375 0.0106631 0.00896158 2024-08-17T17:52:35.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:35.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.864 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:35.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:35.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:36.000 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:36.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:35 smithi152 ceph-mon[93456]: from='client.48335 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:52:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:35 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'62260'}"} : dispatch 2024-08-17T17:52:36.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:36.114 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:35 smithi142 ceph-mon[106729]: from='client.48335 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:52:36.114 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:35 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'62260'}"} : dispatch 2024-08-17T17:52:36.114 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:35 smithi142 ceph-mon[103614]: from='client.48335 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:52:36.114 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:35 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'62260'}"} : dispatch 2024-08-17T17:52:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:36.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:36.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:36.278 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:36.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:36.413 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:36.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.559 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:36.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:36.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:52:36.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:36.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:36.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 30 16 53266 53250 110.925 93.75 0.00430946 0.0090048 2024-08-17T17:52:36.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:36.831 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:36.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:36.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:37.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260'}"}]': finished 2024-08-17T17:52:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:36 smithi152 ceph-mon[93456]: osdmap e474: 8 total, 7 up, 5 in 2024-08-17T17:52:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:36 smithi152 ceph-mon[93456]: pgmap v1875: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s; 4971/146076 objects degraded (3.403%); 687/146076 objects misplaced (0.470%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:37.080 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260'}"}]': finished 2024-08-17T17:52:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:36 smithi142 ceph-mon[106729]: osdmap e474: 8 total, 7 up, 5 in 2024-08-17T17:52:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:36 smithi142 ceph-mon[106729]: pgmap v1875: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s; 4971/146076 objects degraded (3.403%); 687/146076 objects misplaced (0.470%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260'}"}]': finished 2024-08-17T17:52:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:36 smithi142 ceph-mon[103614]: osdmap e474: 8 total, 7 up, 5 in 2024-08-17T17:52:37.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:36 smithi142 ceph-mon[103614]: pgmap v1875: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s; 4971/146076 objects degraded (3.403%); 687/146076 objects misplaced (0.470%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:37.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:37.203 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:37.305 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:37.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:37.429 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:37.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:37.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:37.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:37.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:37.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:37.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:37.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:37.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 31 16 53510 53494 107.839 15.25 0.00660818 0.00926353 2024-08-17T17:52:37.807 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:37.834 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:37.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:37.958 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:38.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:38.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:37 smithi152 ceph-mon[93456]: osdmap e475: 8 total, 7 up, 5 in 2024-08-17T17:52:38.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:38.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:37 smithi142 ceph-mon[106729]: osdmap e475: 8 total, 7 up, 5 in 2024-08-17T17:52:38.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:38.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:37 smithi142 ceph-mon[103614]: osdmap e475: 8 total, 7 up, 5 in 2024-08-17T17:52:38.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:38.182 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:38.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:38.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:38.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:38.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:38.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:38.451 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:38.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:38.576 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:38.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:38.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:38.704 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:38.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 32 15 53672 53657 104.787 10.1875 0.00413326 0.00953683 2024-08-17T17:52:38.806 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:38.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:38.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:38.973 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:38 smithi152 ceph-mon[93456]: osdmap e476: 8 total, 7 up, 5 in 2024-08-17T17:52:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:38 smithi152 ceph-mon[93456]: pgmap v1878: 89 pgs: 1 peering, 1 active+recovery_wait, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 2504/153948 objects degraded (1.627%); 2504/153948 objects misplaced (1.627%); 11 MiB/s, 175 objects/s recovering 2024-08-17T17:52:39.075 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:39.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:38 smithi142 ceph-mon[103614]: osdmap e476: 8 total, 7 up, 5 in 2024-08-17T17:52:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:38 smithi142 ceph-mon[103614]: pgmap v1878: 89 pgs: 1 peering, 1 active+recovery_wait, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 2504/153948 objects degraded (1.627%); 2504/153948 objects misplaced (1.627%); 11 MiB/s, 175 objects/s recovering 2024-08-17T17:52:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:38 smithi142 ceph-mon[106729]: osdmap e476: 8 total, 7 up, 5 in 2024-08-17T17:52:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:38 smithi142 ceph-mon[106729]: pgmap v1878: 89 pgs: 1 peering, 1 active+recovery_wait, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s; 2504/153948 objects degraded (1.627%); 2504/153948 objects misplaced (1.627%); 11 MiB/s, 175 objects/s recovering 2024-08-17T17:52:39.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.254 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:39.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:39.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:39.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:39.401 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:39.401 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:39.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:39.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:39.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:39.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:39.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:39.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:39.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 33 16 55156 55140 104.421 92.6875 0.00995273 0.00957165 2024-08-17T17:52:39.784 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:39.886 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:39.918 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:40.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:40.055 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:40.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:39 smithi152 ceph-mon[93456]: osdmap e477: 8 total, 7 up, 5 in 2024-08-17T17:52:40.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:39 smithi142 ceph-mon[106729]: osdmap e477: 8 total, 7 up, 5 in 2024-08-17T17:52:40.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:39 smithi142 ceph-mon[103614]: osdmap e477: 8 total, 7 up, 5 in 2024-08-17T17:52:40.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:40.198 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:40.299 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:40.330 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:40.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:40.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:40.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:40.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:52:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:52:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:40.598 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:40.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:40.700 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:40.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:40.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:40.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 34 16 56553 56537 103.917 87.3125 0.0152779 0.00961803 2024-08-17T17:52:40.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:40.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:40.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:40.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:41.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:41.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:41.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:40 smithi152 ceph-mon[93456]: pgmap v1880: 89 pgs: 1 peering, 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s; 2504/162258 objects degraded (1.543%); 2179/162258 objects misplaced (1.343%); 12 MiB/s, 199 objects/s recovering 2024-08-17T17:52:41.074 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:40 smithi142 ceph-mon[106729]: pgmap v1880: 89 pgs: 1 peering, 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s; 2504/162258 objects degraded (1.543%); 2179/162258 objects misplaced (1.343%); 12 MiB/s, 199 objects/s recovering 2024-08-17T17:52:41.075 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:40 smithi142 ceph-mon[103614]: pgmap v1880: 89 pgs: 1 peering, 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s; 2504/162258 objects degraded (1.543%); 2179/162258 objects misplaced (1.343%); 12 MiB/s, 199 objects/s recovering 2024-08-17T17:52:41.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:41.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:52:41.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:41.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:41.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:41.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:41.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.428 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:41.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:41.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:41.558 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:41.644 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:41.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:41.705 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:41.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 35 16 57938 57922 103.421 86.5625 0.0107552 0.00966395 2024-08-17T17:52:41.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:41.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:41.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:41.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:41 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2504/162258 objects degraded (1.543%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:41.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:41.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:42.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:42.100 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:42.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:42.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:41 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2504/162258 objects degraded (1.543%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:42.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:42.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:52:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:52:41] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.51.0" 2024-08-17T17:52:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:41 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2504/162258 objects degraded (1.543%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:42.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:42.161 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-17T17:52:42.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd test-reweight-by-utilization 110 1.0 2024-08-17T17:52:42.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:42.240 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:42.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:42.342 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:42.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:42.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.512 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:42.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:42.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:42.655 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:42.684 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:42.757 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:42.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 36 16 59236 59220 102.801 81.125 0.0102596 0.0097219 2024-08-17T17:52:42.790 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:42.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:42.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:42.997 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:42 smithi142 ceph-mon[106729]: pgmap v1881: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 2504/165372 objects degraded (1.514%); 1626/165372 objects misplaced (0.983%); 16 MiB/s, 254 objects/s recovering 2024-08-17T17:52:42.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:42 smithi142 ceph-mon[103614]: pgmap v1881: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 2504/165372 objects degraded (1.514%); 1626/165372 objects misplaced (0.983%); 16 MiB/s, 254 objects/s recovering 2024-08-17T17:52:43.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:43.066 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:43.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:42 smithi152 ceph-mon[93456]: pgmap v1881: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 2504/165372 objects degraded (1.514%); 1626/165372 objects misplaced (0.983%); 16 MiB/s, 254 objects/s recovering 2024-08-17T17:52:43.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:43.208 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:43.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:43.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:43.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:43.490 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:43.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:43.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:43.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:43.718 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:52:43.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:43.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 37 16 60613 60597 102.349 86.0625 0.0121047 0.0097656 2024-08-17T17:52:43.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:43.845 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/897839051' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:43.875 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:43.901 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:43.902 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:43.902 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:43.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:44.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:44.045 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:44.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/897839051' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:44.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/897839051' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:44.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:44.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.183 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:44.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:44.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:44.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:44.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:44.371 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:44.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:44.451 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:44.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:44.588 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:44.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:44.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:44.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:44.771 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 38 16 61920 61904 101.805 81.6875 0.00682904 0.00981774 2024-08-17T17:52:44.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:44.856 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:52:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/897839051' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[103614]: osdmap e478: 8 total, 7 up, 5 in 2024-08-17T17:52:44.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[103614]: pgmap v1883: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s; 2504/179070 objects degraded (1.398%); 1198/179070 objects misplaced (0.669%); 14 MiB/s, 221 objects/s recovering 2024-08-17T17:52:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:52:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/897839051' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[106729]: osdmap e478: 8 total, 7 up, 5 in 2024-08-17T17:52:44.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:44 smithi142 ceph-mon[106729]: pgmap v1883: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s; 2504/179070 objects degraded (1.398%); 1198/179070 objects misplaced (0.669%); 14 MiB/s, 221 objects/s recovering 2024-08-17T17:52:44.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:44.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:44.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:45.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:44 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:52:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/897839051' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:44 smithi152 ceph-mon[93456]: osdmap e478: 8 total, 7 up, 5 in 2024-08-17T17:52:45.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:44 smithi152 ceph-mon[93456]: pgmap v1883: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s; 2504/179070 objects degraded (1.398%); 1198/179070 objects misplaced (0.669%); 14 MiB/s, 221 objects/s recovering 2024-08-17T17:52:45.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:45.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:45.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:45.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:45.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:45.363 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:45.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:45.499 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:45.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:45.632 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:52:45.633 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:45.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:45.663 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:52:45.663 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:45.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:45.692 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:52:45.692 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:45.723 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:52:45.724 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:45.757 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:52:45.757 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:45.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:45.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 39 16 63318 63302 101.435 87.375 0.0153452 0.00985321 2024-08-17T17:52:45.784 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:52:45.784 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:45.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:45.813 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:52:45.814 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:45.814 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:52:45.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:52:45.836 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3070575921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:45.836 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3070575921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:45.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:45.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:46.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:46.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3070575921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:46.101 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:46.203 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:46.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:46.293 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:46.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:46.379 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:46.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:46.520 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:46.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:46.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:46.650 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:46.752 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:46.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:52:46.772891+0000 min lat: 0.00102385 max lat: 2.12417 avg lat: 0.00990161 2024-08-17T17:52:46.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:52:46.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 40 16 64628 64612 100.944 81.875 0.00576763 0.00990161 2024-08-17T17:52:46.784 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:46.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:46.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:46.921 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:47.018 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:46 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2504/179070 objects degraded (1.398%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:47.018 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:46 smithi142 ceph-mon[103614]: pgmap v1884: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 72 MiB/s wr, 1.16k op/s; 2504/181728 objects degraded (1.378%); 1198/181728 objects misplaced (0.659%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:52:47.018 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:46 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2504/179070 objects degraded (1.398%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:47.018 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:46 smithi142 ceph-mon[106729]: pgmap v1884: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 72 MiB/s wr, 1.16k op/s; 2504/181728 objects degraded (1.378%); 1198/181728 objects misplaced (0.659%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:52:47.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:47.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:52:47.058 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:52:47.069 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:52:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:46 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2504/179070 objects degraded (1.398%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:46 smithi152 ceph-mon[93456]: pgmap v1884: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 72 MiB/s wr, 1.16k op/s; 2504/181728 objects degraded (1.378%); 1198/181728 objects misplaced (0.659%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:52:47.160 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:47.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:47.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.340 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:47.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:47.441 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:47.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:47.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:47.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:47.610 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:47.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:47.732 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:47.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:47.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 41 16 65921 65905 100.452 80.8125 0.0058117 0.00994971 2024-08-17T17:52:47.834 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:47.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:47.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:47.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:48.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:48.113 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.144 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:48.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:48.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:48.277 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:48 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:48.277 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:48 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:48.283 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:48.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:48 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:52:48.341 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:48.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:48.418 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:48.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:48.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:48.560 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:moved 0 / 267 (0%) 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:avg 53.4 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:stddev 1.49666 -> 1.49666 (expected baseline 6.53605) 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:min osd.5 with 52 -> 52 pgs (0.973783 -> 0.973783 * mean) 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:max osd.1 with 56 -> 56 pgs (1.04869 -> 1.04869 * mean) 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:oload 110 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:max_change 1 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:max_change_osds 4 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:average_utilization 0.0378 2024-08-17T17:52:48.586 INFO:teuthology.orchestra.run.smithi142.stdout:overload_utilization 0.0416 2024-08-17T17:52:48.587 INFO:teuthology.orchestra.run.smithi142.stderr:no change 2024-08-17T17:52:48.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:48.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:48.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 42 16 67146 67130 99.8834 76.5625 0.0070728 0.0100049 2024-08-17T17:52:48.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:48.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:48.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:48.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:48.963 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:49.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:49.071 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:52:49.109 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:49.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:49.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:49.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:49.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:49 smithi152 ceph-mon[93456]: pgmap v1885: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 2504/190383 objects degraded (1.315%); 455/190383 objects misplaced (0.239%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:49.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:49 smithi152 ceph-mon[93456]: from='client.48350 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:52:49.346 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:49.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:49 smithi142 ceph-mon[103614]: pgmap v1885: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 2504/190383 objects degraded (1.315%); 455/190383 objects misplaced (0.239%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:49.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:49 smithi142 ceph-mon[103614]: from='client.48350 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:52:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:49 smithi142 ceph-mon[106729]: pgmap v1885: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 2504/190383 objects degraded (1.315%); 455/190383 objects misplaced (0.239%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:52:49.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:49 smithi142 ceph-mon[106729]: from='client.48350 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:52:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.379 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:49.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:49.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:49.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:49.610 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:49.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:49.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:49.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:49.772 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 43 16 68407 68391 99.3932 78.8125 0.0271331 0.0100563 2024-08-17T17:52:49.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:49.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:50.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:50.046 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:50.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:50.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:50.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:50.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:50.354 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:52:50.354 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:52:50.354 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:52:50.354 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:52:50.354 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:52:50.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:50.447 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:50.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:52:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:52:50.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:50.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:50.689 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:50.718 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:50.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 44 16 69528 69512 98.7265 70.0625 0.00256656 0.0100919 2024-08-17T17:52:50.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:50.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:50.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:50.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:50.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:51.086 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:51.108 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:52:51.114 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:51.216 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:51.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:51.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:51.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:51.402 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:51.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:51.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:51.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:51.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[106729]: osdmap e479: 8 total, 7 up, 5 in 2024-08-17T17:52:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[106729]: pgmap v1887: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 93 MiB/s wr, 1.49k op/s; 2504/201051 objects degraded (1.245%); 455/201051 objects misplaced (0.226%); 9.1 MiB/s, 146 objects/s recovering 2024-08-17T17:52:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 2504/190383 objects degraded (1.315%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:51.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[106729]: osdmap e480: 8 total, 7 up, 5 in 2024-08-17T17:52:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[103614]: osdmap e479: 8 total, 7 up, 5 in 2024-08-17T17:52:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[103614]: pgmap v1887: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 93 MiB/s wr, 1.49k op/s; 2504/201051 objects degraded (1.245%); 455/201051 objects misplaced (0.226%); 9.1 MiB/s, 146 objects/s recovering 2024-08-17T17:52:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 2504/190383 objects degraded (1.315%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:51.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-mon[103614]: osdmap e480: 8 total, 7 up, 5 in 2024-08-17T17:52:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:51 smithi152 ceph-mon[93456]: osdmap e479: 8 total, 7 up, 5 in 2024-08-17T17:52:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:51 smithi152 ceph-mon[93456]: pgmap v1887: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 93 MiB/s wr, 1.49k op/s; 2504/201051 objects degraded (1.245%); 455/201051 objects misplaced (0.226%); 9.1 MiB/s, 146 objects/s recovering 2024-08-17T17:52:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:51 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 2504/190383 objects degraded (1.315%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:52:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:51 smithi152 ceph-mon[93456]: osdmap e480: 8 total, 7 up, 5 in 2024-08-17T17:52:51.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:51.745 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:51.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:51.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 45 16 69888 69872 97.0326 22.5 0.00319596 0.0101316 2024-08-17T17:52:51.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:51.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:51.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:52.012 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:52.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:52:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:52:51] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.51.0" 2024-08-17T17:52:52.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:52.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:52.290 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:52.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:52.351 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-17T17:52:52.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 1000 1.0 2024-08-17T17:52:52.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.443 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:52.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:52.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:52.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:52.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:52.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.721 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:52.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:52.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 46 16 71133 71117 96.6144 77.8125 0.00879131 0.0103455 2024-08-17T17:52:52.822 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:52.822 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:52.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:52.894 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:52.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:53.028 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:53.047 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:52 smithi142 ceph-mon[106729]: osdmap e481: 8 total, 7 up, 5 in 2024-08-17T17:52:53.048 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:52 smithi142 ceph-mon[106729]: pgmap v1890: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 2504/201051 objects degraded (1.245%); 455/201051 objects misplaced (0.226%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T17:52:53.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:52 smithi142 ceph-mon[103614]: osdmap e481: 8 total, 7 up, 5 in 2024-08-17T17:52:53.048 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:52 smithi142 ceph-mon[103614]: pgmap v1890: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 2504/201051 objects degraded (1.245%); 455/201051 objects misplaced (0.226%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T17:52:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:52 smithi152 ceph-mon[93456]: osdmap e481: 8 total, 7 up, 5 in 2024-08-17T17:52:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:52 smithi152 ceph-mon[93456]: pgmap v1890: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 2504/201051 objects degraded (1.245%); 455/201051 objects misplaced (0.226%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T17:52:53.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:53.172 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:53.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:53.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:53.310 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:53.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:53.447 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:53.548 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:53.579 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:53.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:53.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:53.711 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:53.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 47 16 72404 72388 96.2488 79.4375 0.0105572 0.0103848 2024-08-17T17:52:53.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:53.846 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:53.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:53.982 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:54.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:54.116 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:54.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:54.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:54.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:54.388 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:54.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:54.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:54.631 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:54.660 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:54.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:54.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 48 16 73740 73724 95.983 83.5 0.0110044 0.0104135 2024-08-17T17:52:54.808 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:54.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:54.910 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:54.956 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:54.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:54.973 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:54 smithi142 ceph-mon[106729]: pgmap v1891: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s; 1261/214734 objects degraded (0.587%); 870/214734 objects misplaced (0.405%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:52:54.973 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/607835991' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:54.973 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:54 smithi142 ceph-mon[103614]: pgmap v1891: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s; 1261/214734 objects degraded (0.587%); 870/214734 objects misplaced (0.405%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:52:54.973 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/607835991' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:54.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:54 smithi152 ceph-mon[93456]: pgmap v1891: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s; 1261/214734 objects degraded (0.587%); 870/214734 objects misplaced (0.405%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:52:54.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/607835991' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:55.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.088 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:55.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:55.133 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:52:55.133 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:52:55.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:52:55.133 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:52:55.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198347 12.7 12:edb18e09:::benchmark_data_smithi142_261410_object80150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:37.631482+0000", 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 497.497994506, 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1475268830000001, 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198347 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.134 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631482+0000", 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631482+0000", 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631480+0000", 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631486+0000", 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6969999999999998e-06 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631499+0000", 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2653000000000001e-05 2024-08-17T17:52:55.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.631504+0000", 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0919999999999997e-06 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.662521+0000", 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031017335 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.666399+0000", 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038782230000000001 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.697112+0000", 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030712891999999999 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:37.701434+0000", 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043223760000000002 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.663607+0000", 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96217242300000005 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.667006+0000", 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033988130000000001 2024-08-17T17:52:55.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.673808+0000", 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068021929999999998 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.696925+0000", 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023116927999999998 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.698058+0000", 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011333669999999999 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.699204+0000", 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011459829999999999 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.767572+0000", 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.068367791999999997 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.769521+0000", 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019486320000000001 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.779009+0000", 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094880050000000007 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198563 12.7 12:e8027f39:::benchmark_data_smithi142_261410_object80366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:38.487489+0000", 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 496.64198750399999, 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1351648779999999, 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198563 2024-08-17T17:52:55.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487489+0000", 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487489+0000", 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487487+0000", 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487494+0000", 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3210000000000001e-06 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487507+0000", 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.345e-05 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.487513+0000", 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.806e-06 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.575081+0000", 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.087567542999999998 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.579769+0000", 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046882110000000003 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.645321+0000", 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.065552113999999995 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:38.649276+0000", 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039547619999999997 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.569789+0000", 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92051316299999997 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.573243+0000", 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034539979999999998 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.579849+0000", 2024-08-17T17:52:55.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066063770000000001 2024-08-17T17:52:55.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.607563+0000", 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027713550999999999 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.608747+0000", 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001184428 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.611243+0000", 2024-08-17T17:52:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024959159999999999 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.612429+0000", 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00118545 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.614361+0000", 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019328329999999999 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.622654+0000", 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082922659999999995 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198764 12.7 12:e75f9baa:::benchmark_data_smithi142_261410_object80567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:39.363457+0000", 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 495.76601874699998, 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.131761899, 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.152 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198764 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363457+0000", 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363457+0000", 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363456+0000", 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363462+0000", 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.101e-06 2024-08-17T17:52:55.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363471+0000", 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7899999999999994e-06 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.363477+0000", 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9519999999999999e-06 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.389870+0000", 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026392283999999998 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.393700+0000", 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038300169999999998 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.468266+0000", 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.074566553999999993 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:39.472389+0000", 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041231289999999997 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.423214+0000", 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95082494900000003 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.426690+0000", 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003475803 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.433549+0000", 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068586109999999997 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.450814+0000", 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017264901999999999 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.451972+0000", 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001158145 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.466170+0000", 2024-08-17T17:52:55.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014198275 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.484940+0000", 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018770011 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.486814+0000", 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001873792 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.495219+0000", 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084055919999999999 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198915 12.7 12:e781df6c:::benchmark_data_smithi142_261410_object80718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 494.89980583400001, 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.174704889, 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.161 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198915 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229670+0000", 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229674+0000", 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8970000000000001e-06 2024-08-17T17:52:55.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229681+0000", 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9389999999999997e-06 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.229685+0000", 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3730000000000001e-06 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.311692+0000", 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082006734999999997 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.315505+0000", 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038131139999999998 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.336219+0000", 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020714249000000001 2024-08-17T17:52:55.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.340383+0000", 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041637410000000003 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.342708+0000", 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0023252469999999 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.346270+0000", 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035622259999999999 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.353861+0000", 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0075912849999999997 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.391568+0000", 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037706228000000001 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.392701+0000", 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011330800000000001 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.393099+0000", 2024-08-17T17:52:55.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039868500000000002 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.394236+0000", 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011362169999999999 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.396099+0000", 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018637079999999999 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.404375+0000", 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082759449999999998 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198918 12.7 12:ebaa8f49:::benchmark_data_smithi142_261410_object80721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.237002+0000", 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 494.89247382299999, 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1822594, 2024-08-17T17:52:55.170 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198918 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237002+0000", 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237002+0000", 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237001+0000", 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237007+0000", 2024-08-17T17:52:55.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4999999999999999e-06 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237020+0000", 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.31e-05 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.237026+0000", 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0360000000000003e-06 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.317063+0000", 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.080037013000000004 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.320729+0000", 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036662050000000001 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.355665+0000", 2024-08-17T17:52:55.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.034935739 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.359838+0000", 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041732590000000003 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.373092+0000", 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0132541580000001 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.376509+0000", 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034164769999999998 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.383323+0000", 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068136610000000004 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.406004+0000", 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022681499000000001 2024-08-17T17:52:55.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.406488+0000", 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000484076 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.407757+0000", 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001269318 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.409111+0000", 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001353876 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.410973+0000", 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018619859999999999 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.419262+0000", 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082884780000000002 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.178 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198919 12.7 12:e46f8c19:::benchmark_data_smithi142_261410_object80722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.239145+0000", 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 494.89033168200001, 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2519525709999999, 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.179 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198919 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239145+0000", 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239145+0000", 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239144+0000", 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239149+0000", 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8749999999999999e-06 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239158+0000", 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9709999999999993e-06 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.239161+0000", 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7000000000000002e-06 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.360251+0000", 2024-08-17T17:52:55.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.121089716 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.364026+0000", 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037748669999999999 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.404095+0000", 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040069283999999997 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.408160+0000", 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040646240000000002 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.436458+0000", 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.028298011 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441031+0000", 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004573321 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.447908+0000", 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068769850000000004 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.475585+0000", 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027677013 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.476878+0000", 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012927679999999999 2024-08-17T17:52:55.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.477259+0000", 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038065700000000001 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.479288+0000", 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020290170000000001 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.481744+0000", 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002456906 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.491097+0000", 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093526819999999993 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.187 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198936 12.7 12:e99b51fa:::benchmark_data_smithi142_261410_object80739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.522164+0000", 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 494.60731259800002, 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1296211789999999, 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198936 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522164+0000", 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522164+0000", 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522162+0000", 2024-08-17T17:52:55.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522170+0000", 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6799999999999993e-06 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522182+0000", 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1952e-05 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.522186+0000", 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1219999999999997e-06 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.562515+0000", 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040329190000000001 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.566615+0000", 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040994330000000004 2024-08-17T17:52:55.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.618234+0000", 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.051619273 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.622379+0000", 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041452449999999997 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.597410+0000", 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97503114400000002 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.600747+0000", 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033365970000000002 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.607430+0000", 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066826740000000004 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.636271+0000", 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028841201 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.638583+0000", 2024-08-17T17:52:55.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:55.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023119550000000001 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.639753+0000", 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011705719999999999 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.640922+0000", 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001169128 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.642772+0000", 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018494099999999999 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.651785+0000", 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090129310000000001 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198958 12.7 12:e0f11b00:::benchmark_data_smithi142_261410_object80761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.593679+0000", 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 494.535797403, 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1428628599999999, 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198958 2024-08-17T17:52:55.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593679+0000", 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593679+0000", 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593677+0000", 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593685+0000", 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6229999999999997e-06 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593698+0000", 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3156000000000001e-05 2024-08-17T17:52:55.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.593706+0000", 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7309999999999993e-06 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.638168+0000", 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.044462116000000003 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.641965+0000", 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037972180000000002 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.692915+0000", 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.050950286999999997 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.697078+0000", 2024-08-17T17:52:55.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004162603 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.663251+0000", 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96617332499999997 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.667563+0000", 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043112419999999999 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.674289+0000", 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.006725979 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.691308+0000", 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017019863999999999 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.692448+0000", 2024-08-17T17:52:55.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011392279999999999 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.693632+0000", 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001184762 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.725305+0000", 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031672968000000003 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.727252+0000", 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019466799999999999 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.736542+0000", 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092896290000000006 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:198963 12.7 12:ee7826ac:::benchmark_data_smithi142_261410_object80766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:40.629744+0000", 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 494.49973242900001, 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1424937150000001, 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 198963 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.207 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629744+0000", 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629744+0000", 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629742+0000", 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629751+0000", 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5709999999999998e-06 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629764+0000", 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3567000000000001e-05 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.629771+0000", 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9720000000000003e-06 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.642536+0000", 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01276499 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.646367+0000", 2024-08-17T17:52:55.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038304649999999999 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.716727+0000", 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.070360012 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:40.720845+0000", 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041184680000000001 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.694028+0000", 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.973182938 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.697431+0000", 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003402712 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.704052+0000", 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066207219999999999 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.711653+0000", 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076014120000000001 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.712779+0000", 2024-08-17T17:52:55.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001125544 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.713909+0000", 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001129955 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.762028+0000", 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.048118919000000003 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.763936+0000", 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019082229999999999 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.772238+0000", 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083017330000000004 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199178 12.7 12:e545f06d:::benchmark_data_smithi142_261410_object80981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:41.441883+0000", 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 493.68759320200002, 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.15849951, 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199178 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441883+0000", 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441883+0000", 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441881+0000", 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:52:55.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441892+0000", 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1060000000000001e-05 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441910+0000", 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8453000000000001e-05 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.441918+0000", 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8010000000000004e-06 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.496560+0000", 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.054642124 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.501148+0000", 2024-08-17T17:52:55.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045884860000000001 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.579575+0000", 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.078426483000000005 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.583636+0000", 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040609299999999999 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.549416+0000", 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96577959199999996 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.553775+0000", 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004359145 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.560374+0000", 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065997119999999998 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.567655+0000", 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072809179999999999 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.569162+0000", 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015071850000000001 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.571749+0000", 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025864870000000002 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.589264+0000", 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017515037000000001 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.591105+0000", 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001840639 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.600383+0000", 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092779160000000006 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199190 12.7 12:ee8d4a4a:::benchmark_data_smithi142_261410_object80993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:41.522294+0000", 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 493.607182534, 2024-08-17T17:52:55.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1390958, 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199190 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522294+0000", 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522294+0000", 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522293+0000", 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522297+0000", 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9470000000000002e-06 2024-08-17T17:52:55.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522308+0000", 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0305999999999999e-05 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.522312+0000", 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0130000000000004e-06 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.584040+0000", 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.061727929000000001 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.587799+0000", 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037590089999999998 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.630110+0000", 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042311720999999997 2024-08-17T17:52:55.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.634210+0000", 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040994880000000001 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.623180+0000", 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98897003000000006 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.626652+0000", 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034722490000000002 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.633375+0000", 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067232910000000002 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.645982+0000", 2024-08-17T17:52:55.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012606254000000001 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.647427+0000", 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014450470000000001 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.647809+0000", 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000382526 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.649307+0000", 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014981549999999999 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.651673+0000", 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023651990000000001 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.661389+0000", 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0097168299999999992 2024-08-17T17:52:55.232 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199206 12.7 12:ece9c3ab:::benchmark_data_smithi142_261410_object81009:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:41.750968+0000", 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 493.37850832100003, 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.144830995, 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.233 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199206 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750968+0000", 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750968+0000", 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750967+0000", 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750973+0000", 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4420000000000003e-06 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750984+0000", 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0604999999999999e-05 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.750988+0000", 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.806549+0000", 2024-08-17T17:52:55.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.055560905000000001 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.810354+0000", 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038050599999999999 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.854073+0000", 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043719081999999999 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:41.858133+0000", 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040597200000000002 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.835051+0000", 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97691826800000003 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.838509+0000", 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034577039999999998 2024-08-17T17:52:55.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.845207+0000", 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066979739999999998 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.881443+0000", 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036235410000000003 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.883084+0000", 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016413389999999999 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.884242+0000", 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001158266 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.885750+0000", 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001507988 2024-08-17T17:52:55.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.887617+0000", 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018663670000000001 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.895799+0000", 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081824020000000001 2024-08-17T17:52:55.242 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199449 12.7 12:e5e46031:::benchmark_data_smithi142_261410_object81252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:42.689896+0000", 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 492.43957988199998, 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.176776528, 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.243 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199449 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689896+0000", 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689896+0000", 2024-08-17T17:52:55.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689895+0000", 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689902+0000", 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8590000000000005e-06 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689917+0000", 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4659e-05 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.689925+0000", 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8620000000000003e-06 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.762228+0000", 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.072303386999999997 2024-08-17T17:52:55.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.765932+0000", 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037033679999999999 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.806005+0000", 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040072789999999997 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:42.809901+0000", 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038965800000000002 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.810713+0000", 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000811713 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.814169+0000", 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034557979999999999 2024-08-17T17:52:55.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.821038+0000", 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068688890000000004 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.852300+0000", 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031262184999999998 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.852690+0000", 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039005499999999999 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.854680+0000", 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019902330000000001 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.856368+0000", 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016883079999999999 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.858314+0000", 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00194524 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:43.866673+0000", 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083593409999999993 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199845 12.7 12:eb45e44c:::benchmark_data_smithi142_261410_object81648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:44.899208+0000", 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 490.23026812099999, 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1399064969999999, 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199845 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899208+0000", 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899208+0000", 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899206+0000", 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:55.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899216+0000", 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.448e-06 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899231+0000", 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5106000000000001e-05 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.899239+0000", 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2579999999999995e-06 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.904216+0000", 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049764359999999999 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908071+0000", 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038552119999999998 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.972517+0000", 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.064445767000000001 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.976669+0000", 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041523150000000002 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.979284+0000", 2024-08-17T17:52:55.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0026148180000001 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.983482+0000", 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041981509999999998 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.991310+0000", 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078283929999999995 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.025355+0000", 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.034044645999999998 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.025732+0000", 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037706399999999998 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.027293+0000", 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001561408 2024-08-17T17:52:55.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.028529+0000", 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001235508 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.030941+0000", 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024122589999999999 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.039115+0000", 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081734019999999998 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:199849 12.7 12:e0856a61:::benchmark_data_smithi142_261410_object81652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:44.908424+0000", 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 490.22105204299999, 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1458578130000001, 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.260 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199849 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908424+0000", 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908424+0000", 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908422+0000", 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908429+0000", 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2709999999999998e-06 2024-08-17T17:52:55.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908442+0000", 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3506000000000001e-05 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.908448+0000", 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2329999999999997e-06 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.977216+0000", 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.068768515000000002 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.981719+0000", 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045033110000000003 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:44.999592+0000", 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017873138 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:45.003531+0000", 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039381390000000002 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.007790+0000", 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0042596829999999 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.011070+0000", 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032793549999999999 2024-08-17T17:52:55.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.017571+0000", 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065018080000000004 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.040313+0000", 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022741259999999999 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.041548+0000", 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001235225 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.041917+0000", 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036895999999999999 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.043112+0000", 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011954470000000001 2024-08-17T17:52:55.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.045174+0000", 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020620980000000001 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.054282+0000", 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091075559999999993 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46715.0:200246 12.7 12:e07dd4ec:::benchmark_data_smithi142_261410_object82049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:44:46.619725+0000", 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 488.509751349, 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.150011423, 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46715", 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200246 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619725+0000", 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619725+0000", 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619723+0000", 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619731+0000", 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9379999999999992e-06 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619747+0000", 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5361000000000001e-05 2024-08-17T17:52:55.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.619754+0000", 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2599999999999999e-06 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.640705+0000", 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020951588 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.644209+0000", 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035039759999999998 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.674286+0000", 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030076540999999998 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for rw locks", 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:46.678496+0000", 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042095359999999998 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.714365+0000", 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0358691840000001 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.717503+0000", 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031387659999999999 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5,7", 2024-08-17T17:52:55.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.723757+0000", 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062535209999999997 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.739570+0000", 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015812577000000001 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.757147+0000", 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017577808 2024-08-17T17:52:55.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.758276+0000", 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011289799999999999 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.759426+0000", 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001150082 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.761284+0000", 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001857612 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:44:47.769736+0000", 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084522499999999997 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201795 13.1 13:8ad4d0e6:::benchmark_data_smithi142_393845_object73979:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:52:55.277 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 140.53563319400001, 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0760220620000001, 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201795 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593845+0000", 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9800000000000001e-06 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593850+0000", 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3619999999999997e-06 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593867+0000", 2024-08-17T17:52:55.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6390999999999999e-05 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601117+0000", 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072505139999999996 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601123+0000", 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9089999999999997e-06 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.651452+0000", 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.05032927 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.651561+0000", 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010834 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.669765+0000", 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018204426999999999 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.669865+0000", 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100139 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19432 14.5 14:a0a12a2e:::benchmark_data_smithi142_428506_object19431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.007833+0000", 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 42.121643214999999, 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0758336019999999, 2024-08-17T17:52:55.283 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19432 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007833+0000", 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007833+0000", 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007832+0000", 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007937+0000", 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104953 2024-08-17T17:52:55.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007956+0000", 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9165e-05 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007962+0000", 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4819999999999999e-06 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.008220+0000", 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025818 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.008337+0000", 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116759 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.008477+0000", 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000140261 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.009817+0000", 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013396219999999999 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017464+0000", 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076474519999999999 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017477+0000", 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2092999999999999e-05 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018015+0000", 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000538084 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018024+0000", 2024-08-17T17:52:55.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0969999999999995e-06 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.083579+0000", 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.065555121 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.083667+0000", 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7758999999999998e-05 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19441 14.5 14:a5394b9a:::benchmark_data_smithi142_428506_object19440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.012249+0000", 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 42.117227245000002, 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0783817920000001, 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:52:55.294 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19441 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012249+0000", 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012249+0000", 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012248+0000", 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012290+0000", 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2481000000000002e-05 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012302+0000", 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1325000000000001e-05 2024-08-17T17:52:55.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012318+0000", 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6507999999999998e-05 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017498+0000", 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051802669999999997 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017505+0000", 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2430000000000002e-06 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018748+0000", 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0012432849999999 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018756+0000", 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3559999999999993e-06 2024-08-17T17:52:55.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084557+0000", 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0658009470000001 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084701+0000", 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014419899999999999 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084862+0000", 2024-08-17T17:52:55.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016102299999999999 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.086561+0000", 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016981559999999999 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089458+0000", 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028972920000000001 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089538+0000", 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0406000000000002e-05 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089598+0000", 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9762999999999997e-05 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.090631+0000", 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010327509999999999 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.301 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19451 14.c 14:3fda18e8:::benchmark_data_smithi142_428506_object19450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.028400+0000", 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 42.101076540999998, 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0625520659999999, 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19451 2024-08-17T17:52:55.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028400+0000", 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028400+0000", 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028398+0000", 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028434+0000", 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5178000000000001e-05 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028449+0000", 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5191000000000001e-05 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028455+0000", 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.251e-06 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028732+0000", 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027675300000000003 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028741+0000", 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7110000000000007e-06 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.015985+0000", 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98724387300000005 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.015997+0000", 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1564e-05 2024-08-17T17:52:55.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084473+0000", 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.068476583 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084629+0000", 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015552799999999999 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:52:55.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084789+0000", 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000159803 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.088114+0000", 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033248280000000002 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.088150+0000", 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6554999999999998e-05 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089572+0000", 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014218189999999999 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089618+0000", 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6201e-05 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.090952+0000", 2024-08-17T17:52:55.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013335269999999999 2024-08-17T17:52:55.310 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.310 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.310 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.310 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:52:55.310 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:52:55.310 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:52:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:55.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:55.412 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:55.442 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:55.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:55.544 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:55.576 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:55.608 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:52:55.678 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:55.707 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:55.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 49 16 74936 74920 95.5496 74.75 0.0106545 0.0104611 2024-08-17T17:52:55.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:55.839 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:55 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/607835991' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:52:55.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:55 smithi142 ceph-mon[106729]: osdmap e482: 8 total, 7 up, 5 in 2024-08-17T17:52:55.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:55 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/607835991' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:52:55.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:55 smithi142 ceph-mon[103614]: osdmap e482: 8 total, 7 up, 5 in 2024-08-17T17:52:55.941 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:55.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:55 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/607835991' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:52:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:55 smithi152 ceph-mon[93456]: osdmap e482: 8 total, 7 up, 5 in 2024-08-17T17:52:56.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:56.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:56.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:56.202 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:52:56.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:56.247 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:56.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:56.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:56.394 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:56.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:56.524 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:56.626 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:56.640 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:56.654 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:56.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:56.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:56.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 50 16 76134 76118 95.136 74.875 0.00555961 0.0105061 2024-08-17T17:52:56.789 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:56.891 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:56.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.928 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:56.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:57.029 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:57.060 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:56 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 1261/214734 objects degraded (0.587%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:52:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:56 smithi152 ceph-mon[93456]: pgmap v1893: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 54 MiB/s wr, 869 op/s; 1261/216714 objects degraded (0.582%); 870/216714 objects misplaced (0.401%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:52:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:56 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 1261/214734 objects degraded (0.587%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:52:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:57.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:56 smithi142 ceph-mon[106729]: pgmap v1893: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 54 MiB/s wr, 869 op/s; 1261/216714 objects degraded (0.582%); 870/216714 objects misplaced (0.401%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:52:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:56 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 1261/214734 objects degraded (0.587%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:52:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:56 smithi142 ceph-mon[103614]: pgmap v1893: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 54 MiB/s wr, 869 op/s; 1261/216714 objects degraded (0.582%); 870/216714 objects misplaced (0.401%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:52:57.162 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:57.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:57.295 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:57.329 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:57.383 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:52:57.384 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:57.420 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:52:57.421 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:57.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:57.447 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:52:57.447 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:57.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:57.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:57.471 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:52:57.471 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:57.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:52:57.498 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:57.527 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:52:57.528 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:57.560 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:52:57.560 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:57.560 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:52:57.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:57.604 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:57.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:57.746 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:57.773 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 51 16 77381 77365 94.7987 77.9375 0.0266643 0.010544 2024-08-17T17:52:57.795 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2462729057' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:57.795 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2462729057' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2462729057' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:57.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:57.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:57.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:58.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.026 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:58.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:58.127 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:58.158 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:58.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.291 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:58.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:58.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:58.420 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:58.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.570 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:58.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:52:58.599 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:52:58.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.709 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:58.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:58.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 52 16 78620 78604 94.4647 77.4375 0.00464912 0.0105805 2024-08-17T17:52:58.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:58.811 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:52:58 smithi152 ceph-mon[93456]: pgmap v1894: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s; 1261/224844 objects degraded (0.561%); 870/224844 objects misplaced (0.387%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:52:58.838 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:58.863 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:58.909 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:58.910 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:52:58 smithi142 ceph-mon[106729]: pgmap v1894: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s; 1261/224844 objects degraded (0.561%); 870/224844 objects misplaced (0.387%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:52:58.910 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:52:58 smithi142 ceph-mon[103614]: pgmap v1894: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s; 1261/224844 objects degraded (0.561%); 870/224844 objects misplaced (0.387%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:52:58.939 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:58.975 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:59.077 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.117 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:59.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:59.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:59.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:59.353 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:59.399 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:59.500 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:59.532 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:59.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:59.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:52:59.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:59.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:52:59.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:59.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 53 16 79839 79823 94.1197 76.1875 0.00477081 0.0106192 2024-08-17T17:52:59.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:59.811 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:59.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:59.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.941 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:52:59.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:00.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:00.063 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:53:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:00.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:00.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:00.240 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:00.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:53:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:53:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:00.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:00.377 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:00.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:00.511 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:00.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:00.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:53:00.613 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:00.637 INFO:teuthology.orchestra.run.smithi142.stdout:moved 0 / 267 (0%) 2024-08-17T17:53:00.637 INFO:teuthology.orchestra.run.smithi142.stdout:avg 53.4 2024-08-17T17:53:00.637 INFO:teuthology.orchestra.run.smithi142.stdout:stddev 1.49666 -> 1.49666 (expected baseline 6.53605) 2024-08-17T17:53:00.637 INFO:teuthology.orchestra.run.smithi142.stdout:min osd.5 with 52 -> 52 pgs (0.973783 -> 0.973783 * mean) 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout:max osd.1 with 56 -> 56 pgs (1.04869 -> 1.04869 * mean) 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout:oload 1000 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout:max_change 1 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout:max_change_osds 4 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout:average_utilization 0.0442 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stdout:overload_utilization 0.4423 2024-08-17T17:53:00.638 INFO:teuthology.orchestra.run.smithi142.stderr:no change 2024-08-17T17:53:00.657 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:00.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:00.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:00.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 54 16 80860 80844 93.5584 63.8125 0.0124987 0.0106831 2024-08-17T17:53:00.798 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:00.900 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:00.927 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:01.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:01.064 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:01.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:00 smithi152 ceph-mon[93456]: pgmap v1895: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 4.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 1261/235278 objects degraded (0.536%); 538/235278 objects misplaced (0.229%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:53:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:00 smithi142 ceph-mon[103614]: pgmap v1895: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 4.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 1261/235278 objects degraded (0.536%); 538/235278 objects misplaced (0.229%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:53:01.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:00 smithi142 ceph-mon[106729]: pgmap v1895: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 4.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 1261/235278 objects degraded (0.536%); 538/235278 objects misplaced (0.229%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:53:01.130 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:53:01.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:01.216 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:01.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:01.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:01.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:01.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:01.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:01.479 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:01.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:01.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:01.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:01.740 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:01.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:01 smithi152 ceph-mon[93456]: from='client.38484 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:01.742 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:01 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 1261/235278 objects degraded (0.536%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:53:01.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 55 16 81914 81898 93.0549 65.875 0.0249123 0.0107397 2024-08-17T17:53:01.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:01.905 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:01.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:01.980 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:01 smithi142 ceph-mon[106729]: from='client.38484 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:01.980 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:01 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 1261/235278 objects degraded (0.536%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:53:01.981 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:53:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:53:01] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.51.0" 2024-08-17T17:53:01.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:01 smithi142 ceph-mon[103614]: from='client.38484 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:01.981 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:01 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 1261/235278 objects degraded (0.536%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:53:02.008 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:02.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.029 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:02.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:02.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:02.186 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:02.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:02.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:02.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:02.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.468 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:02.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:02.470 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:02.470 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:02.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:02.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:02.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:02.737 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:02.775 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 56 16 82908 82892 92.5026 62.125 0.0182242 0.0108055 2024-08-17T17:53:02.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:02.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:02.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:02.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.020 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:03.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:02 smithi152 ceph-mon[93456]: pgmap v1896: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 4.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 1261/235278 objects degraded (0.536%); 538/235278 objects misplaced (0.229%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:53:03.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:03.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:02 smithi142 ceph-mon[106729]: pgmap v1896: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 4.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 1261/235278 objects degraded (0.536%); 538/235278 objects misplaced (0.229%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:53:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:02 smithi142 ceph-mon[103614]: pgmap v1896: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 4.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 1261/235278 objects degraded (0.536%); 538/235278 objects misplaced (0.229%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:53:03.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:03.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:03.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:53:03.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.284 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:03.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:03.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.422 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:03.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:03.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:03.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.568 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:03.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:03.660 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:53:03.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:03.696 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:03.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:03.774 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 57 16 84075 84059 92.1592 72.9375 0.00826418 0.0108456 2024-08-17T17:53:03.798 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:03.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:03.944 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:03.971 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:04.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:04.114 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:04.135 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-17T17:53:04.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-08-17T17:53:04.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:04.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:04.314 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:04.327 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:04.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:04.383 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:04.485 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:04.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:04.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:04.652 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:04.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:04 smithi142 ceph-mon[103614]: pgmap v1897: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 1261/249753 objects degraded (0.505%); 206/249753 objects misplaced (0.082%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T17:53:04.719 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:04 smithi142 ceph-mon[106729]: pgmap v1897: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 1261/249753 objects degraded (0.505%); 206/249753 objects misplaced (0.082%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T17:53:04.754 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:04.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 58 16 85223 85207 91.8041 71.75 0.0287188 0.0108873 2024-08-17T17:53:04.796 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:04.898 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:04.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:05.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:05.049 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:04 smithi152 ceph-mon[93456]: pgmap v1897: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 1261/249753 objects degraded (0.505%); 206/249753 objects misplaced (0.082%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T17:53:05.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.074 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:05.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:05.175 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:05.200 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:05.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:05.330 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:05.432 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:05.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:05.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:05.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:05.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:05.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:05.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.724 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:05.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:05.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 59 16 86268 86252 91.3551 65.3125 0.0109961 0.0109413 2024-08-17T17:53:05.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.857 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:05.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:05.958 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:05.990 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:06.092 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:06.095 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:05 smithi142 ceph-mon[106729]: Health check update: Degraded data redundancy: 1261/249753 objects degraded (0.505%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:53:06.095 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:05 smithi142 ceph-mon[103614]: Health check update: Degraded data redundancy: 1261/249753 objects degraded (0.505%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:53:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:06.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:06.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:05 smithi152 ceph-mon[93456]: Health check update: Degraded data redundancy: 1261/249753 objects degraded (0.505%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:53:06.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:06.270 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:06.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:06.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:06.420 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:06.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:06.521 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:06.521 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:06.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:06.565 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:06.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:06.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:06.710 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:06.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:53:06.776728+0000 min lat: 0.00102385 max lat: 2.12417 avg lat: 0.0109771 2024-08-17T17:53:06.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:53:06.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 60 16 87448 87432 91.0616 73.75 0.0152858 0.0109771 2024-08-17T17:53:06.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:06.845 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:06.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:06.985 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:06.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:07.001 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:07.088 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:07.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:07.117 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:06 smithi142 ceph-mon[106729]: pgmap v1898: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 1261/251862 objects degraded (0.501%); 206/251862 objects misplaced (0.082%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:53:07.117 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:06 smithi142 ceph-mon[103614]: pgmap v1898: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 1261/251862 objects degraded (0.501%); 206/251862 objects misplaced (0.082%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:53:07.214 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:07.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:07.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:06 smithi152 ceph-mon[93456]: pgmap v1898: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 1261/251862 objects degraded (0.501%); 206/251862 objects misplaced (0.082%); 3.8 MiB/s, 60 objects/s recovering 2024-08-17T17:53:07.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:07.361 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:07.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:07.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:07.588 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:07.611 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:07.712 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:07.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:07.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 61 16 87816 87800 89.9458 23 0.00339993 0.0109791 2024-08-17T17:53:07.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:07.857 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:07.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:07.959 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:07.994 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:08.096 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:08.128 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:08.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:08.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.267 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:08.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:08.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:07 smithi152 ceph-mon[93456]: osdmap e483: 8 total, 7 up, 5 in 2024-08-17T17:53:08.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3226878736' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:08.363 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:53:08.363 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:07 smithi142 ceph-mon[106729]: osdmap e483: 8 total, 7 up, 5 in 2024-08-17T17:53:08.363 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3226878736' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:08.363 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:07 smithi142 ceph-mon[103614]: osdmap e483: 8 total, 7 up, 5 in 2024-08-17T17:53:08.364 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3226878736' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:08.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:08.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:08.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:08.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:08.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:08.656 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:08.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:08.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:08.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 62 16 88704 88688 89.3902 55.5 0.0179716 0.011182 2024-08-17T17:53:08.789 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:08.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:08.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:53:08.891 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:08.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:09.024 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:09.053 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:08 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3226878736' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:08 smithi152 ceph-mon[93456]: osdmap e484: 8 total, 7 up, 5 in 2024-08-17T17:53:09.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:08 smithi152 ceph-mon[93456]: pgmap v1901: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.3 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 60 MiB/s wr, 954 op/s; 1261/258183 objects degraded (0.488%); 206/258183 objects misplaced (0.080%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:53:09.155 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:09.184 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:09.232 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:53:09.286 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:09.311 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3226878736' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[106729]: osdmap e484: 8 total, 7 up, 5 in 2024-08-17T17:53:09.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[106729]: pgmap v1901: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.3 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 60 MiB/s wr, 954 op/s; 1261/258183 objects degraded (0.488%); 206/258183 objects misplaced (0.080%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:53:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3226878736' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[103614]: osdmap e484: 8 total, 7 up, 5 in 2024-08-17T17:53:09.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:08 smithi142 ceph-mon[103614]: pgmap v1901: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 86 active+clean; 5.3 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 60 MiB/s wr, 954 op/s; 1261/258183 objects degraded (0.488%); 206/258183 objects misplaced (0.080%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T17:53:09.413 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:09.441 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:09.543 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:09.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:09.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:09.699 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:09.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 63 16 89808 89792 89.0665 69 0.0108286 0.0112225 2024-08-17T17:53:09.801 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:09.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:09.931 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:09.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:09.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:10.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:10.101 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:10.202 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:10.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:10.355 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:10.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:10.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:53:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:53:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:10.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:10.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.724 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.754 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:10.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.776 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 64 16 90954 90938 88.7939 71.625 0.0177656 0.0112569 2024-08-17T17:53:10.857 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:10.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:10.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:10 smithi152 ceph-mon[93456]: pgmap v1902: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.4 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 80 MiB/s wr, 1.28k op/s; 957/266142 objects misplaced (0.360%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:53:10.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:10 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1261/258183 objects degraded (0.488%), 1 pg degraded) 2024-08-17T17:53:10.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:11.032 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:10 smithi142 ceph-mon[103614]: pgmap v1902: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.4 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 80 MiB/s wr, 1.28k op/s; 957/266142 objects misplaced (0.360%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:53:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:10 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1261/258183 objects degraded (0.488%), 1 pg degraded) 2024-08-17T17:53:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:10 smithi142 ceph-mon[106729]: pgmap v1902: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.4 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 80 MiB/s wr, 1.28k op/s; 957/266142 objects misplaced (0.360%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T17:53:11.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:10 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1261/258183 objects degraded (0.488%), 1 pg degraded) 2024-08-17T17:53:11.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:11.163 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:11.265 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:11.295 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:11.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:11.423 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:11.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:11.565 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:11.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:11.708 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:11.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:11.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 65 16 92056 92040 88.4874 68.875 0.00937275 0.011296 2024-08-17T17:53:11.809 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:11.834 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:11.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:53:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:53:11] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-08-17T17:53:11.891 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:11.892 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:11.936 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:11.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:12.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:12.102 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:12.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:12.235 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:12.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.376 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:12.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:12.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:12.512 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:12.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:12.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:12.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:12.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:12.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:53:12.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:12.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 66 16 93175 93159 88.2055 69.9375 0.00999139 0.011332 2024-08-17T17:53:12.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:12.809 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:12.809 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:12.809 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:12.809 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:12.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:12.930 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:12.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:13.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:13.065 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:13.167 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:13.179 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:53:13.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:13.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:13.304 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:13.346 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:13.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:13.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.490 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:13.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:13.591 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:13.612 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:13.620 INFO:teuthology.orchestra.run.smithi142.stdout:moved 0 / 267 (0%) 2024-08-17T17:53:13.620 INFO:teuthology.orchestra.run.smithi142.stdout:avg 53.4 2024-08-17T17:53:13.620 INFO:teuthology.orchestra.run.smithi142.stdout:stddev 1.49666 -> 1.49666 (expected baseline 6.53605) 2024-08-17T17:53:13.620 INFO:teuthology.orchestra.run.smithi142.stdout:min osd.5 with 52 -> 52 pgs (0.973783 -> 0.973783 * mean) 2024-08-17T17:53:13.620 INFO:teuthology.orchestra.run.smithi142.stdout:max osd.1 with 56 -> 56 pgs (1.04869 -> 1.04869 * mean) 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stdout:oload 1000 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stdout:max_change 3 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stdout:max_change_osds 4 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stdout:average_utilization 0.0491 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stdout:overload_utilization 0.4906 2024-08-17T17:53:13.621 INFO:teuthology.orchestra.run.smithi142.stderr:no change 2024-08-17T17:53:13.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:53:13.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:13 smithi152 ceph-mon[93456]: pgmap v1903: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.4 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 43 MiB/s wr, 682 op/s; 957/266142 objects misplaced (0.360%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:53:13.713 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.755 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:13.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:13.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 67 16 94162 94146 87.8096 61.6875 0.0163073 0.0113829 2024-08-17T17:53:13.856 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:13.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:13 smithi142 ceph-mon[103614]: pgmap v1903: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.4 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 43 MiB/s wr, 682 op/s; 957/266142 objects misplaced (0.360%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:53:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:13 smithi142 ceph-mon[106729]: pgmap v1903: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.4 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 43 MiB/s wr, 682 op/s; 957/266142 objects misplaced (0.360%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:53:13.893 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:13.995 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:14.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:14.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:14.126 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:14.160 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:14.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:14.262 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:14.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:14.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:14.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:14.449 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:14.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:14.557 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:53:14.558 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:14.584 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:53:14.585 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.597 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:14.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:14.613 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:53:14.613 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:14.641 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:53:14.642 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:14.642 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:14 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3324624261' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:14.642 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:14 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3324624261' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:14.671 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:53:14.671 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:14.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:14.711 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:53:14.712 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:14.730 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:14.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:14.743 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:53:14.743 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:14.744 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:53:14.765 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:14 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3324624261' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:14.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 68 16 95296 95280 87.5605 70.875 0.00896963 0.0114158 2024-08-17T17:53:14.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:14.867 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:14.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:15.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:15.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:15.105 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:15.149 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:15.240 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:53:15.240 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:53:15.240 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:53:15.240 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:53:15.240 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:53:15.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:15.298 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:15.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:15.439 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:15.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:15.540 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:15.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:15.634 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:15.638 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-17T17:53:15.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-08-17T17:53:15.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:15.713 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:15.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 69 16 96405 96389 87.2959 69.3125 0.0270576 0.0114503 2024-08-17T17:53:15.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.853 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:15.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:15 smithi142 ceph-mon[106729]: from='client.57763 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:15.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:15 smithi142 ceph-mon[106729]: pgmap v1904: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 645/280353 objects misplaced (0.230%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:53:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:15 smithi142 ceph-mon[103614]: from='client.57763 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:15.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:15 smithi142 ceph-mon[103614]: pgmap v1904: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 645/280353 objects misplaced (0.230%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:53:15.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:15.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:15.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:15 smithi152 ceph-mon[93456]: from='client.57763 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:15 smithi152 ceph-mon[93456]: pgmap v1904: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 645/280353 objects misplaced (0.230%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:53:16.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:16.120 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:53:16.156 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:53:16.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:16.268 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:16.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:16.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:16.408 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:16.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:16.510 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.548 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:16.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:16.562 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:16.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:16.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:16.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:53:16.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:16 smithi142 ceph-mon[106729]: pgmap v1905: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s; 645/282315 objects misplaced (0.228%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T17:53:16.770 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:16 smithi142 ceph-mon[106729]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-17T17:53:16.770 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:16 smithi142 ceph-mon[103614]: pgmap v1905: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s; 645/282315 objects misplaced (0.228%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T17:53:16.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:16 smithi142 ceph-mon[103614]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-17T17:53:16.777 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 70 16 97539 97523 87.0613 70.875 0.0197828 0.0114809 2024-08-17T17:53:16.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:16.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:16.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:16.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:16.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:17.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:17.075 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:16 smithi152 ceph-mon[93456]: pgmap v1905: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s; 645/282315 objects misplaced (0.228%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T17:53:17.076 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:16 smithi152 ceph-mon[93456]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-17T17:53:17.130 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:17.228 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:53:17.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.268 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:17.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:17.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:17.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:17.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:17.545 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:17.647 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:17.681 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:17.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 71 16 98604 98588 86.7725 66.5625 0.0449326 0.0115183 2024-08-17T17:53:17.783 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:17.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:17.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:17.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:17.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:17.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:18.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.099 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:18.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:18.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:53:18.199 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:18.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:18.336 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:18.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:18.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:18.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.518 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:18.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:18.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:18.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:18.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:18.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:18.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 72 16 99734 99718 86.5482 70.625 0.00978805 0.0115499 2024-08-17T17:53:18.799 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:18.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:18.831 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:18 smithi142 ceph-mon[106729]: pgmap v1906: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.9 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 645/289461 objects misplaced (0.223%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:53:18.831 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:18 smithi142 ceph-mon[103614]: pgmap v1906: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.9 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 645/289461 objects misplaced (0.223%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:53:18.901 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:18.940 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:18.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:18 smithi152 ceph-mon[93456]: pgmap v1906: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 5.9 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 645/289461 objects misplaced (0.223%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:53:19.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:19.112 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:19.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:19.255 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:19.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:19.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:19.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:19.534 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:19.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:19.659 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:53:19.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:53:19.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253451 13.f 13.53bf25f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.632726+0000", 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.039382528, 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99935936599999997, 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253451 2024-08-17T17:53:19.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632726+0000", 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632726+0000", 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632724+0000", 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632731+0000", 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6189999999999996e-06 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632742+0000", 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1131999999999999e-05 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632749+0000", 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1040000000000001e-06 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635110+0000", 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023606259999999998 2024-08-17T17:53:19.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635117+0000", 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1760000000000004e-06 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632064+0000", 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.996947162 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632085+0000", 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1166e-05 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253457 13.f 13.93d985af (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.634981+0000", 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.037127215, 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99713874400000002, 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253457 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634981+0000", 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634981+0000", 2024-08-17T17:53:19.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634980+0000", 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634986+0000", 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7799999999999997e-06 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635000+0000", 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4596000000000001e-05 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635006+0000", 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.037e-06 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635157+0000", 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015080399999999999 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635162+0000", 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.882e-06 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632112+0000", 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99694965400000002 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632120+0000", 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4770000000000003e-06 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.687 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253461 13.1 13.5a2c6081 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.635015+0000", 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.03709367, 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99467669400000003, 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253461 2024-08-17T17:53:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635015+0000", 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635015+0000", 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635014+0000", 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635017+0000", 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8760000000000001e-06 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635023+0000", 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1149999999999999e-06 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635027+0000", 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7859999999999998e-06 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637454+0000", 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002426896 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637461+0000", 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.629679+0000", 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99221840800000005 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.629692+0000", 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2121e-05 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253465 13.f 13.5c3917bf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.692 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.636214+0000", 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.03589442099999, 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99594324999999995, 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253465 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636214+0000", 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636214+0000", 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636213+0000", 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636218+0000", 2024-08-17T17:53:19.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0590000000000002e-06 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636227+0000", 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8389999999999996e-06 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636233+0000", 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3959999999999997e-06 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637290+0000", 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010574200000000001 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637297+0000", 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5599999999999999e-06 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632147+0000", 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99485042400000001 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632157+0000", 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0331e-05 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253474 13.f 13.befb60df (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.649934+0000", 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.022174847, 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98225535799999997, 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.697 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253474 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649934+0000", 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649934+0000", 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649932+0000", 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649938+0000", 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5609999999999998e-06 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649947+0000", 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3719999999999994e-06 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649954+0000", 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7950000000000002e-06 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.650215+0000", 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026070100000000001 2024-08-17T17:53:19.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.650220+0000", 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.327e-06 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632180+0000", 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98196058399999997 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632189+0000", 2024-08-17T17:53:19.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5909999999999996e-06 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253475 13.7 13.5eda9997 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.651001+0000", 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.02110750999999, 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97973126799999999, 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.702 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253475 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651001+0000", 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651001+0000", 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651000+0000", 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651005+0000", 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5979999999999999e-06 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651014+0000", 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6319999999999997e-06 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651019+0000", 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8499999999999999e-06 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651133+0000", 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113795 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651141+0000", 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9410000000000007e-06 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630715+0000", 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97957360199999999 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630732+0000", 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7581e-05 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.707 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253477 13.7 13.4a9300f7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.652914+0000", 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.019194974, 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97785777200000001, 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253477 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652914+0000", 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652914+0000", 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652913+0000", 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652921+0000", 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2290000000000003e-06 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652936+0000", 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4642e-05 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652942+0000", 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2759999999999997e-06 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652982+0000", 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9894000000000001e-05 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652996+0000", 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4136e-05 2024-08-17T17:53:19.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630762+0000", 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97776666999999995 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630771+0000", 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8599999999999999e-06 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253479 13.f 13.4d8bcbdf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.655935+0000", 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 120.016173553, 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97635279200000002, 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253479 2024-08-17T17:53:19.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655935+0000", 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655935+0000", 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655934+0000", 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655938+0000", 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.25e-06 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655948+0000", 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0138e-05 2024-08-17T17:53:19.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655955+0000", 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2009999999999996e-06 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655989+0000", 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4171999999999997e-05 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655994+0000", 2024-08-17T17:53:19.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5609999999999998e-06 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632274+0000", 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97627963500000003 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632288+0000", 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3783e-05 2024-08-17T17:53:19.717 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:17921 14.2 14:4ae35ce4:::benchmark_data_smithi142_428506_object17920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:12.298240+0000", 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.373868427999994, 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.103513878, 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.718 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17921 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298240+0000", 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298240+0000", 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298239+0000", 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298294+0000", 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5248999999999999e-05 2024-08-17T17:53:19.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298304+0000", 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5349999999999993e-06 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298314+0000", 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0397000000000001e-05 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298356+0000", 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1950000000000003e-05 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298435+0000", 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8924999999999994e-05 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298560+0000", 2024-08-17T17:53:19.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012495400000000001 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.299215+0000", 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000654629 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.302264+0000", 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030490209999999998 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.401671+0000", 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.099407521999999998 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.401716+0000", 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4910999999999997e-05 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.401754+0000", 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7790000000000002e-05 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:17929 14.4 14:2b55cf13:::benchmark_data_smithi142_428506_object17928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:12.303506+0000", 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 67.368602487000004, 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.100531996, 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.725 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17929 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303506+0000", 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303506+0000", 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303505+0000", 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303569+0000", 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4308999999999997e-05 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303578+0000", 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7669999999999992e-06 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303582+0000", 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8020000000000002e-06 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303613+0000", 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1173000000000001e-05 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303690+0000", 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7256999999999996e-05 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:53:19.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.303778+0000", 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7805000000000004e-05 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.304428+0000", 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064954499999999996 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.304868+0000", 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000440567 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.403956+0000", 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.099087258999999997 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.403999+0000", 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3251999999999999e-05 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.404038+0000", 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9146000000000001e-05 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31202 14.4 14:2ed62245:::benchmark_data_smithi142_428506_object31201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.294152+0000", 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.377956183000002, 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16861922200000001, 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31202 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.732 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294152+0000", 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294152+0000", 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294151+0000", 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294299+0000", 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148536 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294311+0000", 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1907e-05 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294317+0000", 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5160000000000002e-06 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294364+0000", 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.71e-05 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294501+0000", 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000137026 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294644+0000", 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143466 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297581+0000", 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002937055 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445885+0000", 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14830384499999999 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.462644+0000", 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016758975999999998 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.462732+0000", 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8158999999999994e-05 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.462772+0000", 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9403999999999997e-05 2024-08-17T17:53:19.737 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31205 14.d 14:bb2a3e9e:::benchmark_data_smithi142_428506_object31204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.296230+0000", 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.375878854, 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16761810299999999, 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:53:19.738 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31205 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296230+0000", 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296230+0000", 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296228+0000", 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296275+0000", 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7293000000000003e-05 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296287+0000", 2024-08-17T17:53:19.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1579e-05 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296292+0000", 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1279999999999999e-06 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296329+0000", 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7184999999999997e-05 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296406+0000", 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6779000000000005e-05 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296487+0000", 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0684000000000002e-05 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297726+0000", 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012392690000000001 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445865+0000", 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14813903 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.463722+0000", 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017856674999999999 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.463811+0000", 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9608e-05 2024-08-17T17:53:19.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.463848+0000", 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6455999999999998e-05 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48311.0:31206 14.e e469/460 14:7831df26:::benchmark_data_smithi142_428506_object31205:head v 469'1965, mlcod=469'1963)", 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.374279408, 2024-08-17T17:53:19.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14908843099999999, 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297828+0000", 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297847+0000", 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8865e-05 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297863+0000", 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5965999999999999e-05 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297867+0000", 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.391e-06 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297906+0000", 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8538999999999998e-05 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297921+0000", 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4949999999999999e-05 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446876+0000", 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14895483500000001 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446918+0000", 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1888000000000003e-05 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48311.0:31200 14.6 e469/466 14:61488a53:::benchmark_data_smithi142_428506_object31199:head v 469'1914, mlcod=464'650)", 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.297830+0000", 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.374278736999997, 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14994484399999999, 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.749 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297830+0000", 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297830+0000", 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297864+0000", 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5240999999999997e-05 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297875+0000", 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1144e-05 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297881+0000", 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3340000000000001e-06 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297957+0000", 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6713999999999998e-05 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297975+0000", 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8131999999999999e-05 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447739+0000", 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14976406 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447775+0000", 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5163999999999997e-05 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.48311.0:31205 14.d e469/460 ondisk, result = 0)", 2024-08-17T17:53:19.754 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.373979179000003, 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14773847200000001, 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298134+0000", 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.78e-06 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298143+0000", 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2259999999999994e-06 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298147+0000", 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4700000000000004e-06 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445838+0000", 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.147690721 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445850+0000", 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1626e-05 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445868+0000", 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8291000000000002e-05 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.758 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.48311.0:31202 14.4 e469/460 ondisk, result = 0)", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.299312+0000", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.372796551999997, 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.146575443, 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299312+0000", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299312+0000", 2024-08-17T17:53:19.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299311+0000", 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299316+0000", 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1400000000000002e-06 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445780+0000", 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14646446699999999 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445789+0000", 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5399999999999996e-06 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445872+0000", 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.331e-05 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445879+0000", 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445887+0000", 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1159999999999999e-06 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.762 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(14.6 469/466 [PushOp(14:63d9078e:::benchmark_data_smithi142_428506_object11236:head, version: 464'656, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:63d9078e:::benchmark_data_smithi142_428506_object11236:head@464'656, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.300794+0000", 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.371314884999997, 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.146983531, 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300794+0000", 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300794+0000", 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300793+0000", 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300863+0000", 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0047000000000002e-05 2024-08-17T17:53:19.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445713+0000", 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14485007699999999 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445757+0000", 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4274000000000003e-05 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446965+0000", 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012074659999999999 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446972+0000", 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.058e-06 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447777+0000", 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00080517399999999997 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48311.0:31212 14.e e469/460 14:7651fe2b:::benchmark_data_smithi142_428506_object31211:head v 469'1966, mlcod=469'1963)", 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 58.369878149999998, 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14557045499999999, 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.767 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302249+0000", 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9341999999999999e-05 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445736+0000", 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.143487059 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445747+0000", 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0898e-05 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445911+0000", 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000164339 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445935+0000", 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3632e-05 2024-08-17T17:53:19.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447779+0000", 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018443400000000001 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447801+0000", 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1501e-05 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31956 14.2 14:4c283338:::benchmark_data_smithi142_428506_object31955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.914275+0000", 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 57.757833228999999, 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26473845299999998, 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31956 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914275+0000", 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914275+0000", 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914274+0000", 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914342+0000", 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8796999999999999e-05 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914355+0000", 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.243e-05 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914361+0000", 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:53:19.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914419+0000", 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7663e-05 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914529+0000", 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109883 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914647+0000", 2024-08-17T17:53:19.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011777699999999999 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.916187+0000", 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015400419999999999 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.917003+0000", 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00081600499999999996 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.178920+0000", 2024-08-17T17:53:19.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26191723099999997 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.178974+0000", 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3569999999999997e-05 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179014+0000", 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0258e-05 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.777 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31964 14.2 14:432e571c:::benchmark_data_smithi142_428506_object31963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.920722+0000", 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 57.751386867000001, 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25839342100000001, 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:53:19.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31964 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920722+0000", 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920722+0000", 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:53:19.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920721+0000", 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920766+0000", 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5485999999999998e-05 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920777+0000", 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.04e-05 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920782+0000", 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5099999999999998e-06 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920826+0000", 2024-08-17T17:53:19.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3488000000000001e-05 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920939+0000", 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112961 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.921063+0000", 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012485900000000001 2024-08-17T17:53:19.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.922398+0000", 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001334996 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.922459+0000", 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0294999999999998e-05 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:19.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179046+0000", 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25658730200000002 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179084+0000", 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8390999999999999e-05 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179115+0000", 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0679000000000002e-05 2024-08-17T17:53:19.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:53:19.785 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:53:19.785 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:53:19.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:19.786 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 73 16 100760 100744 86.241 64.125 0.00861378 0.0115901 2024-08-17T17:53:19.815 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:19 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/486220600' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:19.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:19 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:19.917 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:19.949 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:19.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:19.951 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:19 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/486220600' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:19.951 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:19 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:19.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:19 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/486220600' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:19.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:19 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.051 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:20.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:20.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:20.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:20.327 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.363 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:20.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:20.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:53:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:53:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:20.604 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.642 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:20.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:20.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:53:20.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 74 16 101929 101913 86.0628 73.0625 0.0189438 0.0116144 2024-08-17T17:53:20.813 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:20.915 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.955 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:21.057 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:21.069 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:21.069 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:53:21.069 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[103614]: osdmap e485: 8 total, 7 up, 5 in 2024-08-17T17:53:21.070 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[103614]: pgmap v1908: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 6.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 280/298842 objects misplaced (0.094%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:53:21.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:21.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:53:21.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[106729]: osdmap e485: 8 total, 7 up, 5 in 2024-08-17T17:53:21.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:20 smithi142 ceph-mon[106729]: pgmap v1908: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 6.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 280/298842 objects misplaced (0.094%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:53:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:20 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:20 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:53:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:20 smithi152 ceph-mon[93456]: osdmap e485: 8 total, 7 up, 5 in 2024-08-17T17:53:21.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:20 smithi152 ceph-mon[93456]: pgmap v1908: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 6.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 280/298842 objects misplaced (0.094%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:53:21.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:21.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:21.229 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:21.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.362 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:21.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:21.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:21.504 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:21.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:21.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:21.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:21.761 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:21.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:21.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 75 16 103062 103046 85.8594 70.8125 0.0150816 0.0116367 2024-08-17T17:53:21.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:21.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:21.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:21.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:22.034 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:53:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:53:21] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-08-17T17:53:22.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:22.168 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:22.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:22.304 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:22.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:22.437 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:22.539 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:22.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:22.676 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:22.708 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:22.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 76 16 104174 104158 85.6441 69.5 0.0106555 0.0116711 2024-08-17T17:53:22.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.851 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:22.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:22.921 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:22 smithi142 ceph-mon[106729]: pgmap v1909: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 6.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 280/298842 objects misplaced (0.094%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:53:22.921 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:22 smithi142 ceph-mon[103614]: pgmap v1909: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 6.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 280/298842 objects misplaced (0.094%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:53:22.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:22.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:22 smithi152 ceph-mon[93456]: pgmap v1909: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 87 active+clean; 6.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 68 MiB/s wr, 1.09k op/s; 280/298842 objects misplaced (0.094%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:53:23.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:23.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:23.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:23.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:23.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:23.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:23.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:moved 0 / 267 (0%) 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:avg 53.4 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:stddev 1.49666 -> 1.49666 (expected baseline 6.53605) 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:min osd.5 with 52 -> 52 pgs (0.973783 -> 0.973783 * mean) 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:max osd.1 with 56 -> 56 pgs (1.04869 -> 1.04869 * mean) 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:oload 110 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:max_change 0.05 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:max_change_osds 4 2024-08-17T17:53:23.411 INFO:teuthology.orchestra.run.smithi142.stdout:average_utilization 0.0535 2024-08-17T17:53:23.412 INFO:teuthology.orchestra.run.smithi142.stdout:overload_utilization 0.0588 2024-08-17T17:53:23.412 INFO:teuthology.orchestra.run.smithi142.stderr:no change 2024-08-17T17:53:23.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:23.521 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:23.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:23.661 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:23.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:23.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 77 16 105306 105290 85.4506 70.75 0.010613 0.0116954 2024-08-17T17:53:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.797 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:23.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:23.898 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:23.926 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:24.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:24.034 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:23 smithi142 ceph-mon[103614]: from='client.48395 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:24.035 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:23 smithi142 ceph-mon[106729]: from='client.48395 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.052 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:24.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:24.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:23 smithi152 ceph-mon[93456]: from='client.48395 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:53:24.153 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:24.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:24.253 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:24.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:24.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:24.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:24.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:24.429 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:24.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:24.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:24.556 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:24.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:24.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:24.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:24.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 78 16 105611 105595 84.5995 19.0625 0.012472 0.0118161 2024-08-17T17:53:24.787 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:24.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:24.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:24.960 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:24.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:24.988 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:25.062 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:24 smithi152 ceph-mon[93456]: osdmap e486: 8 total, 7 up, 5 in 2024-08-17T17:53:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:24 smithi152 ceph-mon[93456]: pgmap v1911: 89 pgs: 1 peering, 88 active+clean; 6.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:53:25.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:24 smithi142 ceph-mon[106729]: osdmap e486: 8 total, 7 up, 5 in 2024-08-17T17:53:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:24 smithi142 ceph-mon[106729]: pgmap v1911: 89 pgs: 1 peering, 88 active+clean; 6.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:53:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:24 smithi142 ceph-mon[103614]: osdmap e486: 8 total, 7 up, 5 in 2024-08-17T17:53:25.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:24 smithi142 ceph-mon[103614]: pgmap v1911: 89 pgs: 1 peering, 88 active+clean; 6.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:53:25.192 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:25.220 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:25.322 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:25.352 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:25.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:25.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:25.491 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:25.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.629 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:25.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:25.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:25.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:25.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 79 16 106818 106802 84.4835 75.4375 0.0109434 0.0118317 2024-08-17T17:53:25.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:25 smithi142 ceph-mon[106729]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T17:53:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:25 smithi142 ceph-mon[106729]: osdmap e487: 8 total, 7 up, 5 in 2024-08-17T17:53:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:25 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:53:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:25 smithi142 ceph-mon[103614]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T17:53:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:25 smithi142 ceph-mon[103614]: osdmap e487: 8 total, 7 up, 5 in 2024-08-17T17:53:25.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:25 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:53:25.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:25.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:26.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:26.071 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:25 smithi152 ceph-mon[93456]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T17:53:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:25 smithi152 ceph-mon[93456]: osdmap e487: 8 total, 7 up, 5 in 2024-08-17T17:53:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:25 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:53:26.172 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.206 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:26.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:26.241 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:53:26.308 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:26.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:26.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:26.453 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:26.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:26.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:26.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:26.734 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:26.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.767 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:26.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:26.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:2024-08-17T17:53:26.778973+0000 min lat: 0.00102385 max lat: 2.12417 avg lat: 0.0118599 2024-08-17T17:53:26.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:53:26.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 80 16 107917 107901 84.286 68.6875 0.0104178 0.0118599 2024-08-17T17:53:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:26 smithi152 ceph-mon[93456]: pgmap v1913: 89 pgs: 1 peering, 88 active+clean; 6.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 55 MiB/s wr, 887 op/s; 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:53:26.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:26.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:27.005 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:27.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:27.076 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:27.076 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:26 smithi142 ceph-mon[106729]: pgmap v1913: 89 pgs: 1 peering, 88 active+clean; 6.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 55 MiB/s wr, 887 op/s; 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:53:27.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:27.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:26 smithi142 ceph-mon[103614]: pgmap v1913: 89 pgs: 1 peering, 88 active+clean; 6.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 55 MiB/s wr, 887 op/s; 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:53:27.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.190 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:27.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:27.291 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:27.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:27.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:27.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:27.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:27.608 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:27.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:27.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:27.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:27.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 81 16 109057 109041 84.1249 71.25 0.0104444 0.0118826 2024-08-17T17:53:27.860 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:53:27.860 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:27.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:27.897 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:53:27.897 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:27.899 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:27.928 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:53:27.928 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:27.937 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:53:27.937 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:53:27.937 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:53:27.937 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:53:27.956 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:53:27.956 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:27.983 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:53:27.983 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:28.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:28.008 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:53:28.009 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:28.037 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:53:28.037 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:28.037 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:53:28.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.042 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:28.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:28.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:27 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/4218960758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:28.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:27 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/4218960758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:28.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:27 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/4218960758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:28.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:28.169 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:28.180 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:28.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:28.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:28.420 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:28.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:28.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:28.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:28.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:28.741 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:28.778 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 82 16 110158 110142 83.9381 68.8125 0.0348993 0.0119083 2024-08-17T17:53:28.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:28.873 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:53:28.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:28.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:28.997 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.031 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:29.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:29.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:28 smithi142 ceph-mon[106729]: pgmap v1914: 89 pgs: 1 peering, 88 active+clean; 6.5 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 59 MiB/s wr, 951 op/s; 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:53:29.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:28 smithi142 ceph-mon[103614]: pgmap v1914: 89 pgs: 1 peering, 88 active+clean; 6.5 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 59 MiB/s wr, 951 op/s; 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:53:29.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:29.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:29.200 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6] out_osds: [2, 7, 0] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:53:29.200 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:53:29.200 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:53:29.200 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:53:29.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:29.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:28 smithi152 ceph-mon[93456]: pgmap v1914: 89 pgs: 1 peering, 88 active+clean; 6.5 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 59 MiB/s wr, 951 op/s; 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:53:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.326 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:29.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:29.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:29.449 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:53:29.450 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:53:29.469 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:29.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:29.600 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:29.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:29.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:29.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:29.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 83 16 111266 111250 83.761 69.25 0.0156125 0.0119342 2024-08-17T17:53:29.863 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:29.906 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:30.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:30.046 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:30.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:30.185 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:30.287 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:30.319 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:30.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.458 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:30.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:30.558 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:53:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:53:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:30.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:30.601 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:30.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:30.731 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:30.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:30.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 84 16 112300 112284 83.5332 64.625 0.0121785 0.0119657 2024-08-17T17:53:30.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:30 smithi152 ceph-mon[93456]: pgmap v1915: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 82 MiB/s wr, 1.30k op/s; 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:53:30.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:30.873 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:30.974 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:30.990 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:30 smithi142 ceph-mon[106729]: pgmap v1915: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 82 MiB/s wr, 1.30k op/s; 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:53:30.991 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:30 smithi142 ceph-mon[103614]: pgmap v1915: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 82 MiB/s wr, 1.30k op/s; 2.2 MiB/s, 34 objects/s recovering 2024-08-17T17:53:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:31.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:31.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:31.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:31.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:31.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:31.289 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:31.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:31.390 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:31.416 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:31.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:53:31.518 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:31.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:31.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:31.689 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:31.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:31.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 85 16 113268 113252 83.2618 60.5 0.0282368 0.0120052 2024-08-17T17:53:31.791 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:31.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:53:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:53:31] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.51.0" 2024-08-17T17:53:31.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:31.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:31.972 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:53:32.068 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:32.081 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2539961150' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:32.081 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:31 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:32.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.121 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:32.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:32.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:32.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:32.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:32.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2539961150' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:32.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:31 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:32.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2539961150' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:32.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:31 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:32.406 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:32.508 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:32.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:32.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:32.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:32.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:32.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 86 16 114375 114359 83.098 69.1875 0.0101531 0.0120291 2024-08-17T17:53:32.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:32.793 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:32.794 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:32.824 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:32.863 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:32.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.979 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:32.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:33.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:33.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:33.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:33.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:33.258 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:32 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:53:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:32 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2539961150' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:32 smithi152 ceph-mon[93456]: osdmap e488: 8 total, 7 up, 5 in 2024-08-17T17:53:33.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:32 smithi152 ceph-mon[93456]: pgmap v1917: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 44 MiB/s wr, 696 op/s 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2539961150' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[106729]: osdmap e488: 8 total, 7 up, 5 in 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[106729]: pgmap v1917: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 44 MiB/s wr, 696 op/s 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2539961150' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[103614]: osdmap e488: 8 total, 7 up, 5 in 2024-08-17T17:53:33.336 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:32 smithi142 ceph-mon[103614]: pgmap v1917: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 44 MiB/s wr, 696 op/s 2024-08-17T17:53:33.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:33.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.397 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:33.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:33.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.538 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:33.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:33.639 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:33.664 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:33.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:33.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:33.779 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 87 16 115413 115397 82.8885 64.875 0.00594649 0.0120553 2024-08-17T17:53:33.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:33.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:33.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:33.930 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:33.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:34.032 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:34.063 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:34.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:34.165 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:34.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:34.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:34.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:34.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:34.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:34.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:34.554 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:34.590 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:34.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:34.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:34.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 88 16 116451 116435 82.6838 64.875 0.00861928 0.0120897 2024-08-17T17:53:34.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:34.861 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:34.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:34.913 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:34 smithi142 ceph-mon[103614]: pgmap v1918: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:53:34.913 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:34 smithi142 ceph-mon[106729]: pgmap v1918: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:53:34.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:35.002 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:35.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:34 smithi152 ceph-mon[93456]: pgmap v1918: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:53:35.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:35.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:35.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:35.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:35.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:35.415 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:35.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.551 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:35.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:35.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:35.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.685 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:35.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:35.766 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:53:35.780 INFO:tasks.radosbench.radosbench.0.smithi142.stdout: 89 16 117571 117555 82.5406 70 0.00785724 0.0121105 2024-08-17T17:53:35.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:35.826 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:35.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:35.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.969 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:35.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:36.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:36.105 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:36.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:36.244 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:36.346 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:36.391 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:36.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:36.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:36.526 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:36.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:36.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:36.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:36.759 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total time run: 90.0099 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Total writes made: 118697 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Write size: 65536 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Object size: 65536 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Bandwidth (MB/sec): 82.4194 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Bandwidth: 39.5288 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max bandwidth (MB/sec): 237.625 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average IOPS: 1318 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev IOPS: 632.466 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max IOPS: 3802 2024-08-17T17:53:36.781 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min IOPS: 0 2024-08-17T17:53:36.782 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Average Latency(s): 0.012129 2024-08-17T17:53:36.782 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Stddev Latency(s): 0.0363567 2024-08-17T17:53:36.782 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Max latency(s): 2.12417 2024-08-17T17:53:36.782 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Min latency(s): 0.00102385 2024-08-17T17:53:36.785 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:53:36.789 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:36.891 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:36.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:37.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:37.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:37.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:37.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:37.297 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:37.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:37.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:37.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:37.466 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:37.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:37.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:37.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:37.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:37.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[106729]: pgmap v1919: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:53:37.768 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[103614]: pgmap v1919: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:53:37.769 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:37 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:37 smithi152 ceph-mon[93456]: pgmap v1919: 89 pgs: 89 active+clean; 7.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:53:37.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:37 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:37.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:37.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:37.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:37.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:38.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.020 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:38.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:38.121 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:38.154 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:38.256 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:38.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:38.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:38.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:38.421 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:38.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:38.523 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:38.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:38.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:38.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:38.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:38.827 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:38.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:38.962 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:39.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:38 smithi152 ceph-mon[93456]: pgmap v1920: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 62 MiB/s wr, 1.06k op/s 2024-08-17T17:53:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.103 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:39.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:39.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:38 smithi142 ceph-mon[106729]: pgmap v1920: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 62 MiB/s wr, 1.06k op/s 2024-08-17T17:53:39.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:38 smithi142 ceph-mon[103614]: pgmap v1920: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 62 MiB/s wr, 1.06k op/s 2024-08-17T17:53:39.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:39.246 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:39.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:39.378 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:39.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:39.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:39.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:39.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.631 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:39.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:39.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:39.764 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:39.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:39.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:40.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:40.038 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:40.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:40.171 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:40.273 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:40.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:40.414 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:40.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:40.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:40.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:40.567 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:40.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:53:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:53:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:40.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:40.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:40.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:40.841 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:40.943 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:40.968 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:41.008 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:40 smithi152 ceph-mon[93456]: pgmap v1921: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 54 MiB/s wr, 1.10k op/s 2024-08-17T17:53:41.012 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:40 smithi142 ceph-mon[106729]: pgmap v1921: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 54 MiB/s wr, 1.10k op/s 2024-08-17T17:53:41.012 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:40 smithi142 ceph-mon[103614]: pgmap v1921: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 54 MiB/s wr, 1.10k op/s 2024-08-17T17:53:41.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:41.091 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:41.193 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:41.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:41.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:41.363 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:41.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:41.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:41.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:41.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:41.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:41.646 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:41.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:41.780 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:41.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.915 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:41.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:41.957 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:53:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:53:41] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-17T17:53:42.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:42.054 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:42.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:42.192 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:42.294 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:42.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:42.420 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:53:42.422 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:53:42.446 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:42.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:42.582 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:42.609 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:42.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:42.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:42.711 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:42.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:42.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:42.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:42.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:43.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:43.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:43.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:43.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:43.210 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:53:43.210 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:43.243 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:53:43.243 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:43.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:43.269 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:53:43.270 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:43.272 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:43.272 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.285 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:43.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:43.303 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:53:43.303 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:43.327 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:53:43.327 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:43.359 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:53:43.360 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:43.386 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:43.390 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:53:43.390 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:43.393 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:53:43.418 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:43.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:43.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:43.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:43.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:43.768 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:43.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:43 smithi152 ceph-mon[93456]: from='client.48413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:43 smithi152 ceph-mon[93456]: pgmap v1922: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 96 B/s rd, 51 MiB/s wr, 1.04k op/s 2024-08-17T17:53:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:43 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/41823465' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:43.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:43 smithi142 ceph-mon[106729]: from='client.48413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:43 smithi142 ceph-mon[106729]: pgmap v1922: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 96 B/s rd, 51 MiB/s wr, 1.04k op/s 2024-08-17T17:53:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:43 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/41823465' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:43 smithi142 ceph-mon[103614]: from='client.48413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:43 smithi142 ceph-mon[103614]: pgmap v1922: 89 pgs: 89 active+clean; 7.1 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 96 B/s rd, 51 MiB/s wr, 1.04k op/s 2024-08-17T17:53:43.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:43 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/41823465' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:43.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:43.962 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:44.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:44.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:44.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:44.194 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:44.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:44.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:44.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:53:44.358 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:44.460 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:44.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:44.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:44.635 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:44.737 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:44.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:44.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:53:44.802 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:53:44.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:44.912 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:45.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:45.050 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:45.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:45.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:44 smithi152 ceph-mon[93456]: pgmap v1923: 89 pgs: 89 active+clean; 6.8 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.36k op/s 2024-08-17T17:53:45.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:44 smithi142 ceph-mon[103614]: pgmap v1923: 89 pgs: 89 active+clean; 6.8 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.36k op/s 2024-08-17T17:53:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:44 smithi142 ceph-mon[106729]: pgmap v1923: 89 pgs: 89 active+clean; 6.8 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.36k op/s 2024-08-17T17:53:45.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:45.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:45.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:45.327 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:45.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:45.463 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:45.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:45.604 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:45.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:45.679 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":1921,"stamp":"2024-08-17T17:53:40.560943+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7623022565,"num_objects":116336,"num_object_clones":0,"num_object_copies":349008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116336,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":121364,"num_write_kb":7599056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9857,"num_bytes_recovered":645807138,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90493,"ondisk_log_size":90493,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":28308968,"kb_used_data":22428480,"kb_used_omap":32,"kb_used_meta":5880223,"kb_avail":440416792,"statfs":{"total":479975178240,"available":450986795008,"internally_reserved":0,"allocated":22966763520,"data_stored":22953660336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33479,"internal_metadata":6021348665},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":411500568,"num_objects":6280,"num_object_clones":0,"num_object_copies":18840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6280,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11056,"num_write_kb":554689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002499"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":571,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.690868+0000","last_change":"2024-08-17T17:49:32.579092+0000","last_active":"2024-08-17T17:53:24.690868+0000","last_peered":"2024-08-17T17:53:24.690868+0000","last_clean":"2024-08-17T17:53:24.690868+0000","last_became_active":"2024-08-17T17:49:32.578298+0000","last_became_peered":"2024-08-17T17:49:32.578298+0000","last_unstale":"2024-08-17T17:53:24.690868+0000","last_undegraded":"2024-08-17T17:53:24.690868+0000","last_fullsized":"2024-08-17T17:53:24.690868+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:42.168553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":230,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673991+0000","last_change":"2024-08-17T17:52:14.041411+0000","last_active":"2024-08-17T17:53:24.673991+0000","last_peered":"2024-08-17T17:53:24.673991+0000","last_clean":"2024-08-17T17:53:24.673991+0000","last_became_active":"2024-08-17T17:52:14.028369+0000","last_became_peered":"2024-08-17T17:52:14.028369+0000","last_unstale":"2024-08-17T17:53:24.673991+0000","last_undegraded":"2024-08-17T17:53:24.673991+0000","last_fullsized":"2024-08-17T17:53:24.673991+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:07.705463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":408,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.692230+0000","last_change":"2024-08-17T17:51:20.644166+0000","last_active":"2024-08-17T17:53:24.692230+0000","last_peered":"2024-08-17T17:53:24.692230+0000","last_clean":"2024-08-17T17:53:24.692230+0000","last_became_active":"2024-08-17T17:51:20.642200+0000","last_became_peered":"2024-08-17T17:51:20.642200+0000","last_unstale":"2024-08-17T17:53:24.692230+0000","last_undegraded":"2024-08-17T17:53:24.692230+0000","last_fullsized":"2024-08-17T17:53:24.692230+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:55.769398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":464,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.980284+0000","last_change":"2024-08-17T17:52:14.027776+0000","last_active":"2024-08-17T17:53:31.980284+0000","last_peered":"2024-08-17T17:53:31.980284+0000","last_clean":"2024-08-17T17:53:31.980284+0000","last_became_active":"2024-08-17T17:52:14.027413+0000","last_became_peered":"2024-08-17T17:52:14.027413+0000","last_unstale":"2024-08-17T17:53:31.980284+0000","last_undegraded":"2024-08-17T17:53:31.980284+0000","last_fullsized":"2024-08-17T17:53:31.980284+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:13:00.739156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":717,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.984062+0000","last_change":"2024-08-17T17:52:14.024678+0000","last_active":"2024-08-17T17:53:31.984062+0000","last_peered":"2024-08-17T17:53:31.984062+0000","last_clean":"2024-08-17T17:53:31.984062+0000","last_became_active":"2024-08-17T17:52:14.024229+0000","last_became_peered":"2024-08-17T17:52:14.024229+0000","last_unstale":"2024-08-17T17:53:31.984062+0000","last_undegraded":"2024-08-17T17:53:31.984062+0000","last_fullsized":"2024-08-17T17:53:31.984062+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:48.146246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":367,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673826+0000","last_change":"2024-08-17T17:52:14.027666+0000","last_active":"2024-08-17T17:53:24.673826+0000","last_peered":"2024-08-17T17:53:24.673826+0000","last_clean":"2024-08-17T17:53:24.673826+0000","last_became_active":"2024-08-17T17:52:14.027304+0000","last_became_peered":"2024-08-17T17:52:14.027304+0000","last_unstale":"2024-08-17T17:53:24.673826+0000","last_undegraded":"2024-08-17T17:53:24.673826+0000","last_fullsized":"2024-08-17T17:53:24.673826+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:42.148094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":803,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.983627+0000","last_change":"2024-08-17T17:51:20.647775+0000","last_active":"2024-08-17T17:53:31.983627+0000","last_peered":"2024-08-17T17:53:31.983627+0000","last_clean":"2024-08-17T17:53:31.983627+0000","last_became_active":"2024-08-17T17:51:20.647325+0000","last_became_peered":"2024-08-17T17:51:20.647325+0000","last_unstale":"2024-08-17T17:53:31.983627+0000","last_undegraded":"2024-08-17T17:53:31.983627+0000","last_fullsized":"2024-08-17T17:53:31.983627+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:11.898880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":502,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.669498+0000","last_change":"2024-08-17T17:52:14.044135+0000","last_active":"2024-08-17T17:53:24.669498+0000","last_peered":"2024-08-17T17:53:24.669498+0000","last_clean":"2024-08-17T17:53:24.669498+0000","last_became_active":"2024-08-17T17:52:14.026557+0000","last_became_peered":"2024-08-17T17:52:14.026557+0000","last_unstale":"2024-08-17T17:53:24.669498+0000","last_undegraded":"2024-08-17T17:53:24.669498+0000","last_fullsized":"2024-08-17T17:53:24.669498+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:06:56.734524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"488'7673","reported_seq":7709,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.750821+0000","last_change":"2024-08-17T17:52:00.665773+0000","last_active":"2024-08-17T17:53:38.750821+0000","last_peered":"2024-08-17T17:53:38.750821+0000","last_clean":"2024-08-17T17:53:38.750821+0000","last_became_active":"2024-08-17T17:52:00.665505+0000","last_became_peered":"2024-08-17T17:52:00.665505+0000","last_unstale":"2024-08-17T17:53:38.750821+0000","last_undegraded":"2024-08-17T17:53:38.750821+0000","last_fullsized":"2024-08-17T17:53:38.750821+0000","mapping_epoch":460,"log_start":"469'1900","ondisk_log_start":"469'1900","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:18.113786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479526912,"num_objects":7317,"num_object_clones":0,"num_object_copies":21951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7673,"num_write_kb":479680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":207,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674545+0000","last_change":"2024-08-17T17:52:14.038229+0000","last_active":"2024-08-17T17:53:24.674545+0000","last_peered":"2024-08-17T17:53:24.674545+0000","last_clean":"2024-08-17T17:53:24.674545+0000","last_became_active":"2024-08-17T17:52:14.028243+0000","last_became_peered":"2024-08-17T17:52:14.028243+0000","last_unstale":"2024-08-17T17:53:24.674545+0000","last_undegraded":"2024-08-17T17:53:24.674545+0000","last_fullsized":"2024-08-17T17:53:24.674545+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:59.144035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":391,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.690122+0000","last_change":"2024-08-17T17:51:20.646925+0000","last_active":"2024-08-17T17:53:24.690122+0000","last_peered":"2024-08-17T17:53:24.690122+0000","last_clean":"2024-08-17T17:53:24.690122+0000","last_became_active":"2024-08-17T17:51:20.646619+0000","last_became_peered":"2024-08-17T17:51:20.646619+0000","last_unstale":"2024-08-17T17:53:24.690122+0000","last_undegraded":"2024-08-17T17:53:24.690122+0000","last_fullsized":"2024-08-17T17:53:24.690122+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:30.984107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"488'7638","reported_seq":10203,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.280718+0000","last_change":"2024-08-17T17:52:51.848854+0000","last_active":"2024-08-17T17:53:38.280718+0000","last_peered":"2024-08-17T17:53:38.280718+0000","last_clean":"2024-08-17T17:53:38.280718+0000","last_became_active":"2024-08-17T17:52:51.842125+0000","last_became_peered":"2024-08-17T17:52:51.842125+0000","last_unstale":"2024-08-17T17:53:38.280718+0000","last_undegraded":"2024-08-17T17:53:38.280718+0000","last_fullsized":"2024-08-17T17:53:38.280718+0000","mapping_epoch":480,"log_start":"469'2000","ondisk_log_start":"469'2000","created":460,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:55:11.340269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483655680,"num_objects":7380,"num_object_clones":0,"num_object_copies":22140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7638,"num_write_kb":480576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1243,"num_bytes_recovered":81461248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":209,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674562+0000","last_change":"2024-08-17T17:52:14.036602+0000","last_active":"2024-08-17T17:53:24.674562+0000","last_peered":"2024-08-17T17:53:24.674562+0000","last_clean":"2024-08-17T17:53:24.674562+0000","last_became_active":"2024-08-17T17:52:14.028271+0000","last_became_peered":"2024-08-17T17:52:14.028271+0000","last_unstale":"2024-08-17T17:53:24.674562+0000","last_undegraded":"2024-08-17T17:53:24.674562+0000","last_fullsized":"2024-08-17T17:53:24.674562+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:20.275217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":459,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673726+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:53:24.673726+0000","last_peered":"2024-08-17T17:53:24.673726+0000","last_clean":"2024-08-17T17:53:24.673726+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:53:24.673726+0000","last_undegraded":"2024-08-17T17:53:24.673726+0000","last_fullsized":"2024-08-17T17:53:24.673726+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"488'7838","reported_seq":10358,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:40.428733+0000","last_change":"2024-08-17T17:52:38.719475+0000","last_active":"2024-08-17T17:53:40.428733+0000","last_peered":"2024-08-17T17:53:40.428733+0000","last_clean":"2024-08-17T17:53:40.428733+0000","last_became_active":"2024-08-17T17:52:38.686561+0000","last_became_peered":"2024-08-17T17:52:38.686561+0000","last_unstale":"2024-08-17T17:53:40.428733+0000","last_undegraded":"2024-08-17T17:53:40.428733+0000","last_fullsized":"2024-08-17T17:53:40.428733+0000","mapping_epoch":476,"log_start":"469'2000","ondisk_log_start":"469'2000","created":460,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:45.634418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475660288,"num_objects":7258,"num_object_clones":0,"num_object_copies":21774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7838,"num_write_kb":483072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1231,"num_bytes_recovered":80674816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":208,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.692131+0000","last_change":"2024-08-17T17:51:20.644339+0000","last_active":"2024-08-17T17:53:24.692131+0000","last_peered":"2024-08-17T17:53:24.692131+0000","last_clean":"2024-08-17T17:53:24.692131+0000","last_became_active":"2024-08-17T17:51:20.643714+0000","last_became_peered":"2024-08-17T17:51:20.643714+0000","last_unstale":"2024-08-17T17:53:24.692131+0000","last_undegraded":"2024-08-17T17:53:24.692131+0000","last_fullsized":"2024-08-17T17:53:24.692131+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:53.278424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":449,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.669923+0000","last_change":"2024-08-17T17:51:20.638436+0000","last_active":"2024-08-17T17:53:24.669923+0000","last_peered":"2024-08-17T17:53:24.669923+0000","last_clean":"2024-08-17T17:53:24.669923+0000","last_became_active":"2024-08-17T17:51:20.638097+0000","last_became_peered":"2024-08-17T17:51:20.638097+0000","last_unstale":"2024-08-17T17:53:24.669923+0000","last_undegraded":"2024-08-17T17:53:24.669923+0000","last_fullsized":"2024-08-17T17:53:24.669923+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:38:04.339476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"488'7491","reported_seq":7558,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.256394+0000","last_change":"2024-08-17T17:52:00.665375+0000","last_active":"2024-08-17T17:53:38.256394+0000","last_peered":"2024-08-17T17:53:38.256394+0000","last_clean":"2024-08-17T17:53:38.256394+0000","last_became_active":"2024-08-17T17:52:00.664950+0000","last_became_peered":"2024-08-17T17:52:00.664950+0000","last_unstale":"2024-08-17T17:53:38.256394+0000","last_undegraded":"2024-08-17T17:53:38.256394+0000","last_fullsized":"2024-08-17T17:53:38.256394+0000","mapping_epoch":460,"log_start":"469'1900","ondisk_log_start":"469'1900","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:37:10.495090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473890816,"num_objects":7231,"num_object_clones":0,"num_object_copies":21693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":471104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":747,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.982478+0000","last_change":"2024-08-17T17:51:20.661365+0000","last_active":"2024-08-17T17:53:31.982478+0000","last_peered":"2024-08-17T17:53:31.982478+0000","last_clean":"2024-08-17T17:53:31.982478+0000","last_became_active":"2024-08-17T17:51:20.660973+0000","last_became_peered":"2024-08-17T17:51:20.660973+0000","last_unstale":"2024-08-17T17:53:31.982478+0000","last_undegraded":"2024-08-17T17:53:31.982478+0000","last_fullsized":"2024-08-17T17:53:31.982478+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:17.967458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.979136+0000","last_change":"2024-08-17T17:52:14.028797+0000","last_active":"2024-08-17T17:53:31.979136+0000","last_peered":"2024-08-17T17:53:31.979136+0000","last_clean":"2024-08-17T17:53:31.979136+0000","last_became_active":"2024-08-17T17:52:14.028090+0000","last_became_peered":"2024-08-17T17:52:14.028090+0000","last_unstale":"2024-08-17T17:53:31.979136+0000","last_undegraded":"2024-08-17T17:53:31.979136+0000","last_fullsized":"2024-08-17T17:53:31.979136+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:33.905768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"488'7491","reported_seq":9931,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.986796+0000","last_change":"2024-08-17T17:53:22.970424+0000","last_active":"2024-08-17T17:53:38.986796+0000","last_peered":"2024-08-17T17:53:38.986796+0000","last_clean":"2024-08-17T17:53:38.986796+0000","last_became_active":"2024-08-17T17:52:25.843688+0000","last_became_peered":"2024-08-17T17:52:25.843688+0000","last_unstale":"2024-08-17T17:53:38.986796+0000","last_undegraded":"2024-08-17T17:53:38.986796+0000","last_fullsized":"2024-08-17T17:53:38.986796+0000","mapping_epoch":471,"log_start":"469'2100","ondisk_log_start":"469'2100","created":460,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5391,"log_dups_size":0,"ondisk_log_size":5391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:17.229665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":469434368,"num_objects":7163,"num_object_clones":0,"num_object_copies":21489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":468928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1179,"num_bytes_recovered":77266944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":92,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673944+0000","last_change":"2024-08-17T17:52:14.045289+0000","last_active":"2024-08-17T17:53:24.673944+0000","last_peered":"2024-08-17T17:53:24.673944+0000","last_clean":"2024-08-17T17:53:24.673944+0000","last_became_active":"2024-08-17T17:52:14.029278+0000","last_became_peered":"2024-08-17T17:52:14.029278+0000","last_unstale":"2024-08-17T17:53:24.673944+0000","last_undegraded":"2024-08-17T17:53:24.673944+0000","last_fullsized":"2024-08-17T17:53:24.673944+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:08.499760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":459,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674228+0000","last_change":"2024-08-17T17:51:20.661629+0000","last_active":"2024-08-17T17:53:24.674228+0000","last_peered":"2024-08-17T17:53:24.674228+0000","last_clean":"2024-08-17T17:53:24.674228+0000","last_became_active":"2024-08-17T17:51:20.661353+0000","last_became_peered":"2024-08-17T17:51:20.661353+0000","last_unstale":"2024-08-17T17:53:24.674228+0000","last_undegraded":"2024-08-17T17:53:24.674228+0000","last_fullsized":"2024-08-17T17:53:24.674228+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:58.549718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"488'7530","reported_seq":9962,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:40.390839+0000","last_change":"2024-08-17T17:52:26.858802+0000","last_active":"2024-08-17T17:53:40.390839+0000","last_peered":"2024-08-17T17:53:40.390839+0000","last_clean":"2024-08-17T17:53:40.390839+0000","last_became_active":"2024-08-17T17:52:26.838499+0000","last_became_peered":"2024-08-17T17:52:26.838499+0000","last_unstale":"2024-08-17T17:53:40.390839+0000","last_undegraded":"2024-08-17T17:53:40.390839+0000","last_fullsized":"2024-08-17T17:53:40.390839+0000","mapping_epoch":472,"log_start":"469'1700","ondisk_log_start":"469'1700","created":460,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5830,"log_dups_size":0,"ondisk_log_size":5830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:33.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458752000,"num_objects":7000,"num_object_clones":0,"num_object_copies":21000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7530,"num_write_kb":464960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1187,"num_bytes_recovered":77791232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":439,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.669708+0000","last_change":"2024-08-17T17:51:20.643281+0000","last_active":"2024-08-17T17:53:24.669708+0000","last_peered":"2024-08-17T17:53:24.669708+0000","last_clean":"2024-08-17T17:53:24.669708+0000","last_became_active":"2024-08-17T17:51:20.642865+0000","last_became_peered":"2024-08-17T17:51:20.642865+0000","last_unstale":"2024-08-17T17:53:24.669708+0000","last_undegraded":"2024-08-17T17:53:24.669708+0000","last_fullsized":"2024-08-17T17:53:24.669708+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:08.053447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":486,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674786+0000","last_change":"2024-08-17T17:51:20.649486+0000","last_active":"2024-08-17T17:53:24.674786+0000","last_peered":"2024-08-17T17:53:24.674786+0000","last_clean":"2024-08-17T17:53:24.674786+0000","last_became_active":"2024-08-17T17:51:20.648218+0000","last_became_peered":"2024-08-17T17:51:20.648218+0000","last_unstale":"2024-08-17T17:53:24.674786+0000","last_undegraded":"2024-08-17T17:53:24.674786+0000","last_fullsized":"2024-08-17T17:53:24.674786+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.707295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"488'7639","reported_seq":7675,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.767597+0000","last_change":"2024-08-17T17:52:00.668924+0000","last_active":"2024-08-17T17:53:38.767597+0000","last_peered":"2024-08-17T17:53:38.767597+0000","last_clean":"2024-08-17T17:53:38.767597+0000","last_became_active":"2024-08-17T17:52:00.668637+0000","last_became_peered":"2024-08-17T17:52:00.668637+0000","last_unstale":"2024-08-17T17:53:38.767597+0000","last_undegraded":"2024-08-17T17:53:38.767597+0000","last_fullsized":"2024-08-17T17:53:38.767597+0000","mapping_epoch":460,"log_start":"469'1800","ondisk_log_start":"469'1800","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5839,"log_dups_size":0,"ondisk_log_size":5839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:35.483310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479002624,"num_objects":7309,"num_object_clones":0,"num_object_copies":21927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7639,"num_write_kb":478336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":480,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674057+0000","last_change":"2024-08-17T17:33:40.637997+0000","last_active":"2024-08-17T17:53:24.674057+0000","last_peered":"2024-08-17T17:53:24.674057+0000","last_clean":"2024-08-17T17:53:24.674057+0000","last_became_active":"2024-08-17T17:33:40.637656+0000","last_became_peered":"2024-08-17T17:33:40.637656+0000","last_unstale":"2024-08-17T17:53:24.674057+0000","last_undegraded":"2024-08-17T17:53:24.674057+0000","last_fullsized":"2024-08-17T17:53:24.674057+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:09.266671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":469,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674700+0000","last_change":"2024-08-17T17:52:36.687675+0000","last_active":"2024-08-17T17:53:24.674700+0000","last_peered":"2024-08-17T17:53:24.674700+0000","last_clean":"2024-08-17T17:53:24.674700+0000","last_became_active":"2024-08-17T17:52:36.687308+0000","last_became_peered":"2024-08-17T17:52:36.687308+0000","last_unstale":"2024-08-17T17:53:24.674700+0000","last_undegraded":"2024-08-17T17:53:24.674700+0000","last_fullsized":"2024-08-17T17:53:24.674700+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:53.939562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"488'7479","reported_seq":10000,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:37.621866+0000","last_change":"2024-08-17T17:52:37.685439+0000","last_active":"2024-08-17T17:53:37.621866+0000","last_peered":"2024-08-17T17:53:37.621866+0000","last_clean":"2024-08-17T17:53:37.621866+0000","last_became_active":"2024-08-17T17:52:37.680933+0000","last_became_peered":"2024-08-17T17:52:37.680933+0000","last_unstale":"2024-08-17T17:53:37.621866+0000","last_undegraded":"2024-08-17T17:53:37.621866+0000","last_fullsized":"2024-08-17T17:53:37.621866+0000","mapping_epoch":475,"log_start":"469'1800","ondisk_log_start":"469'1800","created":460,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5679,"log_dups_size":0,"ondisk_log_size":5679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:50.035815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":482541568,"num_objects":7363,"num_object_clones":0,"num_object_copies":22089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7479,"num_write_kb":474944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1232,"num_bytes_recovered":80740352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":795,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.980180+0000","last_change":"2024-08-17T17:33:40.658720+0000","last_active":"2024-08-17T17:53:31.980180+0000","last_peered":"2024-08-17T17:53:31.980180+0000","last_clean":"2024-08-17T17:53:31.980180+0000","last_became_active":"2024-08-17T17:33:40.657981+0000","last_became_peered":"2024-08-17T17:33:40.657981+0000","last_unstale":"2024-08-17T17:53:31.980180+0000","last_undegraded":"2024-08-17T17:53:31.980180+0000","last_fullsized":"2024-08-17T17:53:31.980180+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:49.400621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":200,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674481+0000","last_change":"2024-08-17T17:49:32.580175+0000","last_active":"2024-08-17T17:53:24.674481+0000","last_peered":"2024-08-17T17:53:24.674481+0000","last_clean":"2024-08-17T17:53:24.674481+0000","last_became_active":"2024-08-17T17:49:32.579415+0000","last_became_peered":"2024-08-17T17:49:32.579415+0000","last_unstale":"2024-08-17T17:53:24.674481+0000","last_undegraded":"2024-08-17T17:53:24.674481+0000","last_fullsized":"2024-08-17T17:53:24.674481+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:49:08.716633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.f","version":"488'7518","reported_seq":7585,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.260681+0000","last_change":"2024-08-17T17:52:00.667435+0000","last_active":"2024-08-17T17:53:38.260681+0000","last_peered":"2024-08-17T17:53:38.260681+0000","last_clean":"2024-08-17T17:53:38.260681+0000","last_became_active":"2024-08-17T17:52:00.667148+0000","last_became_peered":"2024-08-17T17:52:00.667148+0000","last_unstale":"2024-08-17T17:53:38.260681+0000","last_undegraded":"2024-08-17T17:53:38.260681+0000","last_fullsized":"2024-08-17T17:53:38.260681+0000","mapping_epoch":460,"log_start":"469'1900","ondisk_log_start":"469'1900","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5618,"log_dups_size":0,"ondisk_log_size":5618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:05.576028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475922432,"num_objects":7262,"num_object_clones":0,"num_object_copies":21786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7518,"num_write_kb":472960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":441,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.669271+0000","last_change":"2024-08-17T17:52:16.851952+0000","last_active":"2024-08-17T17:53:24.669271+0000","last_peered":"2024-08-17T17:53:24.669271+0000","last_clean":"2024-08-17T17:53:24.669271+0000","last_became_active":"2024-08-17T17:52:16.851541+0000","last_became_peered":"2024-08-17T17:52:16.851541+0000","last_unstale":"2024-08-17T17:53:24.669271+0000","last_undegraded":"2024-08-17T17:53:24.669271+0000","last_fullsized":"2024-08-17T17:53:24.669271+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:04.782875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":440,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.689889+0000","last_change":"2024-08-17T17:51:20.641328+0000","last_active":"2024-08-17T17:53:24.689889+0000","last_peered":"2024-08-17T17:53:24.689889+0000","last_clean":"2024-08-17T17:53:24.689889+0000","last_became_active":"2024-08-17T17:51:20.640985+0000","last_became_peered":"2024-08-17T17:51:20.640985+0000","last_unstale":"2024-08-17T17:53:24.689889+0000","last_undegraded":"2024-08-17T17:53:24.689889+0000","last_fullsized":"2024-08-17T17:53:24.689889+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:15.201005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":259,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.690001+0000","last_change":"2024-08-17T17:51:20.642519+0000","last_active":"2024-08-17T17:53:24.690001+0000","last_peered":"2024-08-17T17:53:24.690001+0000","last_clean":"2024-08-17T17:53:24.690001+0000","last_became_active":"2024-08-17T17:51:20.642141+0000","last_became_peered":"2024-08-17T17:51:20.642141+0000","last_unstale":"2024-08-17T17:53:24.690001+0000","last_undegraded":"2024-08-17T17:53:24.690001+0000","last_fullsized":"2024-08-17T17:53:24.690001+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:42:39.557563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":442,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.670335+0000","last_change":"2024-08-17T17:52:14.023247+0000","last_active":"2024-08-17T17:53:24.670335+0000","last_peered":"2024-08-17T17:53:24.670335+0000","last_clean":"2024-08-17T17:53:24.670335+0000","last_became_active":"2024-08-17T17:52:14.022868+0000","last_became_peered":"2024-08-17T17:52:14.022868+0000","last_unstale":"2024-08-17T17:53:24.670335+0000","last_undegraded":"2024-08-17T17:53:24.670335+0000","last_fullsized":"2024-08-17T17:53:24.670335+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:15:39.449145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"488'7467","reported_seq":9972,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:37.638031+0000","last_change":"2024-08-17T17:52:50.847763+0000","last_active":"2024-08-17T17:53:37.638031+0000","last_peered":"2024-08-17T17:53:37.638031+0000","last_clean":"2024-08-17T17:53:37.638031+0000","last_became_active":"2024-08-17T17:52:50.841267+0000","last_became_peered":"2024-08-17T17:52:50.841267+0000","last_unstale":"2024-08-17T17:53:37.638031+0000","last_undegraded":"2024-08-17T17:53:37.638031+0000","last_fullsized":"2024-08-17T17:53:37.638031+0000","mapping_epoch":479,"log_start":"469'1800","ondisk_log_start":"469'1800","created":460,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5667,"log_dups_size":0,"ondisk_log_size":5667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:31:45.798814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480837632,"num_objects":7337,"num_object_clones":0,"num_object_copies":22011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7467,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1224,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":598,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.977103+0000","last_change":"2024-08-17T17:52:14.027611+0000","last_active":"2024-08-17T17:53:31.977103+0000","last_peered":"2024-08-17T17:53:31.977103+0000","last_clean":"2024-08-17T17:53:31.977103+0000","last_became_active":"2024-08-17T17:52:14.027285+0000","last_became_peered":"2024-08-17T17:52:14.027285+0000","last_unstale":"2024-08-17T17:53:31.977103+0000","last_undegraded":"2024-08-17T17:53:31.977103+0000","last_fullsized":"2024-08-17T17:53:31.977103+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:38.753886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":434,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673637+0000","last_change":"2024-08-17T17:52:14.030669+0000","last_active":"2024-08-17T17:53:24.673637+0000","last_peered":"2024-08-17T17:53:24.673637+0000","last_clean":"2024-08-17T17:53:24.673637+0000","last_became_active":"2024-08-17T17:52:14.030212+0000","last_became_peered":"2024-08-17T17:52:14.030212+0000","last_unstale":"2024-08-17T17:53:24.673637+0000","last_undegraded":"2024-08-17T17:53:24.673637+0000","last_fullsized":"2024-08-17T17:53:24.673637+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:10.650679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":231,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.691306+0000","last_change":"2024-08-17T17:52:14.021088+0000","last_active":"2024-08-17T17:53:24.691306+0000","last_peered":"2024-08-17T17:53:24.691306+0000","last_clean":"2024-08-17T17:53:24.691306+0000","last_became_active":"2024-08-17T17:52:14.020764+0000","last_became_peered":"2024-08-17T17:52:14.020764+0000","last_unstale":"2024-08-17T17:53:24.691306+0000","last_undegraded":"2024-08-17T17:53:24.691306+0000","last_fullsized":"2024-08-17T17:53:24.691306+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:16:49.630128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.d","version":"488'7462","reported_seq":7499,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.795243+0000","last_change":"2024-08-17T17:52:00.668044+0000","last_active":"2024-08-17T17:53:38.795243+0000","last_peered":"2024-08-17T17:53:38.795243+0000","last_clean":"2024-08-17T17:53:38.795243+0000","last_became_active":"2024-08-17T17:52:00.667599+0000","last_became_peered":"2024-08-17T17:52:00.667599+0000","last_unstale":"2024-08-17T17:53:38.795243+0000","last_undegraded":"2024-08-17T17:53:38.795243+0000","last_fullsized":"2024-08-17T17:53:38.795243+0000","mapping_epoch":460,"log_start":"469'1600","ondisk_log_start":"469'1600","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:18:59.053971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":471007256,"num_objects":7188,"num_object_clones":0,"num_object_copies":21564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7188,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7462,"num_write_kb":468737,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":453,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.977495+0000","last_change":"2024-08-17T17:52:14.027479+0000","last_active":"2024-08-17T17:53:31.977495+0000","last_peered":"2024-08-17T17:53:31.977495+0000","last_clean":"2024-08-17T17:53:31.977495+0000","last_became_active":"2024-08-17T17:52:14.027157+0000","last_became_peered":"2024-08-17T17:52:14.027157+0000","last_unstale":"2024-08-17T17:53:31.977495+0000","last_undegraded":"2024-08-17T17:53:31.977495+0000","last_fullsized":"2024-08-17T17:53:31.977495+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:39:49.386152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":209,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.669634+0000","last_change":"2024-08-17T17:52:14.023986+0000","last_active":"2024-08-17T17:53:24.669634+0000","last_peered":"2024-08-17T17:53:24.669634+0000","last_clean":"2024-08-17T17:53:24.669634+0000","last_became_active":"2024-08-17T17:52:14.023669+0000","last_became_peered":"2024-08-17T17:52:14.023669+0000","last_unstale":"2024-08-17T17:53:24.669634+0000","last_undegraded":"2024-08-17T17:53:24.669634+0000","last_fullsized":"2024-08-17T17:53:24.669634+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:56.702701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":392,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.691712+0000","last_change":"2024-08-17T17:52:36.689871+0000","last_active":"2024-08-17T17:53:24.691712+0000","last_peered":"2024-08-17T17:53:24.691712+0000","last_clean":"2024-08-17T17:53:24.691712+0000","last_became_active":"2024-08-17T17:52:36.689541+0000","last_became_peered":"2024-08-17T17:52:36.689541+0000","last_unstale":"2024-08-17T17:53:24.691712+0000","last_undegraded":"2024-08-17T17:53:24.691712+0000","last_fullsized":"2024-08-17T17:53:24.691712+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:57.603728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"488'7501","reported_seq":7537,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:37.579759+0000","last_change":"2024-08-17T17:52:00.665676+0000","last_active":"2024-08-17T17:53:37.579759+0000","last_peered":"2024-08-17T17:53:37.579759+0000","last_clean":"2024-08-17T17:53:37.579759+0000","last_became_active":"2024-08-17T17:52:00.665412+0000","last_became_peered":"2024-08-17T17:52:00.665412+0000","last_unstale":"2024-08-17T17:53:37.579759+0000","last_undegraded":"2024-08-17T17:53:37.579759+0000","last_fullsized":"2024-08-17T17:53:37.579759+0000","mapping_epoch":460,"log_start":"469'1800","ondisk_log_start":"469'1800","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:05.862567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483459072,"num_objects":7377,"num_object_clones":0,"num_object_copies":22131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7501,"num_write_kb":476096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":670,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.978985+0000","last_change":"2024-08-17T17:51:20.648575+0000","last_active":"2024-08-17T17:53:31.978985+0000","last_peered":"2024-08-17T17:53:31.978985+0000","last_clean":"2024-08-17T17:53:31.978985+0000","last_became_active":"2024-08-17T17:51:20.647824+0000","last_became_peered":"2024-08-17T17:51:20.647824+0000","last_unstale":"2024-08-17T17:53:31.978985+0000","last_undegraded":"2024-08-17T17:53:31.978985+0000","last_fullsized":"2024-08-17T17:53:31.978985+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:02.577961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":490,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674575+0000","last_change":"2024-08-17T17:52:14.028985+0000","last_active":"2024-08-17T17:53:24.674575+0000","last_peered":"2024-08-17T17:53:24.674575+0000","last_clean":"2024-08-17T17:53:24.674575+0000","last_became_active":"2024-08-17T17:52:14.028425+0000","last_became_peered":"2024-08-17T17:52:14.028425+0000","last_unstale":"2024-08-17T17:53:24.674575+0000","last_undegraded":"2024-08-17T17:53:24.674575+0000","last_fullsized":"2024-08-17T17:53:24.674575+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:56.050426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.670314+0000","last_change":"2024-08-17T17:51:20.645425+0000","last_active":"2024-08-17T17:53:24.670314+0000","last_peered":"2024-08-17T17:53:24.670314+0000","last_clean":"2024-08-17T17:53:24.670314+0000","last_became_active":"2024-08-17T17:51:20.645132+0000","last_became_peered":"2024-08-17T17:51:20.645132+0000","last_unstale":"2024-08-17T17:53:24.670314+0000","last_undegraded":"2024-08-17T17:53:24.670314+0000","last_fullsized":"2024-08-17T17:53:24.670314+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:43.941748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"488'7590","reported_seq":10154,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:39.014623+0000","last_change":"2024-08-17T17:53:07.925383+0000","last_active":"2024-08-17T17:53:39.014623+0000","last_peered":"2024-08-17T17:53:39.014623+0000","last_clean":"2024-08-17T17:53:39.014623+0000","last_became_active":"2024-08-17T17:53:07.917594+0000","last_became_peered":"2024-08-17T17:53:07.917594+0000","last_unstale":"2024-08-17T17:53:39.014623+0000","last_undegraded":"2024-08-17T17:53:39.014623+0000","last_fullsized":"2024-08-17T17:53:39.014623+0000","mapping_epoch":483,"log_start":"469'2200","ondisk_log_start":"469'2200","created":460,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5390,"log_dups_size":0,"ondisk_log_size":5390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:13.225171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":475136000,"num_objects":7250,"num_object_clones":0,"num_object_copies":21750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7590,"num_write_kb":474880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1243,"num_bytes_recovered":81461248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":419,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.692034+0000","last_change":"2024-08-17T17:51:20.644005+0000","last_active":"2024-08-17T17:53:24.692034+0000","last_peered":"2024-08-17T17:53:24.692034+0000","last_clean":"2024-08-17T17:53:24.692034+0000","last_became_active":"2024-08-17T17:51:20.641777+0000","last_became_peered":"2024-08-17T17:51:20.641777+0000","last_unstale":"2024-08-17T17:53:24.692034+0000","last_undegraded":"2024-08-17T17:53:24.692034+0000","last_fullsized":"2024-08-17T17:53:24.692034+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:51.469586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":32,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674024+0000","last_change":"2024-08-17T17:52:14.029912+0000","last_active":"2024-08-17T17:53:24.674024+0000","last_peered":"2024-08-17T17:53:24.674024+0000","last_clean":"2024-08-17T17:53:24.674024+0000","last_became_active":"2024-08-17T17:52:14.029317+0000","last_became_peered":"2024-08-17T17:52:14.029317+0000","last_unstale":"2024-08-17T17:53:24.674024+0000","last_undegraded":"2024-08-17T17:53:24.674024+0000","last_fullsized":"2024-08-17T17:53:24.674024+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:40.211185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":818,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.979131+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:53:31.979131+0000","last_peered":"2024-08-17T17:53:31.979131+0000","last_clean":"2024-08-17T17:53:31.979131+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:53:31.979131+0000","last_undegraded":"2024-08-17T17:53:31.979131+0000","last_fullsized":"2024-08-17T17:53:31.979131+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.9","version":"488'7475","reported_seq":7542,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:38.253889+0000","last_change":"2024-08-17T17:52:00.668222+0000","last_active":"2024-08-17T17:53:38.253889+0000","last_peered":"2024-08-17T17:53:38.253889+0000","last_clean":"2024-08-17T17:53:38.253889+0000","last_became_active":"2024-08-17T17:52:00.667938+0000","last_became_peered":"2024-08-17T17:52:00.667938+0000","last_unstale":"2024-08-17T17:53:38.253889+0000","last_undegraded":"2024-08-17T17:53:38.253889+0000","last_fullsized":"2024-08-17T17:53:38.253889+0000","mapping_epoch":460,"log_start":"469'1900","ondisk_log_start":"469'1900","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:01.682762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":476905472,"num_objects":7277,"num_object_clones":0,"num_object_copies":21831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7475,"num_write_kb":472064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":408,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.690965+0000","last_change":"2024-08-17T17:52:14.021166+0000","last_active":"2024-08-17T17:53:24.690965+0000","last_peered":"2024-08-17T17:53:24.690965+0000","last_clean":"2024-08-17T17:53:24.690965+0000","last_became_active":"2024-08-17T17:52:14.020863+0000","last_became_peered":"2024-08-17T17:52:14.020863+0000","last_unstale":"2024-08-17T17:53:24.690965+0000","last_undegraded":"2024-08-17T17:53:24.690965+0000","last_fullsized":"2024-08-17T17:53:24.690965+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:54.866611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":314,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674803+0000","last_change":"2024-08-17T17:52:14.030241+0000","last_active":"2024-08-17T17:53:24.674803+0000","last_peered":"2024-08-17T17:53:24.674803+0000","last_clean":"2024-08-17T17:53:24.674803+0000","last_became_active":"2024-08-17T17:52:14.029809+0000","last_became_peered":"2024-08-17T17:52:14.029809+0000","last_unstale":"2024-08-17T17:53:24.674803+0000","last_undegraded":"2024-08-17T17:53:24.674803+0000","last_fullsized":"2024-08-17T17:53:24.674803+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:45.205827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":550,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.689810+0000","last_change":"2024-08-17T17:52:14.022195+0000","last_active":"2024-08-17T17:53:24.689810+0000","last_peered":"2024-08-17T17:53:24.689810+0000","last_clean":"2024-08-17T17:53:24.689810+0000","last_became_active":"2024-08-17T17:52:14.021868+0000","last_became_peered":"2024-08-17T17:52:14.021868+0000","last_unstale":"2024-08-17T17:53:24.689810+0000","last_undegraded":"2024-08-17T17:53:24.689810+0000","last_fullsized":"2024-08-17T17:53:24.689810+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:13:31.516281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"488'7647","reported_seq":7714,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:39.032128+0000","last_change":"2024-08-17T17:52:00.667994+0000","last_active":"2024-08-17T17:53:39.032128+0000","last_peered":"2024-08-17T17:53:39.032128+0000","last_clean":"2024-08-17T17:53:39.032128+0000","last_became_active":"2024-08-17T17:52:00.667705+0000","last_became_peered":"2024-08-17T17:52:00.667705+0000","last_unstale":"2024-08-17T17:53:39.032128+0000","last_undegraded":"2024-08-17T17:53:39.032128+0000","last_fullsized":"2024-08-17T17:53:39.032128+0000","mapping_epoch":460,"log_start":"470'2200","ondisk_log_start":"470'2200","created":460,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5447,"log_dups_size":0,"ondisk_log_size":5447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:15:34.447631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479920128,"num_objects":7323,"num_object_clones":0,"num_object_copies":21969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7647,"num_write_kb":479040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":589,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.691622+0000","last_change":"2024-08-17T17:52:14.038643+0000","last_active":"2024-08-17T17:53:24.691622+0000","last_peered":"2024-08-17T17:53:24.691622+0000","last_clean":"2024-08-17T17:53:24.691622+0000","last_became_active":"2024-08-17T17:52:14.026859+0000","last_became_peered":"2024-08-17T17:52:14.026859+0000","last_unstale":"2024-08-17T17:53:24.691622+0000","last_undegraded":"2024-08-17T17:53:24.691622+0000","last_fullsized":"2024-08-17T17:53:24.691622+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:03:46.040346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":828,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.981189+0000","last_change":"2024-08-17T17:51:20.639615+0000","last_active":"2024-08-17T17:53:31.981189+0000","last_peered":"2024-08-17T17:53:31.981189+0000","last_clean":"2024-08-17T17:53:31.981189+0000","last_became_active":"2024-08-17T17:51:20.639330+0000","last_became_peered":"2024-08-17T17:51:20.639330+0000","last_unstale":"2024-08-17T17:53:31.981189+0000","last_undegraded":"2024-08-17T17:53:31.981189+0000","last_fullsized":"2024-08-17T17:53:31.981189+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:41.751965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":842,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.979164+0000","last_change":"2024-08-17T17:52:14.027581+0000","last_active":"2024-08-17T17:53:31.979164+0000","last_peered":"2024-08-17T17:53:31.979164+0000","last_clean":"2024-08-17T17:53:31.979164+0000","last_became_active":"2024-08-17T17:52:14.027147+0000","last_became_peered":"2024-08-17T17:52:14.027147+0000","last_unstale":"2024-08-17T17:53:31.979164+0000","last_undegraded":"2024-08-17T17:53:31.979164+0000","last_fullsized":"2024-08-17T17:53:31.979164+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:10.108108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"488'7647","reported_seq":10248,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:39.013943+0000","last_change":"2024-08-17T17:53:24.681651+0000","last_active":"2024-08-17T17:53:39.013943+0000","last_peered":"2024-08-17T17:53:39.013943+0000","last_clean":"2024-08-17T17:53:39.013943+0000","last_became_active":"2024-08-17T17:53:24.674963+0000","last_became_peered":"2024-08-17T17:53:24.674963+0000","last_unstale":"2024-08-17T17:53:39.013943+0000","last_undegraded":"2024-08-17T17:53:39.013943+0000","last_fullsized":"2024-08-17T17:53:39.013943+0000","mapping_epoch":486,"log_start":"469'2200","ondisk_log_start":"469'2200","created":460,"last_epoch_clean":487,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:51:59.636871+0000","last_clean_scrub_stamp":"2024-08-17T17:51:59.636871+0000","objects_scrubbed":0,"log_size":5447,"log_dups_size":0,"ondisk_log_size":5447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:03.247437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":476774400,"num_objects":7275,"num_object_clones":0,"num_object_copies":21825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7647,"num_write_kb":477504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1261,"num_bytes_recovered":82640896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":112,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.979985+0000","last_change":"2024-08-17T17:52:14.040438+0000","last_active":"2024-08-17T17:53:31.979985+0000","last_peered":"2024-08-17T17:53:31.979985+0000","last_clean":"2024-08-17T17:53:31.979985+0000","last_became_active":"2024-08-17T17:52:14.026447+0000","last_became_peered":"2024-08-17T17:52:14.026447+0000","last_unstale":"2024-08-17T17:53:31.979985+0000","last_undegraded":"2024-08-17T17:53:31.979985+0000","last_fullsized":"2024-08-17T17:53:31.979985+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:01.606586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":472,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.670423+0000","last_change":"2024-08-17T17:51:20.643413+0000","last_active":"2024-08-17T17:53:24.670423+0000","last_peered":"2024-08-17T17:53:24.670423+0000","last_clean":"2024-08-17T17:53:24.670423+0000","last_became_active":"2024-08-17T17:51:20.643147+0000","last_became_peered":"2024-08-17T17:51:20.643147+0000","last_unstale":"2024-08-17T17:53:24.670423+0000","last_undegraded":"2024-08-17T17:53:24.670423+0000","last_fullsized":"2024-08-17T17:53:24.670423+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:43.313148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":459,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673882+0000","last_change":"2024-08-17T17:51:20.651231+0000","last_active":"2024-08-17T17:53:24.673882+0000","last_peered":"2024-08-17T17:53:24.673882+0000","last_clean":"2024-08-17T17:53:24.673882+0000","last_became_active":"2024-08-17T17:51:20.650932+0000","last_became_peered":"2024-08-17T17:51:20.650932+0000","last_unstale":"2024-08-17T17:53:24.673882+0000","last_undegraded":"2024-08-17T17:53:24.673882+0000","last_fullsized":"2024-08-17T17:53:24.673882+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:55.890364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":459,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674316+0000","last_change":"2024-08-17T17:51:20.654600+0000","last_active":"2024-08-17T17:53:24.674316+0000","last_peered":"2024-08-17T17:53:24.674316+0000","last_clean":"2024-08-17T17:53:24.674316+0000","last_became_active":"2024-08-17T17:51:20.653501+0000","last_became_peered":"2024-08-17T17:51:20.653501+0000","last_unstale":"2024-08-17T17:53:24.674316+0000","last_undegraded":"2024-08-17T17:53:24.674316+0000","last_fullsized":"2024-08-17T17:53:24.674316+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:31.178218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":522,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674354+0000","last_change":"2024-08-17T17:52:14.034792+0000","last_active":"2024-08-17T17:53:24.674354+0000","last_peered":"2024-08-17T17:53:24.674354+0000","last_clean":"2024-08-17T17:53:24.674354+0000","last_became_active":"2024-08-17T17:52:14.022642+0000","last_became_peered":"2024-08-17T17:52:14.022642+0000","last_unstale":"2024-08-17T17:53:24.674354+0000","last_undegraded":"2024-08-17T17:53:24.674354+0000","last_fullsized":"2024-08-17T17:53:24.674354+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:51.278615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":812,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.669605+0000","last_change":"2024-08-17T17:52:36.686555+0000","last_active":"2024-08-17T17:53:24.669605+0000","last_peered":"2024-08-17T17:53:24.669605+0000","last_clean":"2024-08-17T17:53:24.669605+0000","last_became_active":"2024-08-17T17:52:36.686271+0000","last_became_peered":"2024-08-17T17:52:36.686271+0000","last_unstale":"2024-08-17T17:53:24.669605+0000","last_undegraded":"2024-08-17T17:53:24.669605+0000","last_fullsized":"2024-08-17T17:53:24.669605+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:15.945933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":376,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.982232+0000","last_change":"2024-08-17T17:52:14.037188+0000","last_active":"2024-08-17T17:53:31.982232+0000","last_peered":"2024-08-17T17:53:31.982232+0000","last_clean":"2024-08-17T17:53:31.982232+0000","last_became_active":"2024-08-17T17:52:14.023714+0000","last_became_peered":"2024-08-17T17:52:14.023714+0000","last_unstale":"2024-08-17T17:53:31.982232+0000","last_undegraded":"2024-08-17T17:53:31.982232+0000","last_fullsized":"2024-08-17T17:53:31.982232+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:37:35.866252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":392,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.691275+0000","last_change":"2024-08-17T17:33:40.651206+0000","last_active":"2024-08-17T17:53:24.691275+0000","last_peered":"2024-08-17T17:53:24.691275+0000","last_clean":"2024-08-17T17:53:24.691275+0000","last_became_active":"2024-08-17T17:33:40.650888+0000","last_became_peered":"2024-08-17T17:33:40.650888+0000","last_unstale":"2024-08-17T17:53:24.691275+0000","last_undegraded":"2024-08-17T17:53:24.691275+0000","last_fullsized":"2024-08-17T17:53:24.691275+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:14.661018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":792,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.980534+0000","last_change":"2024-08-17T17:51:20.673791+0000","last_active":"2024-08-17T17:53:31.980534+0000","last_peered":"2024-08-17T17:53:31.980534+0000","last_clean":"2024-08-17T17:53:31.980534+0000","last_became_active":"2024-08-17T17:51:20.673556+0000","last_became_peered":"2024-08-17T17:51:20.673556+0000","last_unstale":"2024-08-17T17:53:31.980534+0000","last_undegraded":"2024-08-17T17:53:31.980534+0000","last_fullsized":"2024-08-17T17:53:31.980534+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:35.726596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":546,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.691931+0000","last_change":"2024-08-17T17:52:14.026943+0000","last_active":"2024-08-17T17:53:24.691931+0000","last_peered":"2024-08-17T17:53:24.691931+0000","last_clean":"2024-08-17T17:53:24.691931+0000","last_became_active":"2024-08-17T17:52:14.026579+0000","last_became_peered":"2024-08-17T17:52:14.026579+0000","last_unstale":"2024-08-17T17:53:24.691931+0000","last_undegraded":"2024-08-17T17:53:24.691931+0000","last_fullsized":"2024-08-17T17:53:24.691931+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:15:36.124076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":786,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.979081+0000","last_change":"2024-08-17T17:51:20.673866+0000","last_active":"2024-08-17T17:53:31.979081+0000","last_peered":"2024-08-17T17:53:31.979081+0000","last_clean":"2024-08-17T17:53:31.979081+0000","last_became_active":"2024-08-17T17:51:20.673619+0000","last_became_peered":"2024-08-17T17:51:20.673619+0000","last_unstale":"2024-08-17T17:53:31.979081+0000","last_undegraded":"2024-08-17T17:53:31.979081+0000","last_fullsized":"2024-08-17T17:53:31.979081+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:42.965391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":425,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.670554+0000","last_change":"2024-08-17T17:51:20.642743+0000","last_active":"2024-08-17T17:53:24.670554+0000","last_peered":"2024-08-17T17:53:24.670554+0000","last_clean":"2024-08-17T17:53:24.670554+0000","last_became_active":"2024-08-17T17:51:20.642129+0000","last_became_peered":"2024-08-17T17:51:20.642129+0000","last_unstale":"2024-08-17T17:53:24.670554+0000","last_undegraded":"2024-08-17T17:53:24.670554+0000","last_fullsized":"2024-08-17T17:53:24.670554+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:09.593211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":32,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.690089+0000","last_change":"2024-08-17T17:52:14.024789+0000","last_active":"2024-08-17T17:53:24.690089+0000","last_peered":"2024-08-17T17:53:24.690089+0000","last_clean":"2024-08-17T17:53:24.690089+0000","last_became_active":"2024-08-17T17:52:14.024470+0000","last_became_peered":"2024-08-17T17:52:14.024470+0000","last_unstale":"2024-08-17T17:53:24.690089+0000","last_undegraded":"2024-08-17T17:53:24.690089+0000","last_fullsized":"2024-08-17T17:53:24.690089+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:45:57.403920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":530,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.983979+0000","last_change":"2024-08-17T17:52:14.028432+0000","last_active":"2024-08-17T17:53:31.983979+0000","last_peered":"2024-08-17T17:53:31.983979+0000","last_clean":"2024-08-17T17:53:31.983979+0000","last_became_active":"2024-08-17T17:52:14.028047+0000","last_became_peered":"2024-08-17T17:52:14.028047+0000","last_unstale":"2024-08-17T17:53:31.983979+0000","last_undegraded":"2024-08-17T17:53:31.983979+0000","last_fullsized":"2024-08-17T17:53:31.983979+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:38:32.673978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":814,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.977643+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:53:31.977643+0000","last_peered":"2024-08-17T17:53:31.977643+0000","last_clean":"2024-08-17T17:53:31.977643+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:53:31.977643+0000","last_undegraded":"2024-08-17T17:53:31.977643+0000","last_fullsized":"2024-08-17T17:53:31.977643+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":189,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.979806+0000","last_change":"2024-08-17T17:49:32.576719+0000","last_active":"2024-08-17T17:53:31.979806+0000","last_peered":"2024-08-17T17:53:31.979806+0000","last_clean":"2024-08-17T17:53:31.979806+0000","last_became_active":"2024-08-17T17:49:32.576409+0000","last_became_peered":"2024-08-17T17:49:32.576409+0000","last_unstale":"2024-08-17T17:53:31.979806+0000","last_undegraded":"2024-08-17T17:53:31.979806+0000","last_fullsized":"2024-08-17T17:53:31.979806+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:40:16.038780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":481,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674032+0000","last_change":"2024-08-17T17:52:14.030544+0000","last_active":"2024-08-17T17:53:24.674032+0000","last_peered":"2024-08-17T17:53:24.674032+0000","last_clean":"2024-08-17T17:53:24.674032+0000","last_became_active":"2024-08-17T17:52:14.029494+0000","last_became_peered":"2024-08-17T17:52:14.029494+0000","last_unstale":"2024-08-17T17:53:24.674032+0000","last_undegraded":"2024-08-17T17:53:24.674032+0000","last_fullsized":"2024-08-17T17:53:24.674032+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:22.443249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":470,"reported_epoch":488,"state":"active+clean","last_fresh":"2024-08-17T17:53:31.982403+0000","last_change":"2024-08-17T17:52:14.027313+0000","last_active":"2024-08-17T17:53:31.982403+0000","last_peered":"2024-08-17T17:53:31.982403+0000","last_clean":"2024-08-17T17:53:31.982403+0000","last_became_active":"2024-08-17T17:52:14.026991+0000","last_became_peered":"2024-08-17T17:52:14.026991+0000","last_unstale":"2024-08-17T17:53:31.982403+0000","last_undegraded":"2024-08-17T17:53:31.982403+0000","last_fullsized":"2024-08-17T17:53:31.982403+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:04:30.114980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":101,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674415+0000","last_change":"2024-08-17T17:52:36.689456+0000","last_active":"2024-08-17T17:53:24.674415+0000","last_peered":"2024-08-17T17:53:24.674415+0000","last_clean":"2024-08-17T17:53:24.674415+0000","last_became_active":"2024-08-17T17:52:36.689021+0000","last_became_peered":"2024-08-17T17:52:36.689021+0000","last_unstale":"2024-08-17T17:53:24.674415+0000","last_undegraded":"2024-08-17T17:53:24.674415+0000","last_fullsized":"2024-08-17T17:53:24.674415+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:39.334874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":479,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674233+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:53:24.674233+0000","last_peered":"2024-08-17T17:53:24.674233+0000","last_clean":"2024-08-17T17:53:24.674233+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:53:24.674233+0000","last_undegraded":"2024-08-17T17:53:24.674233+0000","last_fullsized":"2024-08-17T17:53:24.674233+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":342,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674256+0000","last_change":"2024-08-17T17:21:31.889877+0000","last_active":"2024-08-17T17:53:24.674256+0000","last_peered":"2024-08-17T17:53:24.674256+0000","last_clean":"2024-08-17T17:53:24.674256+0000","last_became_active":"2024-08-17T17:21:31.889329+0000","last_became_peered":"2024-08-17T17:21:31.889329+0000","last_unstale":"2024-08-17T17:53:24.674256+0000","last_undegraded":"2024-08-17T17:53:24.674256+0000","last_fullsized":"2024-08-17T17:53:24.674256+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:29.096949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":489,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673739+0000","last_change":"2024-08-17T17:52:14.029877+0000","last_active":"2024-08-17T17:53:24.673739+0000","last_peered":"2024-08-17T17:53:24.673739+0000","last_clean":"2024-08-17T17:53:24.673739+0000","last_became_active":"2024-08-17T17:52:14.029571+0000","last_became_peered":"2024-08-17T17:52:14.029571+0000","last_unstale":"2024-08-17T17:53:24.673739+0000","last_undegraded":"2024-08-17T17:53:24.673739+0000","last_fullsized":"2024-08-17T17:53:24.673739+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:00.520717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":359,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.674615+0000","last_change":"2024-08-17T17:51:20.651988+0000","last_active":"2024-08-17T17:53:24.674615+0000","last_peered":"2024-08-17T17:53:24.674615+0000","last_clean":"2024-08-17T17:53:24.674615+0000","last_became_active":"2024-08-17T17:51:20.651495+0000","last_became_peered":"2024-08-17T17:51:20.651495+0000","last_unstale":"2024-08-17T17:53:24.674615+0000","last_undegraded":"2024-08-17T17:53:24.674615+0000","last_fullsized":"2024-08-17T17:53:24.674615+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:40.749601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":560,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.689684+0000","last_change":"2024-08-17T17:52:14.026879+0000","last_active":"2024-08-17T17:53:24.689684+0000","last_peered":"2024-08-17T17:53:24.689684+0000","last_clean":"2024-08-17T17:53:24.689684+0000","last_became_active":"2024-08-17T17:52:14.026565+0000","last_became_peered":"2024-08-17T17:52:14.026565+0000","last_unstale":"2024-08-17T17:53:24.689684+0000","last_undegraded":"2024-08-17T17:53:24.689684+0000","last_fullsized":"2024-08-17T17:53:24.689684+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:57:18.198277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":71,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.670190+0000","last_change":"2024-08-17T17:51:20.647384+0000","last_active":"2024-08-17T17:53:24.670190+0000","last_peered":"2024-08-17T17:53:24.670190+0000","last_clean":"2024-08-17T17:53:24.670190+0000","last_became_active":"2024-08-17T17:51:20.646381+0000","last_became_peered":"2024-08-17T17:51:20.646381+0000","last_unstale":"2024-08-17T17:53:24.670190+0000","last_undegraded":"2024-08-17T17:53:24.670190+0000","last_fullsized":"2024-08-17T17:53:24.670190+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:09.901450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":459,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.673743+0000","last_change":"2024-08-17T17:51:20.676098+0000","last_active":"2024-08-17T17:53:24.673743+0000","last_peered":"2024-08-17T17:53:24.673743+0000","last_clean":"2024-08-17T17:53:24.673743+0000","last_became_active":"2024-08-17T17:51:20.675786+0000","last_became_peered":"2024-08-17T17:51:20.675786+0000","last_unstale":"2024-08-17T17:53:24.673743+0000","last_undegraded":"2024-08-17T17:53:24.673743+0000","last_fullsized":"2024-08-17T17:53:24.673743+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:19.690923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":250,"reported_epoch":487,"state":"active+clean","last_fresh":"2024-08-17T17:53:24.670237+0000","last_change":"2024-08-17T17:52:14.028431+0000","last_active":"2024-08-17T17:53:24.670237+0000","last_peered":"2024-08-17T17:53:24.670237+0000","last_clean":"2024-08-17T17:53:24.670237+0000","last_became_active":"2024-08-17T17:52:14.020685+0000","last_became_peered":"2024-08-17T17:52:14.020685+0000","last_unstale":"2024-08-17T17:53:24.670237+0000","last_undegraded":"2024-08-17T17:53:24.670237+0000","last_fullsized":"2024-08-17T17:53:24.670237+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:56.855538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2280,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":12130,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":163840,"data_stored":27614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41055,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":7622426648,"num_objects":116310,"num_object_clones":0,"num_object_copies":348930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116310,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121086,"num_write_kb":7596609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9800,"num_bytes_recovered":642252800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22986764288,"data_stored":22986752072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90286,"ondisk_log_size":90286,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":1159641170198,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":453,"seq":1945620185117,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4582976,"kb_used_data":4204216,"kb_used_omap":13,"kb_used_meta":378674,"kb_avail":89162176,"statfs":{"total":95995035648,"available":91302068224,"internally_reserved":0,"allocated":4305117184,"data_stored":4302481532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13340,"internal_metadata":387763172},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492291,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5540080,"kb_used_data":4575336,"kb_used_omap":7,"kb_used_meta":964728,"kb_avail":88205072,"statfs":{"total":95995035648,"available":90321993728,"internally_reserved":0,"allocated":4685144064,"data_stored":4682523108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":987881805},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986882,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052600,"kb_used_data":4255408,"kb_used_omap":1,"kb_used_meta":1797054,"kb_avail":87692552,"statfs":{"total":95995035648,"available":89797173248,"internally_reserved":0,"allocated":4357537792,"data_stored":4354910016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1840183640},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225710,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6020636,"kb_used_data":4731796,"kb_used_omap":4,"kb_used_meta":1288827,"kb_avail":87724516,"statfs":{"total":95995035648,"available":89829904384,"internally_reserved":0,"allocated":4845359104,"data_stored":4842754136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1319759296},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":515396076035,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533928,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6112676,"kb_used_data":4661724,"kb_used_omap":5,"kb_used_meta":1450938,"kb_avail":87632476,"statfs":{"total":95995035648,"available":89735655424,"internally_reserved":0,"allocated":4773605376,"data_stored":4770991544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1485760752},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":382252089905,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11636,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9541,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":159842304,"data_stored":159842304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4740939776,"data_stored":4740939776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4789370880,"data_stored":4789370880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4336193536,"data_stored":4336189464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4675211264,"data_stored":4675207192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4285206528,"data_stored":4285202456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:53:45.684 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-17T17:53:45.706 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:45.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:45.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.901 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:45.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:46.002 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:46.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.043 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:46.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:46.144 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:46.176 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:46.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:46.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:46.296 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:46.313 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:46.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:46.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:46.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:46.564 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:46.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:46.663 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:46 smithi142 ceph-mon[103614]: pgmap v1924: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.03k op/s 2024-08-17T17:53:46.664 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:46 smithi142 ceph-mon[106729]: pgmap v1924: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.03k op/s 2024-08-17T17:53:46.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:46.732 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:46.803 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:53:46.834 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:46.884 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:46.986 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:47.016 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:47.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:46 smithi152 ceph-mon[93456]: pgmap v1924: 89 pgs: 89 active+clean; 6.7 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.03k op/s 2024-08-17T17:53:47.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:47.148 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:47.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:47.250 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:47.269 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:47.279 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:47.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:47.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:47.521 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:47.550 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:47.652 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:47.683 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:47.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:47.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:47.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.828 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:47.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:47.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:47.967 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:47.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:48.069 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:48.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.104 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:48.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:48.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:48.227 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:48.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:48.353 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:48.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:48.485 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:48.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:48.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:48.624 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:48.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:48.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:48.762 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:48.864 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:48.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.897 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:48.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:48.998 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:49.030 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:49.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:48 smithi152 ceph-mon[93456]: pgmap v1925: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 85 B/s rd, 17 MiB/s wr, 1.21k op/s 2024-08-17T17:53:49.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:48 smithi142 ceph-mon[103614]: pgmap v1925: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 85 B/s rd, 17 MiB/s wr, 1.21k op/s 2024-08-17T17:53:49.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:48 smithi142 ceph-mon[106729]: pgmap v1925: 89 pgs: 89 active+clean; 6.6 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 85 B/s rd, 17 MiB/s wr, 1.21k op/s 2024-08-17T17:53:49.132 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:49.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.167 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:49.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:49.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:49.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:49.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:49.452 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:49.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:49.590 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:49.692 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.727 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:49.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:49.828 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:49.858 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:49.960 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:49.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:50.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:50.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.137 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:50.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:50.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:50.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:50.380 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:50.412 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:50.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:50.514 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:50.547 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:53:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:53:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:50.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:50.682 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:50.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:50.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:50.804 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:50 smithi142 ceph-mon[106729]: pgmap v1926: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 85 B/s rd, 8.1 MiB/s wr, 1.30k op/s 2024-08-17T17:53:50.805 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:50 smithi142 ceph-mon[103614]: pgmap v1926: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 85 B/s rd, 8.1 MiB/s wr, 1.30k op/s 2024-08-17T17:53:50.822 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:50.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:50.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:50.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:51.061 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:50 smithi152 ceph-mon[93456]: pgmap v1926: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 85 B/s rd, 8.1 MiB/s wr, 1.30k op/s 2024-08-17T17:53:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.110 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:51.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:51.211 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:51.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:51.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:51.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:51.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:51.529 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:51.630 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:51.688 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:51.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:51.836 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:51.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:51.981 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:51.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:52.035 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:53:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:53:51] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-17T17:53:52.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:52.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:52.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:52.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:52.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:52.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:52.396 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:52.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:52.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:52.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:52.680 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:52.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:52.826 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:52 smithi142 ceph-mon[106729]: pgmap v1927: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:53:52.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:52 smithi142 ceph-mon[103614]: pgmap v1927: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:53:52.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:52.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.967 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:52.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:53.068 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:52 smithi152 ceph-mon[93456]: pgmap v1927: 89 pgs: 89 active+clean; 6.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:53:53.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:53.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:53.197 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:53.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:53.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:53.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:53.357 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:53.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:53.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:53.492 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:53.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:53.622 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:53.693 INFO:teuthology.orchestra.run.smithi142.stderr:set pool 1 pgp_num to 1 2024-08-17T17:53:53.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:53.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:53.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:53.904 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:54.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:54.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:54.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1033648953' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:53:54.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1033648953' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:53:54.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1033648953' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T17:53:54.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:54.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:54.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:54.322 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:54.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:54.462 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:54.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:54.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:54.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:54.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:54.722 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:54.823 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.853 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1033648953' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:53:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:54 smithi142 ceph-mon[106729]: osdmap e489: 8 total, 7 up, 5 in 2024-08-17T17:53:54.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:54 smithi142 ceph-mon[106729]: pgmap v1929: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:53:54.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1033648953' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:54 smithi142 ceph-mon[103614]: osdmap e489: 8 total, 7 up, 5 in 2024-08-17T17:53:54.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:54 smithi142 ceph-mon[103614]: pgmap v1929: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:53:54.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:54.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1033648953' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T17:53:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:54 smithi152 ceph-mon[93456]: osdmap e489: 8 total, 7 up, 5 in 2024-08-17T17:53:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:54 smithi152 ceph-mon[93456]: pgmap v1929: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:53:55.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:55.115 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:55.217 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:55.248 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:55.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:55.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.390 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:55.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:55.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:55.515 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:55.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:55.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:55.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:55.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:55.775 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:55.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:55.909 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:55.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:56.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:56.043 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:56.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/378273829' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:55 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:56.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/378273829' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:56.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:55 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/378273829' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:56.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:55 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:56.145 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:56.177 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:56.251 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:53:56.279 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:56.305 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:56.406 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:56.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:56.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:56.570 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:56.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.703 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:56.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:56.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:56.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:56.844 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:56.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.966 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:56.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:57.067 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:56 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:53:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:56 smithi152 ceph-mon[93456]: osdmap e490: 8 total, 7 up, 5 in 2024-08-17T17:53:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:56 smithi152 ceph-mon[93456]: pgmap v1931: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:53:57.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:57.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:57.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[103614]: osdmap e490: 8 total, 7 up, 5 in 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[103614]: pgmap v1931: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[106729]: osdmap e490: 8 total, 7 up, 5 in 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:57.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:56 smithi142 ceph-mon[106729]: pgmap v1931: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:53:57.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.231 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:57.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:57.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:57.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:57.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:57.504 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:57.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.636 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:57.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:57.737 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.772 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:57.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:57.873 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:57.915 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:57.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:58.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:58.050 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:58.152 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:58.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:58.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:58.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:58.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:58.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:58.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:58.588 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:58.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:58.690 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.723 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:58.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:58.824 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:58.859 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:58.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230609 12.3 12:c39a7471:::benchmark_data_smithi142_261410_object112412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.063520+0000", 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 452.49555316599998, 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.116453664, 2024-08-17T17:53:58.980 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230609 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063520+0000", 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063520+0000", 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063521+0000", 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.75e-06 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:58.982 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063527+0000", 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.9359999999999996e-06 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063541+0000", 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4239e-05 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063549+0000", 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9240000000000007e-06 2024-08-17T17:53:58.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063690+0000", 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014042899999999999 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.063935+0000", 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024514600000000001 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:53:58.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.064101+0000", 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016561200000000001 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.067394+0000", 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032931660000000001 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.069908+0000", 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025139479999999998 2024-08-17T17:53:58.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.177193+0000", 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10728549900000001 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.177249+0000", 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5362000000000003e-05 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.179973+0000", 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027246530000000001 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230612 12.3 12:c19a6bfb:::benchmark_data_smithi142_261410_object112415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.072285+0000", 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 452.48678771200002, 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11038160299999999, 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:58.987 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230612 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072285+0000", 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072285+0000", 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072286+0000", 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2270000000000001e-06 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072291+0000", 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0209999999999999e-06 2024-08-17T17:53:58.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072302+0000", 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0749e-05 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072307+0000", 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4330000000000003e-06 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:58.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072352+0000", 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4472999999999999e-05 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072470+0000", 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000117989 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.072577+0000", 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000106969 2024-08-17T17:53:58.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.080526+0000", 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079486669999999995 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.080568+0000", 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2459000000000001e-05 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.180059+0000", 2024-08-17T17:53:58.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.099491065000000004 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.180127+0000", 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7873999999999994e-05 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.182667+0000", 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025396770000000002 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:58.993 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230613 12.3 12:ccaeed19:::benchmark_data_smithi142_261410_object112416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.075193+0000", 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 452.48387949599999, 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.111719124, 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:58.994 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230613 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075193+0000", 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075193+0000", 2024-08-17T17:53:58.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075195+0000", 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.426e-06 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075200+0000", 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7479999999999999e-06 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075215+0000", 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4907e-05 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075224+0000", 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4700000000000002e-06 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:58.997 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075579+0000", 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035526500000000001 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.075756+0000", 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017674499999999999 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.076310+0000", 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00055457499999999997 2024-08-17T17:53:58.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.080746+0000", 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0044359009999999999 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.081342+0000", 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00059561900000000003 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:58.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.182729+0000", 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.101387585 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.182790+0000", 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0915999999999997e-05 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.186912+0000", 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041219669999999998 2024-08-17T17:53:59.000 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:230624 12.3 12:c1d2b2f4:::benchmark_data_smithi142_261410_object112427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:20.089050+0000", 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 452.47002262000001, 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10048264599999999, 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.001 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 230624 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089050+0000", 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089050+0000", 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089051+0000", 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.296e-06 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089056+0000", 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.4780000000000002e-06 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089067+0000", 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1053000000000001e-05 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089073+0000", 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5829999999999997e-06 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089123+0000", 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0204000000000002e-05 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089226+0000", 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010301700000000001 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:53:59.005 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.089323+0000", 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6947000000000005e-05 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.092609+0000", 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032867740000000001 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.093048+0000", 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00043857299999999999 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.186994+0000", 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.093945852999999996 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.187055+0000", 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0851999999999999e-05 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:20.189533+0000", 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002478016 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232569 12.c 12:3177c3d4:::benchmark_data_smithi142_261410_object114372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.956407+0000", 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 448.602665968, 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.129661522, 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.008 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232569 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956407+0000", 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956407+0000", 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956408+0000", 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.672e-06 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956414+0000", 2024-08-17T17:53:59.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4600000000000002e-06 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956430+0000", 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6198e-05 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956438+0000", 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956504+0000", 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6289000000000002e-05 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956659+0000", 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015552400000000001 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.956839+0000", 2024-08-17T17:53:59.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000179607 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.961488+0000", 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0046491960000000004 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969337+0000", 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0078488870000000006 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.013 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.082814+0000", 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11347713299999999 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.082890+0000", 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5660000000000004e-05 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.086068+0000", 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031784560000000001 2024-08-17T17:53:59.014 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232573 12.3 12:c9a7ac8e:::benchmark_data_smithi142_261410_object114376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.960989+0000", 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 448.598083927, 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.13106198399999999, 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.015 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232573 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960989+0000", 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.016 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960989+0000", 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960990+0000", 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2079999999999999e-06 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.960995+0000", 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5299999999999998e-06 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.017 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.961007+0000", 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2485e-05 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.961015+0000", 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5390000000000002e-06 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.964990+0000", 2024-08-17T17:53:59.018 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0039751730000000002 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.965151+0000", 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016096400000000001 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.965285+0000", 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013430699999999999 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.019 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969150+0000", 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038647500000000001 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969445+0000", 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000295261 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089421+0000", 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11997569 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089486+0000", 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5208000000000003e-05 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.092051+0000", 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002564869 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232575 12.c 12:33878e1a:::benchmark_data_smithi142_261410_object114378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.962425+0000", 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 448.59664770299997, 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.12697347, 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232575 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.022 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962425+0000", 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962425+0000", 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962426+0000", 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2839999999999999e-06 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962432+0000", 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6930000000000004e-06 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962447+0000", 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4898e-05 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962454+0000", 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4050000000000003e-06 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962520+0000", 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.5532999999999998e-05 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962690+0000", 2024-08-17T17:53:59.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016973800000000001 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.962869+0000", 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017956800000000001 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969377+0000", 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0065079279999999996 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.969582+0000", 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020490600000000001 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.086143+0000", 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.116560921 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.086222+0000", 2024-08-17T17:53:59.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8812000000000004e-05 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089398+0000", 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0031767840000000002 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:232582 12.c 12:32c22548:::benchmark_data_smithi142_261410_object114385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:23.978609+0000", 2024-08-17T17:53:59.028 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 448.58046328099999, 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.11411640300000001, 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 232582 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978609+0000", 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978609+0000", 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978611+0000", 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3799999999999999e-06 2024-08-17T17:53:59.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978616+0000", 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7090000000000001e-06 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978627+0000", 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1813e-05 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978633+0000", 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0900000000000001e-06 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978684+0000", 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0775000000000001e-05 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978793+0000", 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000109126 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.978902+0000", 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010854 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.981735+0000", 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0028329660000000001 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:23.982069+0000", 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000334562 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089453+0000", 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10738336499999999 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.089503+0000", 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9768999999999997e-05 2024-08-17T17:53:59.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:24.092726+0000", 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0032233079999999998 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.46715.0:234936 12.c 12:3e968c93:::benchmark_data_smithi142_261410_object116739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:46:28.164896+0000", 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 444.39417706500001, 2024-08-17T17:53:59.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.170004136, 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.46715", 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/391790422", 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 234936 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164896+0000", 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164896+0000", 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164898+0000", 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.7319999999999998e-06 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164907+0000", 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1780000000000003e-06 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164928+0000", 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1078999999999999e-05 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.164938+0000", 2024-08-17T17:53:59.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0637e-05 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.170737+0000", 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.005798292 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.170824+0000", 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7584999999999996e-05 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.296651+0000", 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.125827044 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.296751+0000", 2024-08-17T17:53:59.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.9402999999999997e-05 2024-08-17T17:53:59.040 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:58 smithi142 ceph-mon[103614]: pgmap v1932: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:53:59.041 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:58 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:59.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:53:59.041 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.296980+0000", 2024-08-17T17:53:59.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022975400000000001 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.301124+0000", 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041435020000000003 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.314087+0000", 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012963638 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.042 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.331705+0000", 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.017617287999999998 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.331767+0000", 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2014000000000006e-05 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:46:28.334900+0000", 2024-08-17T17:53:59.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00313299 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:79058 13.0 e425/413 13:074b97a0:::benchmark_data_smithi142_393845_object79057:head v 425'4918, mlcod=425'4916)", 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.512369+0000", 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 327.04670336599997, 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.22531542099999999, 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.044 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512369+0000", 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512369+0000", 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.045 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512370+0000", 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1399999999999996e-07 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512390+0000", 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9884000000000001e-05 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512398+0000", 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2779999999999993e-06 2024-08-17T17:53:59.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512403+0000", 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.746e-06 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512449+0000", 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5952000000000003e-05 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.512462+0000", 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3546999999999999e-05 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737630+0000", 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.22516725400000001 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737685+0000", 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5046000000000001e-05 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.048 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:79063 13.3 13:c3aa0dd5:::benchmark_data_smithi142_393845_object79062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.513029+0000", 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 327.04604416799998, 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.23871411100000001, 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:53:59.049 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 79063 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513029+0000", 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513029+0000", 2024-08-17T17:53:59.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513030+0000", 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.234e-06 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513078+0000", 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8270999999999999e-05 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513089+0000", 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1062e-05 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513096+0000", 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7100000000000001e-06 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513146+0000", 2024-08-17T17:53:59.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0430999999999999e-05 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513269+0000", 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012280200000000001 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.513391+0000", 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000121688 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740373+0000", 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.226982397 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740414+0000", 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0717000000000003e-05 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.751637+0000", 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011222891 2024-08-17T17:53:59.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.751687+0000", 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0217999999999999e-05 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.751743+0000", 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5689999999999997e-05 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:79060 13.e e425/413 13:7e814bd9:::benchmark_data_smithi142_393845_object79059:head v 425'4941, mlcod=425'4939)", 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.514256+0000", 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 327.04481651399999, 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.228641855, 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514256+0000", 2024-08-17T17:53:59.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514256+0000", 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514257+0000", 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7400000000000002e-07 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.514312+0000", 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5068999999999997e-05 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737620+0000", 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.223307486 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737636+0000", 2024-08-17T17:53:59.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6827999999999999e-05 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737990+0000", 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00035326200000000002 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.738007+0000", 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.766e-05 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.742843+0000", 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0048356099999999997 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.742898+0000", 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5065999999999998e-05 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop_reply(client.48005.0:79063 13.3 e425/413 ondisk, result = 0)", 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:48:25.516103+0000", 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 327.04296980300001, 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.22427727, 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "started", 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516103+0000", 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516103+0000", 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516104+0000", 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8000000000000005e-07 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.516107+0000", 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6480000000000001e-06 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737618+0000", 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.221510871 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.737636+0000", 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8065000000000001e-05 2024-08-17T17:53:59.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740360+0000", 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027240670000000002 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740368+0000", 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4200000000000001e-06 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:48:25.740380+0000", 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2418999999999999e-05 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:238938 13.3 e450/448 13:cc8e84cc:::benchmark_data_smithi142_393845_object111122:head v 450'14841, mlcod=450'14838)", 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:51:06.816167+0000", 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 165.74290607399999, 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.110379175, 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.065 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.816167+0000", 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.816167+0000", 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.066 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.816167+0000", 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2600000000000001e-07 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.816173+0000", 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8000000000000004e-06 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.816185+0000", 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1343e-05 2024-08-17T17:53:59.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.816191+0000", 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7710000000000001e-06 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.817774+0000", 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0015823250000000001 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.817793+0000", 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9009e-05 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926505+0000", 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.108712104 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926546+0000", 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0997000000000001e-05 2024-08-17T17:53:59.069 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:238943 13.3 e450/448 13:c3fd4d13:::benchmark_data_smithi142_393845_object111127:head v 450'14842, mlcod=450'14838)", 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:51:06.821355+0000", 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 165.73771808399999, 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.105228127, 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821355+0000", 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821355+0000", 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821356+0000", 2024-08-17T17:53:59.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.063e-06 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821363+0000", 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.888e-06 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821374+0000", 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0991e-05 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821381+0000", 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0990000000000001e-06 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821451+0000", 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0609000000000001e-05 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821469+0000", 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7578e-05 2024-08-17T17:53:59.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926551+0000", 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.105082467 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926583+0000", 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.1432e-05 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.074 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:238934 13.e e450/413 13:7ee09db3:::benchmark_data_smithi142_393845_object111118:head v 450'14871, mlcod=450'14868)", 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:51:06.821742+0000", 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 165.73733036100001, 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10479695, 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821742+0000", 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821742+0000", 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821743+0000", 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.9899999999999999e-07 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821750+0000", 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8839999999999997e-06 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821761+0000", 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0815999999999999e-05 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821767+0000", 2024-08-17T17:53:59.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.3799999999999999e-06 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821834+0000", 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6296000000000004e-05 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.821851+0000", 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7317e-05 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926497+0000", 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.10464635899999999 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926539+0000", 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.2098999999999998e-05 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:238949 13.0 e450/444 13:0b1dac16:::benchmark_data_smithi142_393845_object111133:head v 450'14831, mlcod=450'14828)", 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:51:06.823490+0000", 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 165.735583037, 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.111251556, 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823490+0000", 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823490+0000", 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823491+0000", 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.7300000000000005e-07 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.081 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823497+0000", 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.2949999999999999e-06 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823508+0000", 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1041e-05 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823515+0000", 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9789999999999996e-06 2024-08-17T17:53:59.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823592+0000", 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6959999999999995e-05 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.823611+0000", 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9579e-05 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.934704+0000", 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.111093015 2024-08-17T17:53:59.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.934741+0000", 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6813999999999997e-05 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:238944 13.e e450/413 13:70b3ce1d:::benchmark_data_smithi142_393845_object111128:head v 450'14872, mlcod=450'14868)", 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:51:06.826044+0000", 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 165.73302863000001, 2024-08-17T17:53:59.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.112375377, 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.826044+0000", 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.826044+0000", 2024-08-17T17:53:59.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.826045+0000", 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.001e-06 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.826052+0000", 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2169999999999997e-06 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.086 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926488+0000", 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.100436083 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926501+0000", 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2371999999999999e-05 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926565+0000", 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4159999999999996e-05 2024-08-17T17:53:59.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926634+0000", 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9083000000000002e-05 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.938382+0000", 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011747937 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.938419+0000", 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7524000000000002e-05 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_repop(client.48005.0:238959 13.3 e450/448 13:c1823fc4:::benchmark_data_smithi142_393845_object111143:head v 450'14843, mlcod=450'14839)", 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:51:06.835765+0000", 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 165.723307514, 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.100528255, 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:53:59.089 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.835765+0000", 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.835765+0000", 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.835766+0000", 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.155e-06 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.835776+0000", 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1980000000000007e-06 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926541+0000", 2024-08-17T17:53:59.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.090765920999999999 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926550+0000", 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.4600000000000003e-06 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926603+0000", 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2861000000000001e-05 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.926619+0000", 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6398000000000001e-05 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.936142+0000", 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0095226489999999994 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:51:06.936293+0000", 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015161299999999999 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.093 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.094 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.094 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:53:59.094 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:53:59.094 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:53:59.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:58 smithi142 ceph-mon[106729]: pgmap v1932: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:53:59.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:58 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:59.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.095 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:59.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:59.096 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:58 smithi152 ceph-mon[93456]: pgmap v1932: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:53:59.096 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:58 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:59.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:59.225 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:59.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:59.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:59.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:59.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:59.490 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:59.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:59.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:59.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:53:59.738 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:59.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:53:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:59.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:53:59.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:59.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:53:59 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:59.986 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:53:59 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:53:59.987 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:53:59 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:00.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.053 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:00.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:00.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:00.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:00.212 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:54:00.293 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:00.365 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:54:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:54:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:00.396 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:00.497 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:00.533 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:00.635 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:00.661 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:00.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:00.803 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:00.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:00.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[106729]: pgmap v1933: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:54:00.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:54:00.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:00.997 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:00 smithi142 ceph-mon[103614]: pgmap v1933: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-08-17T17:54:01.040 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:01.070 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:01.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:01.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:54:01.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:54:01.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:01.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:00 smithi152 ceph-mon[93456]: pgmap v1933: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-08-17T17:54:01.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:01.211 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:01.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:01.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:01.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:01.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:01.455 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:01.457 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:01.498 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:54:01.499 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:01.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:01.534 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:54:01.535 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:01.554 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:54:01.554 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:01.584 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:54:01.585 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:01.614 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:54:01.615 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:01.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:01.654 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:54:01.655 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:01.657 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:01.676 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:54:01.677 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:01.677 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:54:01.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:01.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:01.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:54:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:54:01] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.51.0" 2024-08-17T17:54:01.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:01.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.925 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:01.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:02.026 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:02.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:02.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:02.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.195 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:02.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:02.251 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2475657074' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:02.252 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2475657074' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:02.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:02.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2475657074' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:02.325 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:02.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:02.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:02.465 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:02.567 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:02.606 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:02.704 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6] out_osds: [2, 7, 0] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:54:02.704 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:54:02.704 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-08-17T17:54:02.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-08-17T17:54:02.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.786 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:02.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:02.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:02.920 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:03.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:03.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:03.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:03.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:03.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:03.302 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:03.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:03 smithi152 ceph-mon[93456]: pgmap v1934: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:54:03.331 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:03.346 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:03 smithi142 ceph-mon[106729]: pgmap v1934: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:54:03.346 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:03 smithi142 ceph-mon[103614]: pgmap v1934: 89 pgs: 89 active+clean; 5.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:54:03.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:03.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:03.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:03.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:03.607 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:03.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:03.708 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:03.736 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:03.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:03.838 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:03.867 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:03.968 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:04.002 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:04.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:04.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:04.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:04.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:04.271 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:04.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:04.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:04.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:04.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:04.542 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:04.643 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:04.684 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:04.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:54:04.695 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:54:04.771 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:54:04.771 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:54:04.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:04 smithi142 ceph-mon[103614]: pgmap v1935: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:04.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:04 smithi142 ceph-mon[106729]: pgmap v1935: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:04.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:04.816 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:04.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:04.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:04.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:05.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:04 smithi152 ceph-mon[93456]: pgmap v1935: 89 pgs: 89 active+clean; 5.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:05.081 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:05.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:05.221 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:05.324 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:05.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:05.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:05.495 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:05.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:05.589 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:05.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:05.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:05.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:05.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:05.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:05.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:05.916 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:05.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:06.018 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:06.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:54:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.062 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:06.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:06.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:06.200 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:06.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:06.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:06.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:06.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:06.473 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:06.574 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:06.618 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:06.696 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:54:06.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:06.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:06.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:06.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:06 smithi142 ceph-mon[106729]: pgmap v1936: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:54:06.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:06 smithi142 ceph-mon[103614]: pgmap v1936: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:54:06.892 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:06.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:07.029 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:07.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:06 smithi152 ceph-mon[93456]: pgmap v1936: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:54:07.131 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:07.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:07.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:07.315 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:07.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:07.417 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:07.454 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:07.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:07.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:07.671 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:54:07.688 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:07.717 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:07.819 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:07.854 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:07.956 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:07.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:07.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:08.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:07 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2265473991' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:08.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:07 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:08.073 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:08.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:08.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:07 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2265473991' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:08.126 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:07 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:08.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:07 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2265473991' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:08.126 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:07 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:08.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:08.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:08.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:08.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:08.400 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:08.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:08.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:08.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:08.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:08.684 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:08.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:08.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:08.786 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:08.786 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:08.787 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[106729]: osdmap e491: 8 total, 7 up, 5 in 2024-08-17T17:54:08.787 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[106729]: pgmap v1938: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:08.787 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:08.787 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:08.787 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[103614]: osdmap e491: 8 total, 7 up, 5 in 2024-08-17T17:54:08.787 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:08 smithi142 ceph-mon[103614]: pgmap v1938: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:08.823 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:08.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:08.925 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:08.970 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:09.072 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:08 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:08 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:08 smithi152 ceph-mon[93456]: osdmap e491: 8 total, 7 up, 5 in 2024-08-17T17:54:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:08 smithi152 ceph-mon[93456]: pgmap v1938: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:09.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:09.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:09.206 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:09.236 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:09.338 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:09.370 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:09.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:09.472 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:09.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.505 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:09.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:09.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:09.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:09.743 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:09.773 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:09.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:09.911 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:09.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:10.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:10.047 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:10.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.188 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:10.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:10.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.326 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:10.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:10.428 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:10.456 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:10.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:10.562 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:10.571 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:54:10.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:54:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:54:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:10.591 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:10.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-08-17T17:54:10.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:10.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:10.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:10.854 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:10.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:10 smithi142 ceph-mon[106729]: pgmap v1939: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:10 smithi142 ceph-mon[103614]: pgmap v1939: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:10.955 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:11.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:10 smithi152 ceph-mon[93456]: pgmap v1939: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:11.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:11.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:11.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:11.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:11.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:11.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:11.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:11.552 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:11.654 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:11.686 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:11.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:11.788 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:11.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:11.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:11.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:12.014 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:12.049 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:54:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:54:11] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.51.0" 2024-08-17T17:54:12.059 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:12.104 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:12.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:12.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.249 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:12.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:12.350 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:12.375 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:12.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:12.513 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:12.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:12.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.643 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:12.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:12.744 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:12.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:12.864 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:12 smithi142 ceph-mon[106729]: pgmap v1940: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:12.865 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:12 smithi142 ceph-mon[103614]: pgmap v1940: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:12.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:12.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:12.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:13.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:13.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:13.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:12 smithi152 ceph-mon[93456]: pgmap v1940: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:13.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.182 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:13.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:13.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:13.319 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:13.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:13.444 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:54:13.444 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:13.454 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:13.481 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:54:13.482 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:13.511 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:54:13.511 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:13.541 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:54:13.542 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:13.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:13.573 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:54:13.574 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:13.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:13.603 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:54:13.604 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:13.635 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:54:13.636 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:13.636 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:54:13.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:13.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:13.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:13.865 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:13.967 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:13.996 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3166960492' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:13.996 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3166960492' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:14.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:14.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:14.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3166960492' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:14.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.151 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:14.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:14.206 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:54:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:54:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:54:14.206 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:54:14.207 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:54:14.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:14.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:14.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:14.407 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:14.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:14.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.542 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:14.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:14.643 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:14.679 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:14.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:14.817 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:14.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:14.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:14.961 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:15.063 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:14 smithi152 ceph-mon[93456]: pgmap v1941: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.25k op/s 2024-08-17T17:54:15.093 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:15.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:14 smithi142 ceph-mon[106729]: pgmap v1941: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.25k op/s 2024-08-17T17:54:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:14 smithi142 ceph-mon[103614]: pgmap v1941: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.25k op/s 2024-08-17T17:54:15.195 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:15.226 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:15.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:15.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:15.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:15.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:15.498 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:15.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:15.627 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:15.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:15.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:15.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:15.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:15.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:16.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:16.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:16.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:16.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:16.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:16.292 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:16.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:16.422 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:16.524 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:16.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:16.655 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.689 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:16.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:16.790 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:16.819 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:16.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.959 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:16.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:17.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:17.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:16 smithi152 ceph-mon[93456]: pgmap v1942: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.25k op/s 2024-08-17T17:54:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.087 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:17.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:16 smithi142 ceph-mon[106729]: pgmap v1942: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.25k op/s 2024-08-17T17:54:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:16 smithi142 ceph-mon[103614]: pgmap v1942: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.25k op/s 2024-08-17T17:54:17.189 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.217 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:17.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:17.318 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:17.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:17.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:17.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:17.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:17.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:17.717 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:17.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:17.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:17.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:17.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:17.990 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:18.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:18.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.162 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:18.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:18.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:18.302 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:18.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:18.435 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:18.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:18.568 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:18.670 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:18.702 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:18.804 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:18.840 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:18.942 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:18.972 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:19.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:18 smithi152 ceph-mon[93456]: pgmap v1943: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.39k op/s 2024-08-17T17:54:19.074 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:19.103 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:19.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:18 smithi142 ceph-mon[103614]: pgmap v1943: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.39k op/s 2024-08-17T17:54:19.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:18 smithi142 ceph-mon[106729]: pgmap v1943: 89 pgs: 89 active+clean; 4.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.39k op/s 2024-08-17T17:54:19.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:19.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:19.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:19.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:19.488 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:19.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:19.618 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:19.640 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:19.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.777 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:19.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:19.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:19.909 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:20.011 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:20.037 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:20.139 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:20.147 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:54:20.147 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:54:20.158 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:20.166 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:20.185 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:20.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:20.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:20.324 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:20.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:20.465 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:20.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:54:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:54:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:20.597 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:20.698 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:20.735 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:20.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:20.756 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:54:20.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:20.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:20.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:20.993 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:20 smithi142 ceph-mon[106729]: pgmap v1944: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s 2024-08-17T17:54:20.995 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:20 smithi142 ceph-mon[103614]: pgmap v1944: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s 2024-08-17T17:54:21.038 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:20 smithi152 ceph-mon[93456]: pgmap v1944: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s 2024-08-17T17:54:21.140 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:21.181 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:21.283 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:21.323 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:21.425 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:21.461 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:21.563 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:21.596 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:21.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:21.743 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:21.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:21.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:54:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:54:21] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.51.0" 2024-08-17T17:54:21.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:21.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:22.013 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:22.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:22.144 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:22.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:22.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:54:22.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.277 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:22.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:22.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:22.410 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:22.512 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:22.546 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:22.648 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:22.677 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:22.779 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:22.831 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:22 smithi142 ceph-mon[106729]: pgmap v1945: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:54:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:22 smithi142 ceph-mon[103614]: pgmap v1945: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:54:22.933 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:22.969 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:23.070 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:22 smithi152 ceph-mon[93456]: pgmap v1945: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:54:23.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.096 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:23.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:23.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:23.232 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:23.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:23.333 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:23.364 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:23.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:23.502 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:23.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:23.641 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:23.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:23.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:23.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:23.914 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:23.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:24.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:24.044 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:24.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:24.186 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:24.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:24.335 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:24.437 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:24.476 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:24.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:24.503 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:24.505 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:24.578 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:24.585 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:54:24.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:24.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:24.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:24.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1486823333' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:24.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:24 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:24.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:24.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1486823333' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:24.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:24 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:24.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1486823333' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:24.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:24 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.885 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:24.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:24.986 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:25.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:25.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:25.153 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:25.255 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:25.281 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:25.383 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:25.411 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:25.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:25.541 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:25.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:25.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.681 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:25.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:25.782 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:25.816 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[106729]: pgmap v1946: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.47k op/s 2024-08-17T17:54:25.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:54:25.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[106729]: osdmap e492: 8 total, 7 up, 5 in 2024-08-17T17:54:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[103614]: pgmap v1946: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.47k op/s 2024-08-17T17:54:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:54:25.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:25 smithi142 ceph-mon[103614]: osdmap e492: 8 total, 7 up, 5 in 2024-08-17T17:54:25.919 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:25.950 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:25.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:26.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:25 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:25 smithi152 ceph-mon[93456]: pgmap v1946: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.47k op/s 2024-08-17T17:54:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:25 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:54:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:25 smithi152 ceph-mon[93456]: osdmap e492: 8 total, 7 up, 5 in 2024-08-17T17:54:26.085 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:26.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-08-17T17:54:26.186 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.230 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:26.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:26.331 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:26.383 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:26.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:26.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.517 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:26.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:26.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:26.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:26.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:26.651 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:26.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:26.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:26.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:26.914 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:27.016 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:27.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:27.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:27.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:27.148 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:27.174 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:27.276 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:27.311 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:27.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:27.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:27.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:27.555 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:27.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:27.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:27.723 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:27.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.860 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:27.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:27 smithi142 ceph-mon[106729]: pgmap v1948: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:27.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:27 smithi142 ceph-mon[103614]: pgmap v1948: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:27.961 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:27.989 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:27 smithi152 ceph-mon[93456]: pgmap v1948: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:28.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:28.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:28.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:28.262 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:28.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:28.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.403 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:28.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:28.504 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:28.536 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:28.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:28.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:28.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:28.811 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:28.913 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:28.944 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:29.046 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:29.076 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:29.177 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:29.204 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:29.306 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:29.341 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:29.443 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:29.481 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:29.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:29.617 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:29.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:29.683 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:29 smithi142 ceph-mon[106729]: pgmap v1949: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:29.683 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:29 smithi142 ceph-mon[103614]: pgmap v1949: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:29.719 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:29.749 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:29.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:29.851 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:29.874 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:29 smithi152 ceph-mon[93456]: pgmap v1949: 89 pgs: 89 active+clean; 3.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:29.895 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:29.908 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: "size": 20, 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 600, 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [ 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48005.0:201805 13.0 13:0b9d7d9e:::benchmark_data_smithi142_393845_object73989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:50:34.589153+0000", 2024-08-17T17:54:29.946 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 235.35210817999999, 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0370088049999999, 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48005", 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 201805 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.947 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.589153+0000", 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.589153+0000", 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.589155+0000", 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5039999999999999e-06 2024-08-17T17:54:29.948 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.589159+0000", 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7050000000000001e-06 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.589168+0000", 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0969999999999995e-06 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.949 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.589175+0000", 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.041e-06 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.592916+0000", 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0037414689999999999 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:34.592931+0000", 2024-08-17T17:54:29.950 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4827999999999999e-05 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.622503+0000", 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.029571585 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:29.951 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.622634+0000", 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013180399999999999 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0", 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.622765+0000", 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000130572 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:29.952 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.625803+0000", 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030384750000000001 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.626005+0000", 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000201428 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.626058+0000", 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.2941e-05 2024-08-17T17:54:29.953 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:50:35.626162+0000", 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000104356 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:19436 14.8 14:160d850e:::benchmark_data_smithi142_428506_object19435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:54:29.954 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:52:13.011180+0000", 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 136.930081191, 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0759751679999998, 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "reached pg", 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 19436 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.955 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011180+0000", 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011180+0000", 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011182+0000", 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.269e-06 2024-08-17T17:54:29.956 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011222+0000", 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0698999999999998e-05 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011235+0000", 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2272e-05 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:29.957 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011242+0000", 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.5510000000000002e-06 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011296+0000", 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4187000000000001e-05 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011481+0000", 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000184568 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.958 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.011643+0000", 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000162652 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.026494+0000", 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.014850398000000001 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.959 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.026506+0000", 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2203e-05 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.012273+0000", 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98576725700000001 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.012291+0000", 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7357e-05 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.960 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.087099+0000", 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0748084659999999 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.087155+0000", 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.6289000000000003e-05 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.961 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:19443 14.8 14:171e7e71:::benchmark_data_smithi142_428506_object19442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:52:13.020585+0000", 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 136.92067691700001, 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0848200490000002, 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 19443 2024-08-17T17:54:29.962 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.020585+0000", 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.020585+0000", 2024-08-17T17:54:29.963 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.020586+0000", 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.883e-06 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.020616+0000", 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9581e-05 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.964 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.020626+0000", 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8400000000000007e-06 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.020633+0000", 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0829999999999998e-06 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.026535+0000", 2024-08-17T17:54:29.965 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059017840000000002 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.026542+0000", 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.934e-06 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.012507+0000", 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98596522900000005 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.966 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.012516+0000", 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.1949999999999992e-06 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.087630+0000", 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0751135949999999 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:29.967 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.087728+0000", 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7969999999999999e-05 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.087833+0000", 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000105596 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.088942+0000", 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001108588 2024-08-17T17:54:29.968 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.089338+0000", 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00039663300000000002 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.104675+0000", 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.015336351 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.969 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.104716+0000", 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1155999999999997e-05 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.105405+0000", 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00068863099999999999 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.970 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:19463 14.6 14:6f518124:::benchmark_data_smithi142_428506_object19462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:52:13.036380+0000", 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 136.90488141500001, 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0427457539999998, 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 19463 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.971 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036380+0000", 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036380+0000", 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036382+0000", 2024-08-17T17:54:29.972 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.514e-06 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036404+0000", 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.2523000000000001e-05 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036413+0000", 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.8610000000000004e-06 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.973 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036418+0000", 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1989999999999997e-06 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036465+0000", 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6740999999999999e-05 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036471+0000", 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1369999999999999e-06 2024-08-17T17:54:29.974 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.018670+0000", 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98219847900000001 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.018684+0000", 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4742e-05 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.975 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.072822+0000", 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0541376730000001 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.072957+0000", 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013466300000000001 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.073064+0000", 2024-08-17T17:54:29.976 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000107604 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.074215+0000", 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.001150468 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.977 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.978 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.978 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.074314+0000", 2024-08-17T17:54:29.978 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8944000000000007e-05 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.078510+0000", 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0041966310000000001 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.078597+0000", 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6348999999999996e-05 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.979 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.079126+0000", 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00052922600000000005 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:19464 14.b 14:db85b17a:::benchmark_data_smithi142_428506_object19463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:52:13.036863+0000", 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 136.90439893800001, 2024-08-17T17:54:29.980 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1175469269999998, 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 19464 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.981 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036863+0000", 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036863+0000", 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.036864+0000", 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3060000000000001e-06 2024-08-17T17:54:29.982 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.037066+0000", 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00020173899999999999 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.037077+0000", 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.188e-05 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.983 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.037085+0000", 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.233e-06 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.037140+0000", 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.5157000000000001e-05 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:13.037147+0000", 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7340000000000002e-06 2024-08-17T17:54:29.984 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.019191+0000", 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.98204392699999998 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:14.019200+0000", 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.5340000000000005e-06 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.985 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.140217+0000", 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.121016633 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.140366+0000", 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014888299999999999 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.140538+0000", 2024-08-17T17:54:29.986 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017213299999999999 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.143163+0000", 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026252850000000002 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.145468+0000", 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0023046920000000001 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.987 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.153407+0000", 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0079397180000000001 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.153468+0000", 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0665999999999998e-05 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:29.988 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:52:15.154409+0000", 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00094140700000000001 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105176 14.b 14:d539c71f:::benchmark_data_smithi142_428506_object105175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.669565+0000", 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.271696040999998, 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0180395390000001, 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.989 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105176 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669565+0000", 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.990 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669565+0000", 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669567+0000", 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.4959999999999999e-06 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.991 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669592+0000", 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4666999999999999e-05 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669608+0000", 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6472000000000001e-05 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669616+0000", 2024-08-17T17:54:29.992 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.1000000000000004e-06 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669674+0000", 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7689999999999998e-05 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.669683+0000", 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3479999999999993e-06 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.993 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.678795+0000", 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0091113030000001 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.678957+0000", 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000162783 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.679109+0000", 2024-08-17T17:54:29.994 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015181099999999999 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687429+0000", 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0083199429999999998 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687478+0000", 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9104000000000003e-05 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.995 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687519+0000", 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1190999999999999e-05 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687567+0000", 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.8072000000000003e-05 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687605+0000", 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7558999999999997e-05 2024-08-17T17:54:29.996 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105189 14.b 14:d7be77ec:::benchmark_data_smithi142_428506_object105188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.680806+0000", 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.260455796000002, 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.006983789, 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:29.997 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105189 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680806+0000", 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:29.998 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680806+0000", 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680807+0000", 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.209e-06 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680831+0000", 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.374e-05 2024-08-17T17:54:29.999 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680843+0000", 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2014999999999999e-05 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680850+0000", 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.4930000000000001e-06 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.000 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680963+0000", 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011328399999999999 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.680974+0000", 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.075e-05 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.681805+0000", 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.000830965 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.001 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.681971+0000", 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016558 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.682131+0000", 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016053000000000001 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.002 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687659+0000", 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0055273070000000004 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687694+0000", 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5162e-05 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687730+0000", 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.6353000000000002e-05 2024-08-17T17:54:30.003 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687759+0000", 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9414999999999999e-05 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687789+0000", 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9986000000000001e-05 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.004 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105234 14.b 14:d5ca0832:::benchmark_data_smithi142_428506_object105233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.718784+0000", 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.222477256999994, 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.975517097, 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.005 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105234 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718784+0000", 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718784+0000", 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.006 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718785+0000", 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6300000000000004e-07 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718807+0000", 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1566999999999999e-05 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718816+0000", 2024-08-17T17:54:30.007 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.0480000000000004e-06 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718820+0000", 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7060000000000003e-06 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718885+0000", 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.4157000000000003e-05 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.008 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.718894+0000", 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7650000000000005e-06 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.685115+0000", 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.96622099900000002 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.685231+0000", 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00011565699999999999 2024-08-17T17:54:30.009 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.685325+0000", 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.3902999999999999e-05 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687800+0000", 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.002475264 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.010 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694142+0000", 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063415299999999997 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694200+0000", 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7833e-05 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.011 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694260+0000", 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.0783999999999999e-05 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694301+0000", 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.1020999999999999e-05 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.012 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105264 14.b 14:d48e31d2:::benchmark_data_smithi142_428506_object105263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.740672+0000", 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.200589678, 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.956854172, 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.013 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105264 2024-08-17T17:54:30.014 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.014 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.014 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.014 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.014 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740672+0000", 2024-08-17T17:54:30.014 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.020 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.020 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.020 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.020 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740672+0000", 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740673+0000", 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.33e-06 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740698+0000", 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4777999999999999e-05 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.021 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740710+0000", 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2581e-05 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740718+0000", 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1679999999999998e-06 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.022 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740785+0000", 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7124000000000003e-05 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.740794+0000", 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.4180000000000003e-06 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.687859+0000", 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.94706446200000005 2024-08-17T17:54:30.023 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.688112+0000", 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00025351399999999999 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.688260+0000", 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014806699999999999 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.024 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694370+0000", 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0061093340000000001 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694383+0000", 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3183999999999999e-05 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697395+0000", 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0030120220000000001 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.025 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697476+0000", 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0851000000000002e-05 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697526+0000", 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0339e-05 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.026 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105291 14.b 14:d2bd06f1:::benchmark_data_smithi142_428506_object105290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.765168+0000", 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.176093891999997, 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93886015899999997, 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105291 2024-08-17T17:54:30.027 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765168+0000", 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765168+0000", 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.028 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765169+0000", 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.328e-06 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765192+0000", 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.3300999999999999e-05 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765206+0000", 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.3701e-05 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.029 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765213+0000", 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9789999999999996e-06 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765264+0000", 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0691e-05 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.030 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.765271+0000", 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.7430000000000002e-06 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.691253+0000", 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.92598203300000004 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.691409+0000", 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015610299999999999 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.031 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.691547+0000", 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013801500000000001 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697353+0000", 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0058057170000000002 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697596+0000", 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00024271000000000001 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.032 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.703912+0000", 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0063162970000000002 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.703981+0000", 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8751999999999995e-05 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.704028+0000", 2024-08-17T17:54:30.033 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6789000000000002e-05 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105308 14.b 14:d7336a60:::benchmark_data_smithi142_428506_object105307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.781091+0000", 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.160170751999999, 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93086617800000004, 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.034 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105308 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781091+0000", 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781091+0000", 2024-08-17T17:54:30.035 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781091+0000", 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8899999999999999e-07 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781121+0000", 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.9448999999999999e-05 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.036 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781137+0000", 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.6441000000000001e-05 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781162+0000", 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.4470000000000001e-05 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781242+0000", 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.0207e-05 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.037 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781253+0000", 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0903999999999999e-05 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694407+0000", 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.91315440699999995 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694540+0000", 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000132521 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.038 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.694670+0000", 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013032199999999999 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.703899+0000", 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0092289439999999993 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.039 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.709185+0000", 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0052856020000000004 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711859+0000", 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0026741740000000001 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711916+0000", 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7119e-05 2024-08-17T17:54:30.040 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711957+0000", 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0929000000000001e-05 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105312 14.b 14:d4fca65e:::benchmark_data_smithi142_428506_object105311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.781655+0000", 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.159606181000001, 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.93049283100000002, 2024-08-17T17:54:30.041 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105312 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.042 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781655+0000", 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781655+0000", 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781656+0000", 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6599999999999996e-07 2024-08-17T17:54:30.043 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781682+0000", 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.6013999999999999e-05 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781698+0000", 2024-08-17T17:54:30.044 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5801e-05 2024-08-17T17:54:30.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781705+0000", 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.9600000000000003e-06 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781776+0000", 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.0994999999999994e-05 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.046 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.781785+0000", 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.6870000000000006e-06 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697820+0000", 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.91603501600000004 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.697968+0000", 2024-08-17T17:54:30.047 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00014792900000000001 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.698103+0000", 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013480100000000001 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.709189+0000", 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.011085866 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.048 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711764+0000", 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025746950000000001 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712071+0000", 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00030700299999999997 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712108+0000", 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.7069e-05 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.049 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712148+0000", 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.0329e-05 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105343 14.b 14:d873532c:::benchmark_data_smithi142_428506_object105342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.808260+0000", 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.133001483000001, 2024-08-17T17:54:30.050 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.90401082300000002, 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105343 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.808260+0000", 2024-08-17T17:54:30.051 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.808260+0000", 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.808261+0000", 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.6799999999999999e-07 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.052 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.808281+0000", 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.9797e-05 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.808291+0000", 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.978e-06 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.053 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.808296+0000", 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.1880000000000001e-06 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.812134+0000", 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0038382989999999999 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.812144+0000", 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0064999999999999e-05 2024-08-17T17:54:30.054 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.700942+0000", 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88879808199999999 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.701097+0000", 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015446899999999999 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.701248+0000", 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015101099999999999 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711776+0000", 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010528172000000001 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711815+0000", 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.8729000000000001e-05 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.056 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712192+0000", 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00037744599999999998 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712238+0000", 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.5420000000000002e-05 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712271+0000", 2024-08-17T17:54:30.057 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.3299000000000001e-05 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105360 14.b 14:d028ac33:::benchmark_data_smithi142_428506_object105359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.825365+0000", 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.115896860999996, 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.88703458899999998, 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.058 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105360 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825365+0000", 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825365+0000", 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.059 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825366+0000", 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.827e-06 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825439+0000", 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.2593000000000001e-05 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.060 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825452+0000", 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2962999999999999e-05 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825459+0000", 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8909999999999998e-06 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825525+0000", 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6499000000000001e-05 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.061 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.825538+0000", 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2445e-05 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.704059+0000", 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.87852162199999995 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.704230+0000", 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00017042399999999999 2024-08-17T17:54:30.062 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.704393+0000", 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016307800000000001 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.711977+0000", 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0075836769999999996 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.063 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712013+0000", 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5880000000000002e-05 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712314+0000", 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00030145500000000001 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.064 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712365+0000", 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.0618000000000002e-05 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712399+0000", 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4616999999999997e-05 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105391 14.b 14:d2d59cd9:::benchmark_data_smithi142_428506_object105390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.065 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.852175+0000", 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.089086645999998, 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.86651248999999997, 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105391 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.066 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852175+0000", 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852175+0000", 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.067 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852176+0000", 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1030000000000001e-06 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852198+0000", 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.1685999999999999e-05 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852210+0000", 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.237e-05 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.068 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852221+0000", 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.084e-05 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852298+0000", 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.6852000000000002e-05 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.852308+0000", 2024-08-17T17:54:30.069 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.8349999999999999e-06 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.709225+0000", 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.856917869 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.709350+0000", 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000125081 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.070 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.709456+0000", 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00010513700000000001 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712024+0000", 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0025680379999999999 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712438+0000", 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00041459500000000002 2024-08-17T17:54:30.071 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.718553+0000", 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0061148160000000003 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.718621+0000", 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.8198e-05 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.718687+0000", 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.6069999999999996e-05 2024-08-17T17:54:30.072 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105398 14.b 14:d40895ad:::benchmark_data_smithi142_428506_object105397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.857511+0000", 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.083750682000002, 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.87706847600000004, 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.073 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105398 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857511+0000", 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857511+0000", 2024-08-17T17:54:30.074 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857512+0000", 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.2200000000000003e-07 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857529+0000", 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7033000000000001e-05 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.075 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857541+0000", 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.2075999999999999e-05 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857548+0000", 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.3939999999999998e-06 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857607+0000", 2024-08-17T17:54:30.076 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8480999999999998e-05 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.857613+0000", 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7440000000000001e-06 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712468+0000", 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.85485463699999997 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.077 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712629+0000", 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00016069700000000001 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.712782+0000", 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000153764 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.724930+0000", 2024-08-17T17:54:30.078 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.012147557999999999 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.728526+0000", 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0035964510000000001 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.734461+0000", 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0059346679999999997 2024-08-17T17:54:30.079 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.734531+0000", 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.025e-05 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.734579+0000", 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7901000000000001e-05 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.080 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105425 14.b 14:d322c1ba:::benchmark_data_smithi142_428506_object105424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.887647+0000", 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 66.053614972999995, 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.84772610299999995, 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.081 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105425 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887647+0000", 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887647+0000", 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.082 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887647+0000", 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.7399999999999991e-07 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887663+0000", 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.5398999999999999e-05 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887672+0000", 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.9160000000000007e-06 2024-08-17T17:54:30.083 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887678+0000", 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.297e-06 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887728+0000", 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.9536000000000001e-05 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.084 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.887735+0000", 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1930000000000004e-06 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.715641+0000", 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.82790583100000004 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.715765+0000", 2024-08-17T17:54:30.085 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012440300000000001 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.715902+0000", 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000137064 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.724962+0000", 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0090599410000000002 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.086 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.734597+0000", 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0096353610000000003 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.734633+0000", 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5237999999999997e-05 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.734757+0000", 2024-08-17T17:54:30.087 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000124182 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735373+0000", 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00061576800000000004 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105476 14.b 14:d505b9f4:::benchmark_data_smithi142_428506_object105475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.945540+0000", 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 65.995721395999993, 2024-08-17T17:54:30.088 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.79048490900000001, 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105476 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.089 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945540+0000", 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945540+0000", 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945541+0000", 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.54e-07 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.090 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945562+0000", 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 2.0665e-05 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945573+0000", 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1878999999999999e-05 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945580+0000", 2024-08-17T17:54:30.091 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.7320000000000001e-06 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945638+0000", 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.7578999999999999e-05 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.945645+0000", 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.1289999999999998e-06 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.092 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.718716+0000", 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.77307097899999999 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.718847+0000", 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00013152800000000001 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.718986+0000", 2024-08-17T17:54:30.093 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000138629 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.724993+0000", 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0060068550000000002 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735423+0000", 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010429857000000001 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.094 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735648+0000", 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00022548199999999999 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735990+0000", 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00034205800000000001 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.736025+0000", 2024-08-17T17:54:30.095 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.4683e-05 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105483 14.b 14:d46ecf5f:::benchmark_data_smithi142_428506_object105482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.957136+0000", 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 65.984125497999997, 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.77908256600000003, 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.096 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105483 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957136+0000", 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.097 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957136+0000", 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957137+0000", 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1340000000000001e-06 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957155+0000", 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.8000999999999999e-05 2024-08-17T17:54:30.098 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957166+0000", 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.0689e-05 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957171+0000", 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.3940000000000004e-06 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.099 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957226+0000", 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.4756000000000002e-05 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.957234+0000", 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 7.8450000000000003e-06 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.721860+0000", 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.76462602499999999 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.100 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.722006+0000", 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000146563 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.722131+0000", 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012459999999999999 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735460+0000", 2024-08-17T17:54:30.101 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.01332916 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735492+0000", 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.2126999999999999e-05 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.736033+0000", 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00054054600000000004 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.102 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.736068+0000", 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 3.5305999999999997e-05 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.736219+0000", 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00015042000000000001 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.103 INFO:teuthology.orchestra.run.smithi142.stdout: "description": "osd_op(client.48311.0:105495 14.b 14:dfcdce60:::benchmark_data_smithi142_428506_object105494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "initiated_at": "2024-08-17T17:53:23.976170+0000", 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "age": 65.965091772999997, 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.76291032000000003, 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "continuous": false, 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "type_data": { 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "client_info": { 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "client": "client.48311", 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "tid": 105495 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: "events": [ 2024-08-17T17:54:30.104 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "initiated", 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976170+0000", 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "header_read", 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976170+0000", 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "throttled", 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976171+0000", 2024-08-17T17:54:30.105 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 8.85e-07 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "all_read", 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976188+0000", 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.7320999999999999e-05 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "dispatched", 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976197+0000", 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 9.2499999999999995e-06 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.106 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "queued_for_pg", 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976202+0000", 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.6129999999999997e-06 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976260+0000", 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 5.8020999999999999e-05 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for peered", 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:23.976272+0000", 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 1.1732e-05 2024-08-17T17:54:30.107 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "reached_pg", 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.725035+0000", 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.74876375299999998 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "started", 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.725151+0000", 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.000115906 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:54:30.108 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.725278+0000", 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00012672599999999999 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.735524+0000", 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.010245643 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "op_commit", 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.736229+0000", 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.00070502499999999997 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.109 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.738971+0000", 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 0.0027425869999999999 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "commit_sent", 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.739033+0000", 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 6.1433999999999995e-05 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: }, 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: { 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "event": "done", 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "time": "2024-08-17T17:53:24.739080+0000", 2024-08-17T17:54:30.110 INFO:teuthology.orchestra.run.smithi142.stdout: "duration": 4.7423999999999999e-05 2024-08-17T17:54:30.111 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.111 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.111 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.111 INFO:teuthology.orchestra.run.smithi142.stdout: } 2024-08-17T17:54:30.111 INFO:teuthology.orchestra.run.smithi142.stdout: ] 2024-08-17T17:54:30.111 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:54:30.150 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:30.156 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:54:30.251 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:30.283 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:30.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:30.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:30.419 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:30.520 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:30.549 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:30.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:54:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:54:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:30.651 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:30.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:30.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:54:30.704 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:30.705 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:54:30.781 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:30.812 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:30.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3295100276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:30.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3295100276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:30.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:30.951 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:30.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:31.052 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:31.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3295100276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:31.085 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:31.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:31.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:31.315 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:31.345 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:31.447 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:31.477 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:31.578 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:31.609 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:31.710 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:31.745 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:31.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:31.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:31 smithi142 ceph-mon[106729]: pgmap v1950: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:31.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:31 smithi142 ceph-mon[103614]: pgmap v1950: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:54:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:54:31] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.51.0" 2024-08-17T17:54:31.883 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:31.985 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:32.013 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:32.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:31 smithi152 ceph-mon[93456]: pgmap v1950: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:32.115 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:32.149 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:32.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:32.289 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:32.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:32.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:32.427 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:32.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:54:32.464 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:54:32.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:32.566 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:32.668 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:32.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:32.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:32.843 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:32.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.983 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:32.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:33.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:33.115 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:33.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:33.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:33.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.248 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:33.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:33.263 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:54:33.349 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:33.378 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:33.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:33.480 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:33.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:33.621 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:33.655 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:33.724 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:33.757 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:33.795 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:33.855 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:33 smithi142 ceph-mon[103614]: pgmap v1951: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:33.856 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:33 smithi142 ceph-mon[106729]: pgmap v1951: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:33.897 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:33.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.934 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:33.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:34.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.069 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:34.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:34.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:33 smithi152 ceph-mon[93456]: pgmap v1951: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:54:34.170 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.203 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:34.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:34.304 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:34.336 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:34.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:34.465 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:54:34.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:34.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:34.601 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:34.702 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.759 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:34.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:34.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:34.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:34.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:34.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:35.031 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:35.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:35.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:35.166 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:35.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:35.305 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:35.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:35.435 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:35.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:35.537 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:35.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.566 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:35.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:35.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.701 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:35.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:35.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:35.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.840 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:35.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:35.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:35 smithi142 ceph-mon[106729]: pgmap v1952: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:35.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:35 smithi142 ceph-mon[103614]: pgmap v1952: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:35.931 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:35.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:35.976 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:35 smithi152 ceph-mon[93456]: pgmap v1952: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:36.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:36.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:36.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:36.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:36.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:36.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:36.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:36.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:36.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:36.525 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:36.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:36.627 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:36.656 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:36.758 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:36.791 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:36.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:36.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:36 smithi142 ceph-mon[103614]: pgmap v1953: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:54:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:36 smithi142 ceph-mon[106729]: pgmap v1953: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:54:36.893 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:36.931 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:37.033 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:37.066 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:36 smithi152 ceph-mon[93456]: pgmap v1953: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:54:37.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:37.201 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:37.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:37.349 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:37.451 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:37.480 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:37.582 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:37.607 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:37.709 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:37.739 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:37.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:37.871 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:37.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:37.973 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:38.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:38.112 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.152 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:38.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:38.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:38.293 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:38.395 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:38.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:38.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:38.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:38.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:38.719 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:38.820 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:38.849 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2110242170' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[106729]: pgmap v1954: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:54:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:54:38.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2110242170' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:38.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:38.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[103614]: pgmap v1954: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:38.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:54:38.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:38 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:54:38.950 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:38.981 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:39.071 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:38 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2110242170' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:39.071 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:38 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:39.071 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:38 smithi152 ceph-mon[93456]: pgmap v1954: 89 pgs: 89 active+clean; 2.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:39.071 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:38 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:54:39.071 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:38 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:54:39.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:39.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:39.125 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:54:39.225 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:39.253 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:39.356 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:39.382 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:39.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:39.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:39.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:39.640 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:39.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:39.777 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:39.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:39.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:39 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:39.848 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:39 smithi142 ceph-mon[103614]: osdmap e493: 8 total, 7 up, 5 in 2024-08-17T17:54:39.849 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:39 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:39.849 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:39 smithi142 ceph-mon[106729]: osdmap e493: 8 total, 7 up, 5 in 2024-08-17T17:54:39.879 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:39.890 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:39.922 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:39.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:40.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:40.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:40.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:39 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:40.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:39 smithi152 ceph-mon[93456]: osdmap e493: 8 total, 7 up, 5 in 2024-08-17T17:54:40.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:40.186 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:40.288 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:40.315 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:40.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:40.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-08-17T17:54:40.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:40.453 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:40.555 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:54:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:54:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:40.587 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:40.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:40.726 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:40.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:40.864 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:40.897 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:40 smithi142 ceph-mon[106729]: pgmap v1956: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:40.897 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:40 smithi142 ceph-mon[103614]: pgmap v1956: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:40.966 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:41.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:41.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:40 smithi152 ceph-mon[93456]: pgmap v1956: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:41.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:41.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:41.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:41.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:41.265 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:41.367 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:41.395 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:41.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:41.535 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:41.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:41.674 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:41.776 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:41.808 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:41.902 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:54:41.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:41.945 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:42.049 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:42.078 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:42.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:54:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:54:41] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-17T17:54:42.181 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:42.219 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:42.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:42.352 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:42.454 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:42.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:42.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:42.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:42.629 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:42.731 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:42.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:42.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:42.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:42.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:42.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:42.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.896 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:42.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:42.997 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.025 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:43.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:43.126 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:43.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:43.260 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:43.293 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:43.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:43.445 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:43.547 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:43.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:43.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:43.721 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:43.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:43 smithi152 ceph-mon[93456]: pgmap v1957: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:43.870 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:43 smithi142 ceph-mon[106729]: pgmap v1957: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:43 smithi142 ceph-mon[103614]: pgmap v1957: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-08-17T17:54:43.972 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.003 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:44.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:44.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:44.141 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:44.243 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.280 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:44.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:44.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:44.409 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:44.511 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:44.540 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:44.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:44.683 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:44.785 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:44.817 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:44.827 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:54:44.827 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:54:44.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:44.952 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:45.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:45.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:44 smithi152 ceph-mon[93456]: pgmap v1958: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-08-17T17:54:45.089 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:45.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:45.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:44 smithi142 ceph-mon[106729]: pgmap v1958: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-08-17T17:54:45.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:44 smithi142 ceph-mon[103614]: pgmap v1958: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-08-17T17:54:45.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:45.228 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:45.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:45.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:45.359 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:45.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:45.497 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:45.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:45.599 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:45.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:45.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:45.786 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:45.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:45.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:46.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:46.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.072 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:46.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:46.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:46.199 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:46.301 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:46.331 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:46.433 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:46.466 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:46.506 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:54:46.569 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:46.628 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:46.699 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:46.703 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:46.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:46.758 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:54:46.758 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:46.758 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:46 smithi142 ceph-mon[103614]: pgmap v1959: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-08-17T17:54:46.759 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:46 smithi142 ceph-mon[106729]: pgmap v1959: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-08-17T17:54:46.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:46.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:46.806 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:54:46.806 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:46.835 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:54:46.835 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:46.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:46.865 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:54:46.865 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:46.895 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:54:46.895 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:46.900 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:46.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:46.928 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:54:46.929 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:46.954 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:54:46.954 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:46.955 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:54:47.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:47.044 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:46 smithi152 ceph-mon[93456]: pgmap v1959: 89 pgs: 89 active+clean; 2.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-08-17T17:54:47.146 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.187 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:47.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:47.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:47.321 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:47.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:47.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.464 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:47.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:47.565 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:47.595 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:47.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:47.739 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:47.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:47.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:47.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2848895927' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2848895927' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.878 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:47.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:47.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:48.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:48.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:48.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2848895927' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:48.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:48.141 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:48.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.278 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:48.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:48.379 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:48.413 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:48.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:48.553 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:48.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:48.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.698 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:48.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:48.718 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:48 smithi142 ceph-mon[106729]: pgmap v1960: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:48.719 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:48 smithi142 ceph-mon[103614]: pgmap v1960: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:48.799 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.828 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:48.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:48.929 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:48.961 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:49.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:49.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:48 smithi152 ceph-mon[93456]: pgmap v1960: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:49.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:49.196 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:49.224 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:49.326 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:49.359 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:49.389 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:54:49.389 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:54:49.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:49.492 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:49.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:49.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:49.623 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:49.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:49.768 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:49.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:49.907 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:49.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:50.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.042 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:50.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:50.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:50.175 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:50.276 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:50.303 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:50.404 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:50.436 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:50.538 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:50.565 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:54:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:54:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:50.667 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:50.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:50.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:50.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:50.833 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:50.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:50.943 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:50 smithi142 ceph-mon[106729]: pgmap v1961: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.31k op/s 2024-08-17T17:54:50.943 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:50 smithi142 ceph-mon[103614]: pgmap v1961: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.31k op/s 2024-08-17T17:54:50.974 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:50 smithi152 ceph-mon[93456]: pgmap v1961: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.31k op/s 2024-08-17T17:54:51.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:51.111 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:51.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:51.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.250 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:51.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:51.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.383 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:51.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:51.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:54:51.484 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.528 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:51.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:51.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.663 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:51.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:51.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:51.807 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:51.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.944 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:51.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:52.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:52.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.089 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:52.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:52.126 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:54:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:54:51] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-17T17:54:52.190 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:52.222 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:52.323 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:52.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.356 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:52.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:52.457 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:52.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:52.592 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:52.631 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.632 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:52.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:52.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.773 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:52.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:52.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.913 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:52.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:52.934 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:52 smithi142 ceph-mon[103614]: pgmap v1962: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:54:52.935 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:52 smithi142 ceph-mon[106729]: pgmap v1962: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:54:53.014 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:53.047 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:53.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:53.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:52 smithi152 ceph-mon[93456]: pgmap v1962: 89 pgs: 89 active+clean; 1.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:54:53.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:53.191 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:53.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:53.328 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:53.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:53.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-08-17T17:54:53.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:53.487 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:53.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:53.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:53.630 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:53.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.759 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:53.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:53.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.907 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:53.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:54.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:54.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.048 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:54.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:54.079 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:54:54.149 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:54.180 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2206091162' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:54.180 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:54 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:54.180 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2206091162' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:54.180 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:54 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:54.182 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:54.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:54.318 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:54.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:54.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2206091162' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:54.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:54 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:54.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.455 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:54.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:54.556 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:54.587 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:54.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:54.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.725 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:54.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:54.826 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:54.852 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:54.954 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:54.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.985 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:54.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:55.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:55.129 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:55.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:55.231 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:55.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:55 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:55 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:55 smithi152 ceph-mon[93456]: osdmap e494: 8 total, 7 up, 5 in 2024-08-17T17:54:55.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:55 smithi152 ceph-mon[93456]: pgmap v1964: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:55.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:55.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[103614]: osdmap e494: 8 total, 7 up, 5 in 2024-08-17T17:54:55.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[103614]: pgmap v1964: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:55.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:55.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:55.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[106729]: osdmap e494: 8 total, 7 up, 5 in 2024-08-17T17:54:55.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:55 smithi142 ceph-mon[106729]: pgmap v1964: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:55.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:55.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:55.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:55.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:55.622 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:55.649 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:55.751 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:55.783 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:55.885 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:55.917 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:56.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:56.061 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:56.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:56.163 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:56.189 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:56.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:56.290 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:56.321 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:56.422 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:56.453 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:56.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:56.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.590 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:56.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:56.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:56.727 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:56.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.850 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:56.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:56.951 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:56.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:57.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:57.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:57.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:57.254 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:57.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:57.355 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:57.385 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:57.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:57.519 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:57.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:57.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:57.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:57 smithi142 ceph-mon[106729]: pgmap v1965: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:57.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:57 smithi142 ceph-mon[103614]: pgmap v1965: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:57.660 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:57.762 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:57.794 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:57.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:54:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:57.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:57 smithi152 ceph-mon[93456]: pgmap v1965: 89 pgs: 89 active+clean; 1.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:57.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:57.926 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:58.028 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:58.059 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:58.161 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:58.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:58.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:58.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:58.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:58.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:58.468 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:58.570 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:58.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.624 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:58.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:58.725 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:58.735 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:54:58 smithi142 ceph-mon[103614]: pgmap v1966: 89 pgs: 89 active+clean; 1.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:58.736 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:54:58 smithi142 ceph-mon[106729]: pgmap v1966: 89 pgs: 89 active+clean; 1.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:58.764 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:58.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:58.899 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:59.001 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:59.033 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:59.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:54:58 smithi152 ceph-mon[93456]: pgmap v1966: 89 pgs: 89 active+clean; 1.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-08-17T17:54:59.135 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:59.162 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:59.178 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:59.202 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:54:59.202 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:54:59.202 INFO:teuthology.orchestra.run.smithi142.stdout: "complaint_time": 30, 2024-08-17T17:54:59.202 INFO:teuthology.orchestra.run.smithi142.stdout: "num_blocked_ops": 0 2024-08-17T17:54:59.202 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:54:59.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:59.303 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:54:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:59.405 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:59.433 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:59.535 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:59.571 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:59.672 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:59.716 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:59.818 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.851 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:59.952 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:59.982 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:54:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:00.084 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:00.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:00.213 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:00.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:00.296 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:55:00.352 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:00.389 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:00.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:00.499 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:00 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:55:00.499 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:00 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:55:00.525 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:00.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:00 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:55:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:55:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:55:00.216Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:00.624 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:55:00.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:00.705 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:00.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:00.807 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:00.842 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:00.945 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:00.973 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:01.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:01.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:01.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:01.226 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:01.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:01.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:01.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:01.490 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:01.520 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:01 smithi152 ceph-mon[93456]: pgmap v1967: 89 pgs: 89 active+clean; 1.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:55:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:01.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[106729]: pgmap v1967: 89 pgs: 89 active+clean; 1.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[103614]: pgmap v1967: 89 pgs: 89 active+clean; 1.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:01.685 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:01.786 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:01.820 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:01.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:01.963 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:01.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:02.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:02.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:02.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:55:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:55:01] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-08-17T17:55:02.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:02.226 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:02.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:02.361 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:02.463 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.501 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:02.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:02.603 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:02.638 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:02.740 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:02.780 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:02.815 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:55:02.815 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:02.850 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:55:02.851 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:02.880 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:55:02.881 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:02.882 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:02.911 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:55:02.911 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:02.912 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:02 smithi142 ceph-mon[106729]: pgmap v1968: 89 pgs: 89 active+clean; 1.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:55:02.912 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:02 smithi142 ceph-mon[103614]: pgmap v1968: 89 pgs: 89 active+clean; 1.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:55:02.920 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:02.946 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:55:02.946 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:02.971 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:55:02.972 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:02.996 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:55:02.997 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:02.997 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:55:03.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:03.062 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:03.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:02 smithi152 ceph-mon[93456]: pgmap v1968: 89 pgs: 89 active+clean; 1.1 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:55:03.164 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.196 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:03.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:03.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:03.336 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:03.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:03.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.471 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:03.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:03.572 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:03.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:03.704 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:03.733 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:03.835 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.862 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:03.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:03.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:03.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.999 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:03.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:04.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3316904952' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:04.100 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:04.108 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3316904952' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:04.109 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3316904952' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:04.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:55:04.151 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:55:04.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:04.265 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:04.367 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:04.400 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:04.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:04.431 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:04.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:04.536 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:04.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:04.638 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:04.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:04.712 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:04 smithi142 ceph-mon[106729]: pgmap v1969: 89 pgs: 89 active+clean; 832 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.27k op/s 2024-08-17T17:55:04.712 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:04 smithi142 ceph-mon[103614]: pgmap v1969: 89 pgs: 89 active+clean; 832 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.27k op/s 2024-08-17T17:55:04.723 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:04.769 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:04.806 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:04.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:04.934 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:04.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:05.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:05.057 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:05.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:05.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:04 smithi152 ceph-mon[93456]: pgmap v1969: 89 pgs: 89 active+clean; 832 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.27k op/s 2024-08-17T17:55:05.158 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:05.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.194 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:05.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:05.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:05.333 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:05.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:05.467 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:05.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.605 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:05.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:05.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:05.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:05.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:05.876 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:05.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.005 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:06.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:06.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:06.135 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:06.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:06.151 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:55:06.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:06.277 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:06.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.431 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:06.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:06.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:06.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:06.664 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:06.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:06.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.848 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:06.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:06.949 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:06.974 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:07.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:07.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.126 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:07.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:07.129 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:06 smithi142 ceph-mon[106729]: pgmap v1970: 89 pgs: 89 active+clean; 788 MiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:55:07.129 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:06 smithi142 ceph-mon[103614]: pgmap v1970: 89 pgs: 89 active+clean; 788 MiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:55:07.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:07.255 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:07.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:07.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:06 smithi152 ceph-mon[93456]: pgmap v1970: 89 pgs: 89 active+clean; 788 MiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:55:07.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.394 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:07.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:07.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:07.530 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:07.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:07.632 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.667 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:07.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:07.768 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:07.799 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:07.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:07.939 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:08.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:08.080 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:08.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:08.212 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:08.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.339 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:08.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:08.440 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.473 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:08.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:08.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.615 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:08.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:08.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:08.755 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:08.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:08.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:08.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:08 smithi142 ceph-mon[103614]: pgmap v1971: 89 pgs: 89 active+clean; 631 MiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.20k op/s 2024-08-17T17:55:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:08 smithi142 ceph-mon[106729]: pgmap v1971: 89 pgs: 89 active+clean; 631 MiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.20k op/s 2024-08-17T17:55:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.895 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:08.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:08.989 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:08 smithi152 ceph-mon[93456]: pgmap v1971: 89 pgs: 89 active+clean; 631 MiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.20k op/s 2024-08-17T17:55:08.996 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:08.997 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.041 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:09.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:09.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:09.187 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:09.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:09.289 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:09.320 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:09.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:09.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:09.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:09.593 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:09.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:09.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.732 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:09.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:09.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:09.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:09.976 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:10.011 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:10.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:10.158 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:10.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:10.301 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:10.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:10 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3502427090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:10 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:10.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:10 smithi142 ceph-mon[106729]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:10 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3502427090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:10 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:10.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:10 smithi142 ceph-mon[103614]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:10.403 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:10.432 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:10.534 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:10 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3502427090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:10 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:10.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:10 smithi152 ceph-mon[93456]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:10.574 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:55:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:55:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.582 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:10.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:10.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.736 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:10.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:10.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:10.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.894 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:10.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:10.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:11.034 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:11.136 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:11.173 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:11.180 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is set 2024-08-17T17:55:11.275 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.309 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:11.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:11.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[106729]: osdmap e495: 8 total, 7 up, 5 in 2024-08-17T17:55:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:11.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[106729]: pgmap v1973: 89 pgs: 89 active+clean; 429 MiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:11.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:11.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[103614]: osdmap e495: 8 total, 7 up, 5 in 2024-08-17T17:55:11.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:11.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:11.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-mon[103614]: pgmap v1973: 89 pgs: 89 active+clean; 429 MiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:11.376 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6] out_osds: [2, 7, 0] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:55:11.376 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:55:11.376 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:55:11.377 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2024-08-17T17:55:11.377 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-17T17:55:11.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:11.443 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:11.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:11.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:11 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:11 smithi152 ceph-mon[93456]: osdmap e495: 8 total, 7 up, 5 in 2024-08-17T17:55:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:11.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:11 smithi152 ceph-mon[93456]: pgmap v1973: 89 pgs: 89 active+clean; 429 MiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:11.579 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:11.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:11.681 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:11.715 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:11.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:11.852 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:11.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:11.995 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:12.094 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:55:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:55:11] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-17T17:55:12.097 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.145 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:12.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:12.246 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:12.286 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:12.388 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:12.425 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:12.527 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:12.562 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:12.609 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:12.609 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:12.653 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:12.653 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:12.663 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:12.698 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:12.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:12.800 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:12.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:12.841 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:12.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:12.975 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:13.077 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:13.108 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:13.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:13.235 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:13.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:13.337 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.367 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:13.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:13.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:13.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:55:13.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:13.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:13.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:13 smithi142 ceph-mon[106729]: pgmap v1974: 89 pgs: 89 active+clean; 429 MiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:13.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1103555488' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:13 smithi142 ceph-mon[103614]: pgmap v1974: 89 pgs: 89 active+clean; 429 MiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:13.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1103555488' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:13.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:13.675 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:55:13.675 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:13.676 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:13.677 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:55:13.677 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:55:13.677 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:13.677 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:13.677 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:55:13.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:13.787 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:13.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:13 smithi152 ceph-mon[93456]: pgmap v1974: 89 pgs: 89 active+clean; 429 MiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:13.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1103555488' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:13.861 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:55:13.889 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:13.943 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:14.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.082 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:14.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:14.183 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.213 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:14.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:14.314 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:14.343 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:14.445 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:14.472 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:14.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:14.616 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:14.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Removed 118697 objects 2024-08-17T17:55:14.631 INFO:tasks.radosbench.radosbench.0.smithi142.stdout:Clean up completed and total clean up time :97.8458 2024-08-17T17:55:14.718 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.741 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:14.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:14.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:14.863 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:14.965 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:14.987 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:15.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:14 smithi152 ceph-mon[93456]: pgmap v1975: 89 pgs: 89 active+clean; 108 MiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-17T17:55:15.089 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:15.111 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:14 smithi142 ceph-mon[106729]: pgmap v1975: 89 pgs: 89 active+clean; 108 MiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-17T17:55:15.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:14 smithi142 ceph-mon[103614]: pgmap v1975: 89 pgs: 89 active+clean; 108 MiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-17T17:55:15.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:15.235 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:15.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:15.360 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:15.461 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:15.484 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:15.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:15.611 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:15.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:15.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:15.738 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:15.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:15.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:15.864 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2024-08-17T17:55:15.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2024-08-17T17:55:15.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:15.969 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:15.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.991 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:15.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:16.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:16.118 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:16.219 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:16.244 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:16.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:55:16.256 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:55:16.265 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:16.345 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:16.351 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:16.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:16.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:16.477 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:16.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:16.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:16.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:16.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:16.747 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:16.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:16.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:16.875 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:16.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:17.000 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:17.044 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:16 smithi142 ceph-mon[106729]: pgmap v1976: 89 pgs: 89 active+clean; 70 MiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:17.045 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:16 smithi142 ceph-mon[103614]: pgmap v1976: 89 pgs: 89 active+clean; 70 MiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:16 smithi152 ceph-mon[93456]: pgmap v1976: 89 pgs: 89 active+clean; 70 MiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.15k op/s 2024-08-17T17:55:17.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:17.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:17.228 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:17.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:17.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:17.373 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:17.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:17.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:17.501 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:17.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:17.599 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:17.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:17.628 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:17.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:17.756 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:17.858 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:17.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:17.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:18.004 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:18.106 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:18.127 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:18.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:18.251 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:18.257 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:55:18.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:18.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:18.476 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:18.499 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:18.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:18.627 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:18.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:18.729 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:18.750 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:18.774 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:18.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:18.864 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:18 smithi142 ceph-mon[106729]: pgmap v1977: 89 pgs: 89 active+clean; 19 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 978 op/s 2024-08-17T17:55:18.864 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:18 smithi142 ceph-mon[103614]: pgmap v1977: 89 pgs: 89 active+clean; 19 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 978 op/s 2024-08-17T17:55:18.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.876 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:18.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:18.977 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:18.999 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:19.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:18 smithi152 ceph-mon[93456]: pgmap v1977: 89 pgs: 89 active+clean; 19 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 978 op/s 2024-08-17T17:55:19.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:19.127 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:19.229 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:19.250 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:19.352 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:19.374 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:19.475 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:19.497 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:19.598 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:19.621 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:19.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:19.743 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:19.845 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:19.866 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:19.968 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:19.988 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:19.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:20.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:20.116 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:20.218 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:20.240 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:20.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:20.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:20.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:20.469 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:20.494 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:20.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:55:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:55:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:20.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:20.621 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:20.723 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:20.747 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:20.849 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:20.870 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:20.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:20.996 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:21.004 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:20 smithi142 ceph-mon[106729]: pgmap v1978: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 724 op/s 2024-08-17T17:55:21.004 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:20 smithi142 ceph-mon[103614]: pgmap v1978: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 724 op/s 2024-08-17T17:55:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:20 smithi152 ceph-mon[93456]: pgmap v1978: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 724 op/s 2024-08-17T17:55:21.098 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:21.122 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:21.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:21.247 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:21.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:21.349 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:21.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.375 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:21.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:21.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:21.499 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:21.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:21.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:21.625 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:21.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:21.727 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:21.751 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:21.853 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:21.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:55:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:55:21] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-17T17:55:21.875 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:21.976 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:22.002 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:22.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:22.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:22.125 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:22.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:22.253 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:22.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:22.381 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:22.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:22.508 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:22.609 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:22.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:22.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:22.736 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:22.758 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:22.860 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:22.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:22 smithi142 ceph-mon[106729]: pgmap v1979: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 571 op/s 2024-08-17T17:55:22.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:22 smithi142 ceph-mon[103614]: pgmap v1979: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 571 op/s 2024-08-17T17:55:22.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:22.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:23.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:23.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:22 smithi152 ceph-mon[93456]: pgmap v1979: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 571 op/s 2024-08-17T17:55:23.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:23.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.134 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:23.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:23.235 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.257 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:23.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:23.358 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:23.381 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:23.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:23.509 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:23.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:23.634 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:23.676 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:55:23.680 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-17T17:55:23.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:23.761 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:23.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:23.863 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/67023863' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:55:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:23 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:55:23.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:23 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2784738441' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/67023863' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:55:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:23 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:55:23.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:23 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2784738441' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.886 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:23.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:23.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:24.009 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:24.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/67023863' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:55:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:23 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:55:24.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:23 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2784738441' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:24.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:24.136 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:24.237 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:24.259 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:24.360 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:24.383 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:24.485 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:24.488 INFO:teuthology.orchestra.run.smithi142.stdout:{ 2024-08-17T17:55:24.488 INFO:teuthology.orchestra.run.smithi142.stdout: "ops": [], 2024-08-17T17:55:24.488 INFO:teuthology.orchestra.run.smithi142.stdout: "num_ops": 0 2024-08-17T17:55:24.488 INFO:teuthology.orchestra.run.smithi142.stdout:} 2024-08-17T17:55:24.506 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:24.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:24.634 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:24.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:24.778 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:24.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:24.901 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:24.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:25.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:25.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:24 smithi152 ceph-mon[93456]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:24 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:55:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2784738441' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:24 smithi152 ceph-mon[93456]: osdmap e496: 8 total, 7 up, 5 in 2024-08-17T17:55:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:24 smithi152 ceph-mon[93456]: pgmap v1981: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 172 op/s 2024-08-17T17:55:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:24 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/179448390' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:55:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[106729]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:55:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2784738441' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:25.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[106729]: osdmap e496: 8 total, 7 up, 5 in 2024-08-17T17:55:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[106729]: pgmap v1981: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 172 op/s 2024-08-17T17:55:25.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/179448390' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:55:25.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[103614]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:25.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:55:25.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2784738441' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:25.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[103614]: osdmap e496: 8 total, 7 up, 5 in 2024-08-17T17:55:25.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[103614]: pgmap v1981: 89 pgs: 89 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 172 op/s 2024-08-17T17:55:25.127 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:24 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/179448390' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:55:25.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.151 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:25.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:25.160 INFO:teuthology.orchestra.run.smithi142.stderr:pool 'unique_pool_9' removed 2024-08-17T17:55:25.252 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:25.275 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:25.376 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:25.399 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:25.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:25.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:25.523 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:25.625 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:25.647 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:25.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:25.749 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:25.771 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:25.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:25.872 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:25.894 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:25.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:25.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:25 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/179448390' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:55:25.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:25 smithi142 ceph-mon[106729]: osdmap e497: 8 total, 7 up, 5 in 2024-08-17T17:55:25.989 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:25 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/179448390' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:55:25.989 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:25 smithi142 ceph-mon[103614]: osdmap e497: 8 total, 7 up, 5 in 2024-08-17T17:55:25.996 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:26.017 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:25 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/179448390' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:55:26.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:25 smithi152 ceph-mon[93456]: osdmap e497: 8 total, 7 up, 5 in 2024-08-17T17:55:26.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:26.142 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:26.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:26.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:26.266 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:26.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.391 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:26.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:26.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:26.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:26.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:26.641 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:26.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:26.704 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:26.742 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:26.763 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:26.865 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:26.887 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:26.961 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:26.962 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:26 smithi142 ceph-mon[106729]: pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:26.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:26.962 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:26 smithi142 ceph-mon[103614]: pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:26.989 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:27.010 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:27.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:26 smithi152 ceph-mon[93456]: pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:27.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:27.138 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:27.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:27.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:27.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:27.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:27.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.392 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:27.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:27.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:27.517 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:27.619 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:27.639 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:27.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:27.767 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:27.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:27.893 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:27.927 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:55:27.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:28.017 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:28.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.143 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:28.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:28.244 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.266 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:28.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:28.368 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:28.390 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:28.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:28.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:28.514 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:28.616 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:28.636 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:28.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:28.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:28.763 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:28.797 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:28 smithi142 ceph-mon[106729]: pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:28.797 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:28 smithi142 ceph-mon[103614]: pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:28.822 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:55:28.822 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-08-17T17:55:28.822 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-17T17:55:28.828 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-08-17T17:55:28.829 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-17T17:55:28.829 DEBUG:teuthology.orchestra.run.smithi142:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:55:28.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.855 INFO:teuthology.orchestra.run.smithi142.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-17T17:55:28.855 DEBUG:teuthology.orchestra.run.smithi142:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:55:28.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:28.889 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:28.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:28.927 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-17T17:55:28.928 DEBUG:teuthology.orchestra.run.smithi142:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-17T17:55:28.983 INFO:tasks.workunit:timeout=3h 2024-08-17T17:55:28.983 INFO:tasks.workunit:cleanup=True 2024-08-17T17:55:28.984 DEBUG:teuthology.orchestra.run.smithi142:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:55:28.991 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:29.045 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:29.068 INFO:tasks.workunit.client.0.smithi142.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-17T17:55:29.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:28 smithi152 ceph-mon[93456]: pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:29.147 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.169 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:29.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:29.270 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:29.292 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:29.394 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:29.420 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:29.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:29.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:29.532 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:55:29.548 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:29.650 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:29.672 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:29.774 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:29.795 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:29.897 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:29.919 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:30.020 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:30.041 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:30.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:30.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:30.166 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:30.167 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:30.169 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:30.179 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:55:30.180 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:30.195 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:55:30.195 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:30.196 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:55:30.271 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:30.292 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:30.374 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:30 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2830184990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:30.374 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:30 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2830184990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:30.394 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:30.415 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:30.517 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:30.538 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:30.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:30 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2830184990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:55:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:55:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:30.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:30.664 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:30.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:30.765 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:30.787 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:55:30.887 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:55:30.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:30.909 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:30.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:31.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:31.032 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:31.134 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:31.155 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:31.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:31.260 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:31.265 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:31.280 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:31.382 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:31.391 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6] out_osds: [2, 7, 0] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2024-08-17T17:55:31.391 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:55:31.391 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-08-17T17:55:31.392 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 0 2024-08-17T17:55:31.392 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:31 smithi142 ceph-mon[106729]: pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:31.392 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:31 smithi142 ceph-mon[103614]: pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:31.404 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:31.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:31.527 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:31.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:31.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:31 smithi152 ceph-mon[93456]: pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:31.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:31.651 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:31.753 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.778 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:31.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:31.880 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:31.901 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:32.003 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:32.024 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:32.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:55:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:55:31] "GET /metrics HTTP/1.1" 200 34022 "" "Prometheus/2.51.0" 2024-08-17T17:55:32.125 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:32.146 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:32.248 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:32.270 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:32.371 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:32.393 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:32.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:32.556 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:32.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:55:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:55:32.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:55:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:55:32.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:55:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:55:32.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:32.658 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:32.679 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:32.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:32.806 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:32.888 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:55:32.908 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:32.953 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:33.055 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:33.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:33.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:33.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:33.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:33.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:33.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:33.335 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:33.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:33 smithi142 ceph-mon[106729]: pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:33.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:33 smithi142 ceph-mon[103614]: pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:33.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:33.457 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:33.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:33.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:33 smithi152 ceph-mon[93456]: pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:33.584 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:33.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:33.712 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:33.813 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:33.835 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:33.937 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:33.958 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:34.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:34.086 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:34.187 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:34.210 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:34.312 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:34.333 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:34.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:34.435 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:34.457 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:34.559 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:34.580 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:34.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:34.707 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:34.775 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:34 smithi142 ceph-mon[106729]: pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:34.775 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:34 smithi142 ceph-mon[103614]: pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:34.810 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:34.830 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:34.932 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:34.954 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:34.984 DEBUG:teuthology.orchestra.run.smithi142:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-17T17:55:35.034 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:35.034 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:35.056 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:35.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:34 smithi152 ceph-mon[93456]: pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:35.077 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:35.115 INFO:tasks.workunit.client.0.smithi142.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-17T17:55:35.117 INFO:tasks.workunit.client.0.smithi142.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:55:35.117 INFO:tasks.workunit.client.0.smithi142.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-17T17:55:35.179 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:35.205 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:35.307 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:35.328 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:35.430 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:35.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.452 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:35.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:35.553 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:35.575 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:35.677 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:35.701 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:35.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:35.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.829 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:35.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:35.930 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:35.951 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:35.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:36.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:36.079 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:36.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:36.137 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:55:36.180 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:36.202 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:36.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:36.329 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:36.431 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:36.456 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:36.558 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:36.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2040777006' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.581 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:36.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:36.603 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2040777006' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:36.604 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2040777006' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:36.629 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:55:36.682 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:36.734 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:36.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:36.861 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:36.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:36.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:36.988 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:37.090 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:37.112 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:37.154 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:55:37.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:37.242 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:37.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:37.369 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:37.471 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:37.494 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:37 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:55:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:37 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:55:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:37 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2040777006' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:37 smithi152 ceph-mon[93456]: osdmap e498: 8 total, 7 up, 5 in 2024-08-17T17:55:37.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:37 smithi152 ceph-mon[93456]: pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:37.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:37.622 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:55:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:55:37.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2040777006' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:37.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[106729]: osdmap e498: 8 total, 7 up, 5 in 2024-08-17T17:55:37.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[106729]: pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:55:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2040777006' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[103614]: osdmap e498: 8 total, 7 up, 5 in 2024-08-17T17:55:37.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:37 smithi142 ceph-mon[103614]: pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:37.725 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:37.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.751 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:37.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:37.762 INFO:tasks.workunit.client.0.smithi142.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-17T17:55:37.852 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:37.864 INFO:tasks.workunit.client.0.smithi142.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-17T17:55:37.873 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:37.909 INFO:tasks.workunit.client.0.smithi142.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:55:37.910 INFO:tasks.workunit.client.0.smithi142.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:55:37.910 INFO:tasks.workunit.client.0.smithi142.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-17T17:55:37.950 INFO:tasks.workunit.client.0.smithi142.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:55:37.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:37.980 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-08-17T17:55:37.980 DEBUG:teuthology.orchestra.run.smithi142:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-17T17:55:38.001 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:38.066 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-08-17T17:55:38.067 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-08-17T17:55:38.068 DEBUG:teuthology.orchestra.run.smithi142:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-08-17T17:55:38.102 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:38.130 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:38.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:38.232 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:38.252 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:38.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:38.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:38.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:38.404 INFO:tasks.workunit.client.0.smithi142.stdout:Running main() from gmock_main.cc 2024-08-17T17:55:38.404 INFO:tasks.workunit.client.0.smithi142.stdout:[==========] Running 70 tests from 1 test suite. 2024-08-17T17:55:38.404 INFO:tasks.workunit.client.0.smithi142.stdout:[----------] Global test environment set-up. 2024-08-17T17:55:38.404 INFO:tasks.workunit.client.0.smithi142.stdout:[----------] 70 tests from TestClsRbd 2024-08-17T17:55:38.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:38.503 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:38.605 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:38.626 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:38.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:38.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:38.754 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:38.852 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:38.853 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:38.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:38.881 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:38.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:38.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:39.011 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:39.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:39.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:39.139 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:39.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:39.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:39.266 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:39.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:39.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.395 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:39.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:39.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:39.521 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:39.528 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_all_features 2024-08-17T17:55:39.569 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_all_features (41 ms) 2024-08-17T17:55:39.569 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.copyup 2024-08-17T17:55:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[106729]: osdmap e499: 8 total, 7 up, 5 in 2024-08-17T17:55:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1587113059' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:55:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:55:39.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[106729]: pgmap v1991: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:39.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[103614]: osdmap e499: 8 total, 7 up, 5 in 2024-08-17T17:55:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1587113059' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:55:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:55:39.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:39 smithi142 ceph-mon[103614]: pgmap v1991: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:39.623 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:39.645 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:39.746 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:39.753 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.copyup (184 ms) 2024-08-17T17:55:39.753 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-08-17T17:55:39.768 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:39.801 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.sparse_copyup (48 ms) 2024-08-17T17:55:39.801 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-08-17T17:55:39.809 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-08-17T17:55:39.809 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-08-17T17:55:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:39 smithi152 ceph-mon[93456]: osdmap e499: 8 total, 7 up, 5 in 2024-08-17T17:55:39.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:39 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1587113059' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:55:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:39 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:55:39.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:39 smithi152 ceph-mon[93456]: pgmap v1991: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail 2024-08-17T17:55:39.832 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.add_remove_child (24 ms) 2024-08-17T17:55:39.832 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.directory_methods 2024-08-17T17:55:39.861 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2024-08-17T17:55:39.861 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.create 2024-08-17T17:55:39.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:39.890 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:39.892 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.create (31 ms) 2024-08-17T17:55:39.892 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_features 2024-08-17T17:55:39.894 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-08-17T17:55:39.894 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-08-17T17:55:39.895 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-08-17T17:55:39.895 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-08-17T17:55:39.898 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_create_timestamp (3 ms) 2024-08-17T17:55:39.898 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-08-17T17:55:39.900 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-08-17T17:55:39.900 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-08-17T17:55:39.901 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-08-17T17:55:39.901 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-08-17T17:55:39.904 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-08-17T17:55:39.904 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.get_size 2024-08-17T17:55:39.908 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2024-08-17T17:55:39.908 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.set_size 2024-08-17T17:55:39.912 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-08-17T17:55:39.912 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.protection_status 2024-08-17T17:55:39.927 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.protection_status (15 ms) 2024-08-17T17:55:39.927 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-08-17T17:55:39.934 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-08-17T17:55:39.934 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.parents_v1 2024-08-17T17:55:39.981 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.parents_v1 (47 ms) 2024-08-17T17:55:39.981 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.parents_v2 2024-08-17T17:55:39.993 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:40.008 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2024-08-17T17:55:40.008 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.snapshots 2024-08-17T17:55:40.016 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:40.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:40.028 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.snapshots (20 ms) 2024-08-17T17:55:40.028 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.snapid_race 2024-08-17T17:55:40.033 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2024-08-17T17:55:40.033 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.stripingv2 2024-08-17T17:55:40.041 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2024-08-17T17:55:40.041 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.object_map_save 2024-08-17T17:55:40.045 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-08-17T17:55:40.045 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-08-17T17:55:40.053 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-08-17T17:55:40.053 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.object_map_update 2024-08-17T17:55:40.060 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2024-08-17T17:55:40.060 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-08-17T17:55:40.060 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-08-17T17:55:40.060 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-08-17T17:55:40.067 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2024-08-17T17:55:40.067 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-08-17T17:55:40.074 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2024-08-17T17:55:40.074 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.flags 2024-08-17T17:55:40.079 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-08-17T17:55:40.079 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.metadata 2024-08-17T17:55:40.102 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2024-08-17T17:55:40.102 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.set_features 2024-08-17T17:55:40.108 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-08-17T17:55:40.108 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.mirror 2024-08-17T17:55:40.118 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:40.139 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.mirror (31 ms) 2024-08-17T17:55:40.139 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.mirror_image 2024-08-17T17:55:40.141 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:40.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:40.152 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.mirror_image (13 ms) 2024-08-17T17:55:40.152 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-08-17T17:55:40.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:40.265 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:40.366 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:40.389 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:40.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:40.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:40.522 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:55:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:55:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:40.615 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:40.637 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:40.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:40.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:40.793 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:55:40.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:40 smithi152 ceph-mon[93456]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:55:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:40 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:55:40.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:40 smithi152 ceph-mon[93456]: osdmap e500: 8 total, 7 up, 5 in 2024-08-17T17:55:40.826 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:40 smithi142 ceph-mon[106729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:55:40.827 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:40 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:55:40.827 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:40 smithi142 ceph-mon[106729]: osdmap e500: 8 total, 7 up, 5 in 2024-08-17T17:55:40.827 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:40 smithi142 ceph-mon[103614]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:55:40.827 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:40 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi142-456374-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:55:40.827 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:40 smithi142 ceph-mon[103614]: osdmap e500: 8 total, 7 up, 5 in 2024-08-17T17:55:40.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:40.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:40.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:40.988 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:41.012 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:41.114 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:41.137 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:41.239 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:41.262 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:41.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:41.364 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:41.387 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:41.489 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:41.513 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:41.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:41 smithi152 ceph-mon[93456]: osdmap e501: 8 total, 7 up, 5 in 2024-08-17T17:55:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:41 smithi152 ceph-mon[93456]: pgmap v1994: 105 pgs: 12 creating+peering, 16 unknown, 77 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 253 KiB/s rd, 155 KiB/s wr, 425 op/s 2024-08-17T17:55:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:41.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:41.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:41.637 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:41.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:41.739 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:41.762 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[106729]: osdmap e501: 8 total, 7 up, 5 in 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[106729]: pgmap v1994: 105 pgs: 12 creating+peering, 16 unknown, 77 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 253 KiB/s rd, 155 KiB/s wr, 425 op/s 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[103614]: osdmap e501: 8 total, 7 up, 5 in 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[103614]: pgmap v1994: 105 pgs: 12 creating+peering, 16 unknown, 77 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 253 KiB/s rd, 155 KiB/s wr, 425 op/s 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:41.771 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:41.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:41.886 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:41.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:42.009 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:42.111 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:55:41] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-08-17T17:55:42.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:55:41.900+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e501 definitely_dead 0 2024-08-17T17:55:42.136 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:42.238 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:42.263 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:42.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:42.386 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:42.449 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.mirror_image_status (2297 ms) 2024-08-17T17:55:42.449 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-08-17T17:55:42.487 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:42.509 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:42.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:42.527 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.mirror_image_map (78 ms) 2024-08-17T17:55:42.527 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-08-17T17:55:42.540 INFO:teuthology.orchestra.run.smithi142.stderr:marked in osd.0. 2024-08-17T17:55:42.541 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.mirror_instances (14 ms) 2024-08-17T17:55:42.541 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-08-17T17:55:42.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:42.633 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[106729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2643336320' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[103614]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:55:42.681 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2643336320' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:55:42.682 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:42 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:55:42.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:42.757 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:42 smithi152 ceph-mon[93456]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:55:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:42 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:55:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:42 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2643336320' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:55:42.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:42 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:55:42.859 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:42.881 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:42.983 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:43.006 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:43.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:43.107 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:43.108 INFO:teuthology.orchestra.run.smithi142.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:43.109 INFO:teuthology.orchestra.run.smithi142.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.131 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:43.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:43.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:43.256 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:43.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:43.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:43.380 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:43.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:43.507 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:43.586 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.mirror_snapshot (1045 ms) 2024-08-17T17:55:43.586 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-08-17T17:55:43.588 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-08-17T17:55:43.588 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-08-17T17:55:43.590 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2024-08-17T17:55:43.590 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-08-17T17:55:43.593 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-08-17T17:55:43.593 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-08-17T17:55:43.600 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-08-17T17:55:43.600 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-08-17T17:55:43.603 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-08-17T17:55:43.603 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-08-17T17:55:43.608 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2024-08-17T17:55:43.608 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_image_add 2024-08-17T17:55:43.608 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:43.611 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2024-08-17T17:55:43.611 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-08-17T17:55:43.614 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-08-17T17:55:43.614 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_image_list 2024-08-17T17:55:43.616 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2024-08-17T17:55:43.616 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-08-17T17:55:43.620 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-08-17T17:55:43.620 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.image_group_add 2024-08-17T17:55:43.622 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-08-17T17:55:43.622 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-08-17T17:55:43.624 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-08-17T17:55:43.625 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.image_group_get 2024-08-17T17:55:43.626 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-08-17T17:55:43.627 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-08-17T17:55:43.628 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2024-08-17T17:55:43.628 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-08-17T17:55:43.630 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-08-17T17:55:43.630 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-08-17T17:55:43.630 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:43.632 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-08-17T17:55:43.632 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-08-17T17:55:43.634 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-08-17T17:55:43.635 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1404-gf2c39d4d/rpm/el9/BUILD/ceph-17.2.7-1404-gf2c39d4d/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout:Expected equality of these values: 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout: 1U 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout: Which is: 1 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout: keys.size() 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout: Which is: 3 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-08-17T17:55:43.637 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-08-17T17:55:43.640 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-08-17T17:55:43.640 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-08-17T17:55:43.654 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (14 ms) 2024-08-17T17:55:43.654 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-08-17T17:55:43.732 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:43.753 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:43.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:43 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:55:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:43 smithi152 ceph-mon[93456]: osdmap e502: 8 total, 7 up, 6 in 2024-08-17T17:55:43.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:43 smithi152 ceph-mon[93456]: pgmap v1996: 105 pgs: 12 creating+peering, 16 unknown, 77 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 276 KiB/s rd, 169 KiB/s wr, 464 op/s 2024-08-17T17:55:43.832 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (178 ms) 2024-08-17T17:55:43.832 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-08-17T17:55:43.834 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-08-17T17:55:43.835 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-08-17T17:55:43.836 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (1 ms) 2024-08-17T17:55:43.836 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.trash_methods 2024-08-17T17:55:43.843 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-08-17T17:55:43.843 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.op_features 2024-08-17T17:55:43.849 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-08-17T17:55:43.849 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.clone_parent 2024-08-17T17:55:43.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:43.869 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-08-17T17:55:43.869 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-08-17T17:55:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:43 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:55:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:43 smithi142 ceph-mon[103614]: osdmap e502: 8 total, 7 up, 6 in 2024-08-17T17:55:43.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:43 smithi142 ceph-mon[103614]: pgmap v1996: 105 pgs: 12 creating+peering, 16 unknown, 77 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 276 KiB/s rd, 169 KiB/s wr, 464 op/s 2024-08-17T17:55:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:43 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:55:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:43 smithi142 ceph-mon[106729]: osdmap e502: 8 total, 7 up, 6 in 2024-08-17T17:55:43.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:43 smithi142 ceph-mon[106729]: pgmap v1996: 105 pgs: 12 creating+peering, 16 unknown, 77 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 276 KiB/s rd, 169 KiB/s wr, 464 op/s 2024-08-17T17:55:43.877 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-08-17T17:55:43.877 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.clone_child 2024-08-17T17:55:43.877 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:43.886 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-08-17T17:55:43.886 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-08-17T17:55:43.892 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.namespace_methods (5 ms) 2024-08-17T17:55:43.892 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.migration 2024-08-17T17:55:43.901 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-08-17T17:55:43.901 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.migration_v1 2024-08-17T17:55:43.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:55:43.912 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.migration_v1 (12 ms) 2024-08-17T17:55:43.913 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-08-17T17:55:43.979 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:44.001 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:44.103 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:44.124 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:44.173 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:44.227 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:44.249 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:44.351 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:44.367 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-08-17T17:55:44.372 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:44.474 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:44.495 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:44 smithi152 ceph-mon[93456]: osdmap e503: 8 total, 7 up, 6 in 2024-08-17T17:55:44.597 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:44.601 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:44 smithi142 ceph-mon[106729]: osdmap e503: 8 total, 7 up, 6 in 2024-08-17T17:55:44.602 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:44 smithi142 ceph-mon[103614]: osdmap e503: 8 total, 7 up, 6 in 2024-08-17T17:55:44.603 INFO:teuthology.orchestra.run.smithi142.stdout:osd.0: {} 2024-08-17T17:55:44.603 INFO:teuthology.orchestra.run.smithi142.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:44.609 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:44.620 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:44.623 INFO:teuthology.orchestra.run.smithi142.stdout:osd.1: {} 2024-08-17T17:55:44.624 INFO:teuthology.orchestra.run.smithi142.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:44.642 INFO:teuthology.orchestra.run.smithi142.stdout:osd.2: {} 2024-08-17T17:55:44.642 INFO:teuthology.orchestra.run.smithi142.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:44.657 INFO:teuthology.orchestra.run.smithi142.stdout:osd.3: {} 2024-08-17T17:55:44.658 INFO:teuthology.orchestra.run.smithi142.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:44.673 INFO:teuthology.orchestra.run.smithi142.stdout:osd.4: {} 2024-08-17T17:55:44.674 INFO:teuthology.orchestra.run.smithi142.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:44.689 INFO:teuthology.orchestra.run.smithi142.stdout:osd.5: {} 2024-08-17T17:55:44.689 INFO:teuthology.orchestra.run.smithi142.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:44.704 INFO:teuthology.orchestra.run.smithi142.stdout:osd.6: {} 2024-08-17T17:55:44.705 INFO:teuthology.orchestra.run.smithi142.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:44.705 INFO:teuthology.orchestra.run.smithi142.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:55:44.722 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:44.744 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:44.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:44.869 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:44.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:44.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:44.993 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:44.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:45.095 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.118 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:45.219 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:45.242 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:45.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:45.367 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:45.469 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:45.491 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:45.563 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1650 ms) 2024-08-17T17:55:45.563 INFO:tasks.workunit.client.0.smithi142.stdout:[ RUN ] TestClsRbd.sparsify 2024-08-17T17:55:45.593 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:45.603 INFO:tasks.workunit.client.0.smithi142.stdout:[ OK ] TestClsRbd.sparsify (40 ms) 2024-08-17T17:55:45.614 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:45.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:45 smithi152 ceph-mon[93456]: osdmap e504: 8 total, 7 up, 6 in 2024-08-17T17:55:45.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:45 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2885229802' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:45.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:45 smithi152 ceph-mon[93456]: pgmap v1999: 105 pgs: 1 remapped+peering, 22 peering, 82 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 897 op/s; 10 B/s, 4 keys/s, 1 objects/s recovering 2024-08-17T17:55:45.716 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:45.738 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:45.840 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:45.861 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:45 smithi142 ceph-mon[106729]: osdmap e504: 8 total, 7 up, 6 in 2024-08-17T17:55:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:45 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2885229802' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:45.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:45 smithi142 ceph-mon[106729]: pgmap v1999: 105 pgs: 1 remapped+peering, 22 peering, 82 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 897 op/s; 10 B/s, 4 keys/s, 1 objects/s recovering 2024-08-17T17:55:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:45 smithi142 ceph-mon[103614]: osdmap e504: 8 total, 7 up, 6 in 2024-08-17T17:55:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:45 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2885229802' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:45.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:45 smithi142 ceph-mon[103614]: pgmap v1999: 105 pgs: 1 remapped+peering, 22 peering, 82 active+clean; 582 KiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 897 op/s; 10 B/s, 4 keys/s, 1 objects/s recovering 2024-08-17T17:55:45.892 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:55:45.892 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:55:45.892 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:55:45.892 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:55:45.963 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:45.984 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:45.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:46.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:46.107 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:46.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:55:46.145 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:55:46.210 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:46.231 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:46.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:46.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:46.354 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: docker: command not found 2024-08-17T17:55:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:46.456 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:46.478 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:46.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:55:46.568 INFO:tasks.workunit.client.0.smithi142.stdout:[----------] 70 tests from TestClsRbd (6075 ms total) 2024-08-17T17:55:46.568 INFO:tasks.workunit.client.0.smithi142.stdout: 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout:[----------] Global test environment tear-down 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout:[==========] 70 tests from 1 test suite ran. (8164 ms total) 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout:[ PASSED ] 69 tests. 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout:[ FAILED ] 1 test, listed below: 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout: 2024-08-17T17:55:46.569 INFO:tasks.workunit.client.0.smithi142.stdout: 1 FAILED TEST 2024-08-17T17:55:46.580 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.583 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-08-17T17:55:46.583 DEBUG:teuthology.orchestra.run.smithi142:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:55:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.602 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:46.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:46.703 DEBUG:teuthology.orchestra.run.smithi142:> sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:46.726 INFO:teuthology.orchestra.run.smithi142.stderr:sudo: docker: command not found 2024-08-17T17:55:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi142 with status 1: 'sudo docker kill -s 1 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:46.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:46 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:55:46.798 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:46 smithi142 ceph-mon[106729]: osdmap e505: 8 total, 7 up, 6 in 2024-08-17T17:55:46.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:46 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:55:46.799 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:46 smithi142 ceph-mon[103614]: osdmap e505: 8 total, 7 up, 6 in 2024-08-17T17:55:46.800 INFO:tasks.thrashosds:joining thrashosds 2024-08-17T17:55:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:46 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:55:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:46 smithi152 ceph-mon[93456]: osdmap e505: 8 total, 7 up, 6 in 2024-08-17T17:55:46.823 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:47.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:47 smithi152 ceph-mon[93456]: osdmap e506: 8 total, 7 up, 6 in 2024-08-17T17:55:47.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:47 smithi152 ceph-mon[93456]: pgmap v2002: 73 pgs: 1 remapped+peering, 17 peering, 55 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 142 B/s, 1 objects/s recovering 2024-08-17T17:55:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:47 smithi142 ceph-mon[103614]: osdmap e506: 8 total, 7 up, 6 in 2024-08-17T17:55:47.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:47 smithi142 ceph-mon[103614]: pgmap v2002: 73 pgs: 1 remapped+peering, 17 peering, 55 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 142 B/s, 1 objects/s recovering 2024-08-17T17:55:47.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:47 smithi142 ceph-mon[106729]: osdmap e506: 8 total, 7 up, 6 in 2024-08-17T17:55:47.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:47 smithi142 ceph-mon[106729]: pgmap v2002: 73 pgs: 1 remapped+peering, 17 peering, 55 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 142 B/s, 1 objects/s recovering 2024-08-17T17:55:48.470 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:55:48.470 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:55:48.470 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:55:48.470 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:55:48.470 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:55:48.584 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is set 2024-08-17T17:55:48.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:48 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1502349816' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:48 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:48 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1502349816' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:48.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:48 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:49.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:48 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1502349816' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:49.021 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:48 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:49.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:55:49.300 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:49.369 INFO:tasks.thrashosds.thrasher:reviving osd 2024-08-17T17:55:49.369 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2024-08-17T17:55:49.369 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2024-08-17T17:55:49.369 DEBUG:teuthology.orchestra.run.smithi152:osd.7> sudo journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T17:55:49.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl start ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:55:49.726 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:49 smithi152 ceph-mon[93456]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:49.727 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:49 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:55:49.727 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:49 smithi152 ceph-mon[93456]: osdmap e507: 8 total, 7 up, 6 in 2024-08-17T17:55:49.727 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:49 smithi152 ceph-mon[93456]: pgmap v2004: 73 pgs: 1 remapped+peering, 72 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-08-17T17:55:49.727 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:49 smithi152 systemd[1]: Starting Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:55:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[103614]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:55:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[103614]: osdmap e507: 8 total, 7 up, 6 in 2024-08-17T17:55:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[103614]: pgmap v2004: 73 pgs: 1 remapped+peering, 72 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-08-17T17:55:49.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[106729]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:49.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:55:49.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[106729]: osdmap e507: 8 total, 7 up, 6 in 2024-08-17T17:55:49.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:49 smithi142 ceph-mon[106729]: pgmap v2004: 73 pgs: 1 remapped+peering, 72 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-08-17T17:55:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:55:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:55:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:50.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:50 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:55:50.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:50 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:55:50.891 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:50 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:55:50.891 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:50 smithi152 podman[301395]: 2024-08-17 17:55:50.683769646 +0000 UTC m=+0.451654819 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:55:51.125 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201795 13.1 13:8ad4d0e6:::benchmark_data_smithi142_393845_object73979:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e444)", 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 316.50791662300003, 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0760220620000001, 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:55:51.126 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201795 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593843+0000", 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593845+0000", 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9800000000000001e-06 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593850+0000", 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3619999999999997e-06 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.593867+0000", 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6390999999999999e-05 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601117+0000", 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072505139999999996 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601123+0000", 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9089999999999997e-06 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.651452+0000", 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.05032927 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.651561+0000", 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010834 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.669765+0000", 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018204426999999999 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.669865+0000", 2024-08-17T17:55:51.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100139 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19432 14.5 14:a0a12a2e:::benchmark_data_smithi142_428506_object19431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.007833+0000", 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 218.09392664399999, 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0758336019999999, 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.132 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19432 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007833+0000", 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007833+0000", 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007832+0000", 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007937+0000", 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104953 2024-08-17T17:55:51.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007956+0000", 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9165e-05 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.007962+0000", 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4819999999999999e-06 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.008220+0000", 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025818 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.008337+0000", 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116759 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.008477+0000", 2024-08-17T17:55:51.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000140261 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.009817+0000", 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013396219999999999 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017464+0000", 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076474519999999999 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017477+0000", 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2092999999999999e-05 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018015+0000", 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000538084 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018024+0000", 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0969999999999995e-06 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.083579+0000", 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.065555121 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.083667+0000", 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7758999999999998e-05 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19441 14.5 14:a5394b9a:::benchmark_data_smithi142_428506_object19440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:55:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.012249+0000", 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 218.089510674, 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0783817920000001, 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19441 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012249+0000", 2024-08-17T17:55:51.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012249+0000", 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012248+0000", 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012290+0000", 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2481000000000002e-05 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012302+0000", 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1325000000000001e-05 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.012318+0000", 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6507999999999998e-05 2024-08-17T17:55:51.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017498+0000", 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051802669999999997 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.017505+0000", 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2430000000000002e-06 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018748+0000", 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0012432849999999 2024-08-17T17:55:51.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.018756+0000", 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3559999999999993e-06 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084557+0000", 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0658009470000001 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084701+0000", 2024-08-17T17:55:51.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014419899999999999 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084862+0000", 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016102299999999999 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.086561+0000", 2024-08-17T17:55:51.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016981559999999999 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089458+0000", 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028972920000000001 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089538+0000", 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0406000000000002e-05 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089598+0000", 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9762999999999997e-05 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.090631+0000", 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010327509999999999 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19451 14.c 14:3fda18e8:::benchmark_data_smithi142_428506_object19450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.028400+0000", 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 218.07335997000001, 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0625520659999999, 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19451 2024-08-17T17:55:51.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028400+0000", 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028400+0000", 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028398+0000", 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028434+0000", 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5178000000000001e-05 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028449+0000", 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5191000000000001e-05 2024-08-17T17:55:51.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028455+0000", 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.251e-06 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028732+0000", 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027675300000000003 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028741+0000", 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7110000000000007e-06 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.015985+0000", 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98724387300000005 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.015997+0000", 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1564e-05 2024-08-17T17:55:51.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084473+0000", 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.068476583 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084629+0000", 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015552799999999999 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:55:51.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.084789+0000", 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000159803 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.088114+0000", 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033248280000000002 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.088150+0000", 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6554999999999998e-05 2024-08-17T17:55:51.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089572+0000", 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014218189999999999 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.089618+0000", 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6201e-05 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.090952+0000", 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013335269999999999 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.156 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:204973 14.e 14:7bde1989:::benchmark_data_smithi142_428506_object86273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:47.266561+0000", 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 63.835198794, 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042040801000000003, 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 204973 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.266561+0000", 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.266561+0000", 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.266561+0000", 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.266563+0000", 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1670000000000002e-06 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.266568+0000", 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0479999999999998e-06 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.266571+0000", 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1109999999999999e-06 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.280817+0000", 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014246112 2024-08-17T17:55:51.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.280922+0000", 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105406 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.281027+0000", 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104374 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.289283+0000", 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082561519999999992 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.293364+0000", 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040812119999999999 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.308483+0000", 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015119120999999999 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.308564+0000", 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0420000000000006e-05 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:47.308602+0000", 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7996e-05 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:205959 14.5 14:a4b3d0db:::benchmark_data_smithi142_428506_object87259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:48.056953+0000", 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 63.044806756, 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042402522999999998, 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 205959 2024-08-17T17:55:51.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.056953+0000", 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.056953+0000", 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.056952+0000", 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.056966+0000", 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3621999999999999e-05 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.056978+0000", 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2021000000000001e-05 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.056983+0000", 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.485e-06 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.057611+0000", 2024-08-17T17:55:51.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062768999999999998 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.057710+0000", 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9291999999999997e-05 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.057807+0000", 2024-08-17T17:55:51.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7199000000000002e-05 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.060780+0000", 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002972817 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.068778+0000", 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079978559999999994 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.099276+0000", 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030498158000000001 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.099319+0000", 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3062000000000002e-05 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.099355+0000", 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6142000000000001e-05 2024-08-17T17:55:51.170 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:205964 14.5 14:a371e0b1:::benchmark_data_smithi142_428506_object87264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:48.062432+0000", 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 63.039327509000003, 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.042725134999999997, 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.171 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 205964 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062432+0000", 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062432+0000", 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062431+0000", 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062436+0000", 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2070000000000003e-06 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062447+0000", 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0818e-05 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062454+0000", 2024-08-17T17:55:51.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4779999999999996e-06 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062655+0000", 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020135300000000001 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062773+0000", 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011817800000000001 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.062951+0000", 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017827000000000001 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.067180+0000", 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042285589999999998 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.068834+0000", 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001654431 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.105082+0000", 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036247869000000002 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.105123+0000", 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0399999999999999e-05 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:48.105157+0000", 2024-08-17T17:55:51.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4709999999999998e-05 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:207150 14.5 14:adc6e1c3:::benchmark_data_smithi142_428506_object88450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:49.040716+0000", 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 62.061043642999998, 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040590538000000002, 2024-08-17T17:55:51.178 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 207150 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.040716+0000", 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.040716+0000", 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.040715+0000", 2024-08-17T17:55:51.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.040719+0000", 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9759999999999997e-06 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.040730+0000", 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0176e-05 2024-08-17T17:55:51.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.040734+0000", 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4170000000000002e-06 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.045064+0000", 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043299480000000001 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.045159+0000", 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5575999999999994e-05 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.045260+0000", 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100473 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.054216+0000", 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089564989999999997 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.054381+0000", 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000164819 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.081205+0000", 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026823433000000001 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.081274+0000", 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9195000000000004e-05 2024-08-17T17:55:51.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.081307+0000", 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2663000000000002e-05 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:207154 14.5 14:a128cef3:::benchmark_data_smithi142_428506_object88454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2024-08-17T17:55:51.185 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:49.041227+0000", 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 62.060532733000002, 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040195428999999998, 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 207154 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.186 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.041227+0000", 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.041227+0000", 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.041227+0000", 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.041229+0000", 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4159999999999998e-06 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.041234+0000", 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4229999999999996e-06 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.041238+0000", 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4340000000000001e-06 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.051185+0000", 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0099467919999999994 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.051295+0000", 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110807 2024-08-17T17:55:51.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.051404+0000", 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108453 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.058385+0000", 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069811810000000004 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.058752+0000", 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036723600000000001 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.081334+0000", 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.02258129 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.081393+0000", 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9020000000000001e-05 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:49.081422+0000", 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9762000000000001e-05 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:215459 14.e 14:70d83859:::benchmark_data_smithi142_428506_object96759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:55.849142+0000", 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 55.252617548000003, 2024-08-17T17:55:51.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040138264999999999, 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 215459 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849142+0000", 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849142+0000", 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849141+0000", 2024-08-17T17:55:51.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849148+0000", 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8460000000000002e-06 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849164+0000", 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6093e-05 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849170+0000", 2024-08-17T17:55:51.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3670000000000002e-06 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849257+0000", 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6618000000000003e-05 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849368+0000", 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111069 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.849489+0000", 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012134 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.856023+0000", 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065338769999999996 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.856079+0000", 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5686999999999997e-05 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.889188+0000", 2024-08-17T17:55:51.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.033109178000000003 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.889243+0000", 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4945999999999999e-05 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:55.889280+0000", 2024-08-17T17:55:51.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7685999999999998e-05 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:216598 14.e 14:79e3d155:::benchmark_data_smithi142_428506_object97898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:56.796731+0000", 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.305028706000002, 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.078331486000000006, 2024-08-17T17:55:51.200 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 216598 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796731+0000", 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796731+0000", 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796729+0000", 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796736+0000", 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1380000000000001e-06 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796749+0000", 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2543e-05 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796755+0000", 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1940000000000003e-06 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796867+0000", 2024-08-17T17:55:51.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111524 2024-08-17T17:55:51.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.797000+0000", 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000133709 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.797134+0000", 2024-08-17T17:55:51.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013345300000000001 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.800600+0000", 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034662109999999999 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.802106+0000", 2024-08-17T17:55:51.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015059909999999999 2024-08-17T17:55:51.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.874970+0000", 2024-08-17T17:55:51.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.072864334000000003 2024-08-17T17:55:51.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.875022+0000", 2024-08-17T17:55:51.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1397999999999998e-05 2024-08-17T17:55:51.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.875062+0000", 2024-08-17T17:55:51.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0717000000000003e-05 2024-08-17T17:55:51.210 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.210 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.210 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.211 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:219871 14.c 14:3d0f0822:::benchmark_data_smithi142_428506_object101171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.211 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:59.480874+0000", 2024-08-17T17:55:51.211 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 51.620885987000001, 2024-08-17T17:55:51.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.050231682999999999, 2024-08-17T17:55:51.212 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.212 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.212 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.212 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.212 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.213 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 219871 2024-08-17T17:55:51.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.213 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.480874+0000", 2024-08-17T17:55:51.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.480874+0000", 2024-08-17T17:55:51.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.480873+0000", 2024-08-17T17:55:51.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:55:51.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.480878+0000", 2024-08-17T17:55:51.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.242e-06 2024-08-17T17:55:51.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.480890+0000", 2024-08-17T17:55:51.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2159e-05 2024-08-17T17:55:51.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.480896+0000", 2024-08-17T17:55:51.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5559999999999998e-06 2024-08-17T17:55:51.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.487118+0000", 2024-08-17T17:55:51.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062222830000000003 2024-08-17T17:55:51.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.487221+0000", 2024-08-17T17:55:51.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103193 2024-08-17T17:55:51.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:55:51.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.487331+0000", 2024-08-17T17:55:51.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011003 2024-08-17T17:55:51.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.497067+0000", 2024-08-17T17:55:51.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0097359530000000003 2024-08-17T17:55:51.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.503694+0000", 2024-08-17T17:55:51.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066264599999999998 2024-08-17T17:55:51.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.531025+0000", 2024-08-17T17:55:51.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027330896 2024-08-17T17:55:51.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.531069+0000", 2024-08-17T17:55:51.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4678000000000002e-05 2024-08-17T17:55:51.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.531105+0000", 2024-08-17T17:55:51.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6124000000000002e-05 2024-08-17T17:55:51.226 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.226 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.226 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.227 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:219874 14.c 14:3c3b1555:::benchmark_data_smithi142_428506_object101174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.227 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:59.484274+0000", 2024-08-17T17:55:51.227 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 51.617486063999998, 2024-08-17T17:55:51.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.046962073, 2024-08-17T17:55:51.228 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.228 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.228 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.228 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.228 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.228 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 219874 2024-08-17T17:55:51.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.229 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.484274+0000", 2024-08-17T17:55:51.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.484274+0000", 2024-08-17T17:55:51.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.484272+0000", 2024-08-17T17:55:51.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.484278+0000", 2024-08-17T17:55:51.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.981e-06 2024-08-17T17:55:51.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.484289+0000", 2024-08-17T17:55:51.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0597e-05 2024-08-17T17:55:51.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.484294+0000", 2024-08-17T17:55:51.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4160000000000003e-06 2024-08-17T17:55:51.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.490214+0000", 2024-08-17T17:55:51.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00591969 2024-08-17T17:55:51.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.490314+0000", 2024-08-17T17:55:51.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010012499999999999 2024-08-17T17:55:51.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:55:51.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.490416+0000", 2024-08-17T17:55:51.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101585 2024-08-17T17:55:51.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.497326+0000", 2024-08-17T17:55:51.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069101680000000004 2024-08-17T17:55:51.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.505747+0000", 2024-08-17T17:55:51.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084210029999999998 2024-08-17T17:55:51.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.531135+0000", 2024-08-17T17:55:51.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025387743000000001 2024-08-17T17:55:51.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.531205+0000", 2024-08-17T17:55:51.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0246000000000005e-05 2024-08-17T17:55:51.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.531236+0000", 2024-08-17T17:55:51.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0920000000000002e-05 2024-08-17T17:55:51.243 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.243 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.243 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.244 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:220044 14.c 14:35ee51df:::benchmark_data_smithi142_428506_object101344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.244 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:59.617205+0000", 2024-08-17T17:55:51.244 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 51.484555102000002, 2024-08-17T17:55:51.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043413294999999998, 2024-08-17T17:55:51.245 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.245 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.245 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.245 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.245 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.245 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 220044 2024-08-17T17:55:51.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.246 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.617205+0000", 2024-08-17T17:55:51.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.617205+0000", 2024-08-17T17:55:51.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.617204+0000", 2024-08-17T17:55:51.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:55:51.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.617208+0000", 2024-08-17T17:55:51.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2540000000000003e-06 2024-08-17T17:55:51.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.617219+0000", 2024-08-17T17:55:51.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.133e-05 2024-08-17T17:55:51.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.617225+0000", 2024-08-17T17:55:51.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.665e-06 2024-08-17T17:55:51.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.620256+0000", 2024-08-17T17:55:51.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030306859999999999 2024-08-17T17:55:51.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.620376+0000", 2024-08-17T17:55:51.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012071800000000001 2024-08-17T17:55:51.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:55:51.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.620587+0000", 2024-08-17T17:55:51.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000210979 2024-08-17T17:55:51.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.623932+0000", 2024-08-17T17:55:51.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003344133 2024-08-17T17:55:51.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.625896+0000", 2024-08-17T17:55:51.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019641649999999999 2024-08-17T17:55:51.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.660531+0000", 2024-08-17T17:55:51.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.034635302 2024-08-17T17:55:51.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.660571+0000", 2024-08-17T17:55:51.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0451999999999998e-05 2024-08-17T17:55:51.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:59.660618+0000", 2024-08-17T17:55:51.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6431000000000003e-05 2024-08-17T17:55:51.259 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.259 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.259 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.260 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:222490 14.e 14:7041f62f:::benchmark_data_smithi142_428506_object103790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.260 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:55:01.845595+0000", 2024-08-17T17:55:51.260 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.256164501000001, 2024-08-17T17:55:51.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043172296999999998, 2024-08-17T17:55:51.261 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.261 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.261 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.261 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.261 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.262 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 222490 2024-08-17T17:55:51.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.262 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845595+0000", 2024-08-17T17:55:51.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845595+0000", 2024-08-17T17:55:51.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845594+0000", 2024-08-17T17:55:51.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845599+0000", 2024-08-17T17:55:51.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7639999999999996e-06 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845608+0000", 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7510000000000003e-06 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845613+0000", 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0440000000000003e-06 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.849288+0000", 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036743230000000002 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.849396+0000", 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108013 2024-08-17T17:55:51.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.849506+0000", 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109929 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.857588+0000", 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0080818190000000005 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.857621+0000", 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3285999999999999e-05 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.888686+0000", 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031065243999999999 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.888732+0000", 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6190999999999999e-05 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.888767+0000", 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5176999999999999e-05 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:222491 14.e 14:7b9e6226:::benchmark_data_smithi142_428506_object103791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:55:01.845619+0000", 2024-08-17T17:55:51.270 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.256141063999998, 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.043258233, 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 222491 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845619+0000", 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845619+0000", 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845618+0000", 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845620+0000", 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2479999999999999e-06 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845626+0000", 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1030000000000001e-06 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.845629+0000", 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4489999999999999e-06 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.852805+0000", 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071763620000000004 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.852949+0000", 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014388899999999999 2024-08-17T17:55:51.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.853062+0000", 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112497 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.857607+0000", 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045449970000000003 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.857709+0000", 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102233 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.888800+0000", 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031090587999999999 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.888841+0000", 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1226000000000001e-05 2024-08-17T17:55:51.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:01.888877+0000", 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6041000000000003e-05 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:222675 14.c 14:365e8c48:::benchmark_data_smithi142_428506_object103975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:55:02.012604+0000", 2024-08-17T17:55:51.277 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.089155869999999, 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.093266392000000004, 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 222675 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012604+0000", 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012604+0000", 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012603+0000", 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012606+0000", 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9550000000000001e-06 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012613+0000", 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.601e-06 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012617+0000", 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9489999999999998e-06 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012669+0000", 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2380000000000003e-05 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012778+0000", 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108632 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.012890+0000", 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112642 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.022273+0000", 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093826280000000005 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023152+0000", 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00087898299999999996 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.105783+0000", 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082630729999999999 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.105824+0000", 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1085999999999999e-05 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.105870+0000", 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6285000000000002e-05 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:222678 14.5 14:a71901f8:::benchmark_data_smithi142_428506_object103978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:55:02.018706+0000", 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.083053425000003, 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.074665755, 2024-08-17T17:55:51.284 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 222678 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018706+0000", 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018706+0000", 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018705+0000", 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018710+0000", 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2220000000000001e-06 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018719+0000", 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2569999999999996e-06 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018725+0000", 2024-08-17T17:55:51.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.237e-06 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018921+0000", 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019615100000000001 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.019019+0000", 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8515000000000001e-05 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.019120+0000", 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100943 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.022289+0000", 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031691309999999999 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.022896+0000", 2024-08-17T17:55:51.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00060676700000000003 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.093291+0000", 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.070395104999999999 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.093336+0000", 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5197000000000001e-05 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.093372+0000", 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5675e-05 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:222679 14.c 14:32943673:::benchmark_data_smithi142_428506_object103979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:55:02.018729+0000", 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.083030444000002, 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.087312380999999994, 2024-08-17T17:55:51.291 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 222679 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018729+0000", 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018729+0000", 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018729+0000", 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018731+0000", 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2220000000000001e-06 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018736+0000", 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8520000000000003e-06 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018739+0000", 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2959999999999999e-06 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018794+0000", 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5067e-05 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.018929+0000", 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013476499999999999 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.019067+0000", 2024-08-17T17:55:51.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013777400000000001 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.022519+0000", 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034525879999999999 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023298+0000", 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00077881200000000004 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.105932+0000", 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082633542000000004 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.106005+0000", 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3029999999999997e-05 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.106042+0000", 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6736999999999997e-05 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.297 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:222685 14.5 14:a6f65fe4:::benchmark_data_smithi142_428506_object103985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:55:02.023936+0000", 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.077824075000002, 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.077890213, 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:55:51.298 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 222685 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023936+0000", 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023936+0000", 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:55:51.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023934+0000", 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023941+0000", 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3899999999999998e-06 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:55:51.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023952+0000", 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1659e-05 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.023967+0000", 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4453e-05 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.026319+0000", 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023519130000000002 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.026520+0000", 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020103100000000001 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.026629+0000", 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010912 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.029506+0000", 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028770990000000001 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.029604+0000", 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7928000000000002e-05 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.101741+0000", 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.072136979000000004 2024-08-17T17:55:51.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.101784+0000", 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3516999999999997e-05 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:55:02.101826+0000", 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1458000000000002e-05 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:55:51.305 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:55:51.305 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:55:51.473 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:55:51.760 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:51.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:51 smithi142 ceph-mon[106729]: pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 97 KiB/s, 2 objects/s recovering 2024-08-17T17:55:51.772 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:51 smithi142 ceph-mon[103614]: pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 97 KiB/s, 2 objects/s recovering 2024-08-17T17:55:51.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:51 smithi152 ceph-mon[93456]: pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 97 KiB/s, 2 objects/s recovering 2024-08-17T17:55:51.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:55:52.043 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:55:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:55:51] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-08-17T17:55:52.242 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 podman[301395]: 2024-08-17 17:55:52.115541369 +0000 UTC m=+1.883426519 container create 14940adc0785812adcd56cf756971abe32c33136e4323c299552cea8f0e68720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, ceph=True) 2024-08-17T17:55:52.490 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:52.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 podman[301395]: 2024-08-17 17:55:52.352018352 +0000 UTC m=+2.119903529 container init 14940adc0785812adcd56cf756971abe32c33136e4323c299552cea8f0e68720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:55:52.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 podman[301395]: 2024-08-17 17:55:52.357222588 +0000 UTC m=+2.125107745 container start 14940adc0785812adcd56cf756971abe32c33136e4323c299552cea8f0e68720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:55:52.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 podman[301395]: 2024-08-17 17:55:52.451688641 +0000 UTC m=+2.219573805 container attach 14940adc0785812adcd56cf756971abe32c33136e4323c299552cea8f0e68720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:55:53.058 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:52 smithi142 ceph-mon[106729]: pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 83 KiB/s, 1 objects/s recovering 2024-08-17T17:55:53.058 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:52 smithi142 ceph-mon[103614]: pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 83 KiB/s, 1 objects/s recovering 2024-08-17T17:55:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-mon[93456]: pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 83 KiB/s, 1 objects/s recovering 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:55:53.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate[301510]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 bash[301395]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:55:53.074 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:52 smithi152 podman[301395]: 2024-08-17 17:55:52.758880244 +0000 UTC m=+2.526765400 container died 14940adc0785812adcd56cf756971abe32c33136e4323c299552cea8f0e68720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:55:53.410 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:53 smithi152 podman[301700]: 2024-08-17 17:55:53.308339882 +0000 UTC m=+0.622342132 container remove 14940adc0785812adcd56cf756971abe32c33136e4323c299552cea8f0e68720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-activate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T17:55:53.819 INFO:teuthology.orchestra.run.smithi142.stderr:noscrub is unset 2024-08-17T17:55:54.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1044692053' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:54.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1044692053' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:54.138 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1044692053' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:54.594 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:55:54.855 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:55.001 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:55:55.001 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:55:55.002 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:55:55.002 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:55:55.002 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:54 smithi152 ceph-mon[93456]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-08-17T17:55:55.002 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:54 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:55:55.002 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1044692053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:55.002 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:54 smithi152 ceph-mon[93456]: osdmap e508: 8 total, 7 up, 6 in 2024-08-17T17:55:55.002 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:54 smithi152 ceph-mon[93456]: pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 73 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:55:55.119 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[106729]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-08-17T17:55:55.119 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:55:55.119 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1044692053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:55.119 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[106729]: osdmap e508: 8 total, 7 up, 6 in 2024-08-17T17:55:55.119 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[106729]: pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 73 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:55:55.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[103614]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-08-17T17:55:55.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:55:55.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1044692053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:55.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[103614]: osdmap e508: 8 total, 7 up, 6 in 2024-08-17T17:55:55.120 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:54 smithi142 ceph-mon[103614]: pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 73 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:55:55.556 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:55 smithi152 podman[301851]: 2024-08-17 17:55:55.204188353 +0000 UTC m=+0.747572353 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:55:55.583 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:55:55.822 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:55 smithi152 podman[301851]: 2024-08-17 17:55:55.74759364 +0000 UTC m=+1.290977630 container create 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:55:56.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:55:56.279 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:56.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:56 smithi152 podman[301851]: 2024-08-17 17:55:56.000427378 +0000 UTC m=+1.543811380 container init 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:55:56.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:56 smithi152 podman[301851]: 2024-08-17 17:55:56.005343353 +0000 UTC m=+1.548727346 container start 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:55:56.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:56 smithi152 ceph-osd[301961]: -- 172.21.15.152:0/12554763 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556625fca1a0 con 0x556625f9d400 2024-08-17T17:55:56.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:56 smithi152 bash[301851]: 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f 2024-08-17T17:55:56.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:56 smithi152 systemd[1]: Started Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:55:56.712 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:55:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:55:56.558+0000 7f3896f65440 -1 Falling back to public interface 2024-08-17T17:55:57.027 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:57.471 INFO:teuthology.orchestra.run.smithi142.stderr:nodeep-scrub is unset 2024-08-17T17:55:57.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:57.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:57.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:57 smithi152 ceph-mon[93456]: pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T17:55:57.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:57 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2680034072' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:57.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:57 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:57.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:57 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:57.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[106729]: pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2680034072' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[103614]: pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2680034072' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:57.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:57 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:55:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:58 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:58.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:58 smithi142 ceph-mon[103614]: osdmap e509: 8 total, 7 up, 6 in 2024-08-17T17:55:58.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:58 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:58.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:58 smithi142 ceph-mon[106729]: osdmap e509: 8 total, 7 up, 6 in 2024-08-17T17:55:58.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:58 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:58.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:58 smithi152 ceph-mon[93456]: osdmap e509: 8 total, 7 up, 6 in 2024-08-17T17:55:59.675 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:55:59 smithi152 ceph-mon[93456]: pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:0 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:1 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:2 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore bluefs device info [] 2024-08-17T17:55:59.678 INFO:teuthology.orchestra.run.smithi152.stderr:config diff 2024-08-17T17:55:59.679 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2024-08-17T17:55:59.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:55:59 smithi142 ceph-mon[106729]: pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:55:59.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:55:59 smithi142 ceph-mon[103614]: pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:00.537 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:56:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:56:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:00.811 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:00 smithi152 ceph-mon[93456]: pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:56:00.837 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2024-08-17T17:56:01.045 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:01.045 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:56:01.045 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:56:01.045 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:56:01.046 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:01.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:00 smithi142 ceph-mon[106729]: pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:01.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:00 smithi142 ceph-mon[103614]: pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:01.680 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:56:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:56:01.469+0000 7f3896f65440 -1 osd.7 435 log_to_monitors true 2024-08-17T17:56:01.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:56:01.979 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:01 smithi152 ceph-mon[93456]: from='osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:56:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:01 smithi152 ceph-mon[93456]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:56:02.073 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:56:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:56:01.716+0000 7f388d50e640 -1 osd.7 510 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:56:02.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:01 smithi142 ceph-mon[103614]: from='osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:56:02.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:01 smithi142 ceph-mon[103614]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:56:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:01 smithi142 ceph-mon[106729]: from='osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:56:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:01 smithi142 ceph-mon[106729]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:56:02.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:56:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:56:01] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.51.0" 2024-08-17T17:56:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:02 smithi152 ceph-mon[93456]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:56:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:02 smithi152 ceph-mon[93456]: osdmap e510: 8 total, 7 up, 6 in 2024-08-17T17:56:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:02 smithi152 ceph-mon[93456]: from='osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:56:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:02 smithi152 ceph-mon[93456]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:56:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:02 smithi152 ceph-mon[93456]: pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[103614]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:56:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[103614]: osdmap e510: 8 total, 7 up, 6 in 2024-08-17T17:56:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[103614]: from='osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:56:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[103614]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:56:03.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[103614]: pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[106729]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[106729]: osdmap e510: 8 total, 7 up, 6 in 2024-08-17T17:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[106729]: from='osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[106729]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]} : dispatch 2024-08-17T17:56:03.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:02 smithi142 ceph-mon[106729]: pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:03.668 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201788 13.9 13:9ea93233:::benchmark_data_smithi142_393845_object73972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.565727+0000", 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.10012157, 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0761014870000001, 2024-08-17T17:56:03.669 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201788 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.565727+0000", 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.565727+0000", 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.565726+0000", 2024-08-17T17:56:03.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.565730+0000", 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1930000000000002e-06 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.565737+0000", 2024-08-17T17:56:03.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4769999999999997e-06 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.565742+0000", 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4479999999999996e-06 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574352+0000", 2024-08-17T17:56:03.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086107719999999992 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574481+0000", 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012914299999999999 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574598+0000", 2024-08-17T17:56:03.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116452 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.583242+0000", 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086440149999999997 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584185+0000", 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00094325400000000003 2024-08-17T17:56:03.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.609848+0000", 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.02566249 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.609864+0000", 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6115000000000001e-05 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.641687+0000", 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0318231120000001 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.641828+0000", 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000141633 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.677 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201793 13.9 13:9441149f:::benchmark_data_smithi142_393845_object73977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.574886+0000", 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.090962725, 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0845260269999999, 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.678 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201793 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574886+0000", 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574886+0000", 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574885+0000", 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574891+0000", 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1399999999999997e-06 2024-08-17T17:56:03.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574903+0000", 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2310000000000001e-05 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.574909+0000", 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7570000000000001e-06 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584824+0000", 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0099151199999999995 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.585098+0000", 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027409199999999998 2024-08-17T17:56:03.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.585205+0000", 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107242 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.609905+0000", 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024700111 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.609911+0000", 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8900000000000004e-06 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.641862+0000", 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0319507699999999 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.641958+0000", 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6002999999999995e-05 2024-08-17T17:56:03.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.659279+0000", 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01732119 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.659412+0000", 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000132712 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201802 13.8 13:1c26506a:::benchmark_data_smithi142_393845_object73986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.580439+0000", 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.08540932800003, 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.113722621, 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.686 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201802 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580439+0000", 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580439+0000", 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580438+0000", 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580443+0000", 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1440000000000002e-06 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580453+0000", 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5910000000000002e-06 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580457+0000", 2024-08-17T17:56:03.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6840000000000004e-06 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580502+0000", 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4274999999999999e-05 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580603+0000", 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101421 2024-08-17T17:56:03.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.580703+0000", 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100309 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.589777+0000", 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090734869999999999 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601819+0000", 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01204218 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601825+0000", 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3489999999999997e-06 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.672025+0000", 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.070199854 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.672145+0000", 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120118 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.694037+0000", 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021891974000000002 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.694162+0000", 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012449999999999999 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.694 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201804 13.9 13:98328538:::benchmark_data_smithi142_393845_object73988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.584927+0000", 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.08092139199999, 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0747911960000001, 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.695 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201804 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584927+0000", 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584927+0000", 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584926+0000", 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584931+0000", 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2689999999999999e-06 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584939+0000", 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6769999999999995e-06 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.584945+0000", 2024-08-17T17:56:03.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9630000000000003e-06 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.591967+0000", 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070219289999999997 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.592105+0000", 2024-08-17T17:56:03.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013780899999999999 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.592227+0000", 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122039 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.609932+0000", 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017705354999999999 2024-08-17T17:56:03.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.609937+0000", 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6709999999999998e-06 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.641984+0000", 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.03204699 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.642060+0000", 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.661e-05 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.659647+0000", 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017587057999999999 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.659718+0000", 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0963999999999993e-05 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201807 13.8 13:1541f792:::benchmark_data_smithi142_393845_object73991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-17T17:56:03.703 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.601971+0000", 2024-08-17T17:56:03.704 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.06387804899998, 2024-08-17T17:56:03.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0920268740000001, 2024-08-17T17:56:03.704 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.704 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.704 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.705 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201807 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601971+0000", 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601971+0000", 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601969+0000", 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601977+0000", 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5399999999999998e-06 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601988+0000", 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1411e-05 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.601994+0000", 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8579999999999996e-06 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602490+0000", 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00049625600000000004 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602496+0000", 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4750000000000001e-06 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.672170+0000", 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.069674357 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.672278+0000", 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108149 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.672381+0000", 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010254100000000001 2024-08-17T17:56:03.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.683711+0000", 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011330420000000001 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.693805+0000", 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01009351 2024-08-17T17:56:03.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.693914+0000", 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108923 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.693997+0000", 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3778999999999999e-05 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.713 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201808 13.9 13:933f0579:::benchmark_data_smithi142_393845_object73992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.602184+0000", 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.063664647, 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.056819671, 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.714 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201808 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602184+0000", 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602184+0000", 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602183+0000", 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602187+0000", 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.366e-06 2024-08-17T17:56:03.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602195+0000", 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4800000000000001e-06 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.602200+0000", 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8709999999999996e-06 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.610564+0000", 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083642200000000003 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.610570+0000", 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3650000000000003e-06 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.642079+0000", 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.031509732 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.642170+0000", 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0854999999999997e-05 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0", 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.642303+0000", 2024-08-17T17:56:03.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000132865 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.645658+0000", 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003355121 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.658837+0000", 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013178952000000001 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.658914+0000", 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7199000000000004e-05 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.659004+0000", 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9258999999999994e-05 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201810 13.8 13:16b4c40f:::benchmark_data_smithi142_393845_object73994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.603844+0000", 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.06200436, 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.09077757, 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201810 2024-08-17T17:56:03.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603844+0000", 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603844+0000", 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603844+0000", 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603847+0000", 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7359999999999999e-06 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603867+0000", 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9791000000000001e-05 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603872+0000", 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5850000000000001e-06 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603920+0000", 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8436000000000002e-05 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.603927+0000", 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7689999999999999e-06 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.677000+0000", 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.073072893 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.677122+0000", 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121833 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2024-08-17T17:56:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.677243+0000", 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121505 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.694005+0000", 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016762037 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.694290+0000", 2024-08-17T17:56:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00028501500000000001 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.694371+0000", 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1094000000000007e-05 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.694622+0000", 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025047600000000002 2024-08-17T17:56:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:201813 13.f 13:f61e950e:::benchmark_data_smithi142_393845_object73997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:50:34.612687+0000", 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 329.05316108599999, 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.021537264, 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201813 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612687+0000", 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612687+0000", 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612686+0000", 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612693+0000", 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1849999999999998e-06 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612707+0000", 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3878e-05 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612715+0000", 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0809999999999994e-06 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612758+0000", 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2864999999999997e-05 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:34.612763+0000", 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.899e-06 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.618099+0000", 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.005335968 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.620168+0000", 2024-08-17T17:56:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020687150000000001 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4", 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.620237+0000", 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9332999999999995e-05 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.624251+0000", 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040138079999999998 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.634054+0000", 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098025490000000007 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.634140+0000", 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6532000000000004e-05 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:50:35.634225+0000", 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4655999999999997e-05 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19445 14.7 14:e350ab27:::benchmark_data_smithi142_428506_object19444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.022438+0000", 2024-08-17T17:56:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 230.64341045800001, 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0527499410000001, 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19445 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022438+0000", 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022438+0000", 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022437+0000", 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022494+0000", 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7648000000000001e-05 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022505+0000", 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1054e-05 2024-08-17T17:56:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022510+0000", 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2880000000000003e-06 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022545+0000", 2024-08-17T17:56:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5821999999999997e-05 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.022551+0000", 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4970000000000001e-06 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010937+0000", 2024-08-17T17:56:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98838603599999997 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010945+0000", 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0860000000000002e-06 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.068648+0000", 2024-08-17T17:56:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0577028289999999 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.068824+0000", 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017595800000000001 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.069014+0000", 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019011700000000001 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.072953+0000", 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039387729999999996 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.073806+0000", 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00085319699999999998 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.073948+0000", 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014255299999999999 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.074016+0000", 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7965000000000004e-05 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.075188+0000", 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011716020000000001 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19447 14.0 14:05c5e464:::benchmark_data_smithi142_428506_object19446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.024802+0000", 2024-08-17T17:56:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 230.641046101, 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.08369048, 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19447 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024802+0000", 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024802+0000", 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024801+0000", 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024830+0000", 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.883e-05 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024845+0000", 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.524e-05 2024-08-17T17:56:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024856+0000", 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0592e-05 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024904+0000", 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8232999999999997e-05 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.024912+0000", 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3939999999999996e-06 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010397+0000", 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.985484056 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010407+0000", 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0947999999999999e-05 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.094560+0000", 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.084152572 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.094702+0000", 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014240300000000001 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.094861+0000", 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000158627 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104247+0000", 2024-08-17T17:56:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.009386152 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104370+0000", 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012229 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.107262+0000", 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028925359999999998 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.107370+0000", 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010842599999999999 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.108493+0000", 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001122477 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.757 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19450 14.0 14:0a83c86e:::benchmark_data_smithi142_428506_object19449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.028072+0000", 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 230.63777672399999, 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.081337988, 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19450 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028072+0000", 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028072+0000", 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028070+0000", 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028105+0000", 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4918e-05 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028114+0000", 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4489999999999999e-06 2024-08-17T17:56:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028118+0000", 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1760000000000003e-06 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028150+0000", 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1964000000000001e-05 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.028155+0000", 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4210000000000003e-06 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010438+0000", 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98228242899999996 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010445+0000", 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1339999999999998e-06 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.095762+0000", 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.085317197 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.095896+0000", 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134091 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.096332+0000", 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00043544299999999998 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104274+0000", 2024-08-17T17:56:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079428869999999992 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104610+0000", 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000335061 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.108507+0000", 2024-08-17T17:56:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038977030000000002 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.108568+0000", 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0358e-05 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.109410+0000", 2024-08-17T17:56:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00084225599999999999 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19456 14.0 14:05503d65:::benchmark_data_smithi142_428506_object19455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.032874+0000", 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 230.63297444400001, 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0772681710000001, 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19456 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.032874+0000", 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.032874+0000", 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.032873+0000", 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.032960+0000", 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6536e-05 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.032971+0000", 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1039999999999999e-05 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.032977+0000", 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.879e-06 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033013+0000", 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6414000000000001e-05 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033018+0000", 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.254e-06 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010471+0000", 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97745219500000002 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010477+0000", 2024-08-17T17:56:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4910000000000002e-06 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.097556+0000", 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0870788120000001 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.097703+0000", 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000147434 2024-08-17T17:56:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.097915+0000", 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000211852 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104299+0000", 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.006384018 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104642+0000", 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034265400000000002 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.109423+0000", 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047815139999999997 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.109506+0000", 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2177999999999998e-05 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.110142+0000", 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00063678699999999999 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19458 14.7 14:e2b2f591:::benchmark_data_smithi142_428506_object19457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.033040+0000", 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 230.63280837400001, 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0432381720000001, 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19458 2024-08-17T17:56:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033040+0000", 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033040+0000", 2024-08-17T17:56:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033039+0000", 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033108+0000", 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8690999999999997e-05 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033118+0000", 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0175e-05 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033123+0000", 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5789999999999997e-06 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033159+0000", 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5698000000000003e-05 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033163+0000", 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7879999999999997e-06 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010974+0000", 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97781035699999996 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010981+0000", 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5340000000000002e-06 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.070173+0000", 2024-08-17T17:56:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.059191376 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.070294+0000", 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121594 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.070438+0000", 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014377500000000001 2024-08-17T17:56:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.072995+0000", 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002556605 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.073835+0000", 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000840328 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.075199+0000", 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001363967 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.075338+0000", 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013898799999999999 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.076278+0000", 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00094051200000000001 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:19459 14.0 14:051cebcd:::benchmark_data_smithi142_428506_object19458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2024-08-17T17:56:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:13.033142+0000", 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 230.63270639000001, 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0777483590000001, 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 19459 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033142+0000", 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033142+0000", 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033142+0000", 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033177+0000", 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5163999999999997e-05 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033184+0000", 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9229999999999996e-06 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033187+0000", 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.129e-06 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033216+0000", 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9447000000000001e-05 2024-08-17T17:56:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:13.033221+0000", 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.101e-06 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010497+0000", 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97727531700000003 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:14.010503+0000", 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2999999999999998e-06 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.098871+0000", 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0883679500000001 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.098989+0000", 2024-08-17T17:56:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011769600000000001 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.099139+0000", 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015043200000000001 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104323+0000", 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051836690000000001 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.104670+0000", 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034692000000000002 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.110191+0000", 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055216459999999998 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.110253+0000", 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1532999999999995e-05 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:15.110891+0000", 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00063768500000000003 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:37625 14.7 14:ec09a65b:::benchmark_data_smithi142_428506_object37624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:25.857803+0000", 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 217.808045357, 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0012162090000001, 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 37625 2024-08-17T17:56:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857803+0000", 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857803+0000", 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857802+0000", 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857867+0000", 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4840999999999998e-05 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857879+0000", 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2808e-05 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857886+0000", 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1850000000000001e-06 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857948+0000", 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2234000000000001e-05 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.857956+0000", 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8809999999999996e-06 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.840495+0000", 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98253939300000004 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.840578+0000", 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2614999999999994e-05 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.840669+0000", 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1865000000000002e-05 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.845440+0000", 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047705070000000002 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.858894+0000", 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01345447 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.858962+0000", 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7129000000000001e-05 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.858996+0000", 2024-08-17T17:56:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4286999999999999e-05 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859019+0000", 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3533e-05 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:37655 14.7 14:ece46412:::benchmark_data_smithi142_428506_object37654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:25.877559+0000", 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 217.78828971300001, 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98167685199999999, 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 37655 2024-08-17T17:56:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877559+0000", 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877559+0000", 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877551+0000", 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999919 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877601+0000", 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0133000000000003e-05 2024-08-17T17:56:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877613+0000", 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1949e-05 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877618+0000", 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5040000000000002e-06 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877683+0000", 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4388999999999996e-05 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.877693+0000", 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8570000000000007e-06 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.841897+0000", 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96420471500000005 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.841961+0000", 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4039000000000001e-05 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.842052+0000", 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0245000000000001e-05 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.845449+0000", 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033977270000000001 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.858930+0000", 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013480262999999999 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859171+0000", 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000241034 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859216+0000", 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5368999999999999e-05 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859236+0000", 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9701e-05 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:37670 14.7 14:e1049e58:::benchmark_data_smithi142_428506_object37669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:25.886585+0000", 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 217.77926359, 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97277057, 2024-08-17T17:56:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 37670 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886585+0000", 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886585+0000", 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886583+0000", 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886620+0000", 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7676000000000003e-05 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886634+0000", 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3930999999999999e-05 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886640+0000", 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.925e-06 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886701+0000", 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0615999999999997e-05 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.886710+0000", 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9279999999999999e-06 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.842891+0000", 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95618098100000004 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.842998+0000", 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107451 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.843130+0000", 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013145499999999999 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.848808+0000", 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056783609999999998 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859076+0000", 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010268361 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859289+0000", 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021309000000000001 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859322+0000", 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2113999999999998e-05 2024-08-17T17:56:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859356+0000", 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4010000000000001e-05 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:37678 14.7 14:eccb6bfb:::benchmark_data_smithi142_428506_object37677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:25.889280+0000", 2024-08-17T17:56:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 217.77656866300001, 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97028137000000003, 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 37678 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889280+0000", 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889280+0000", 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889278+0000", 2024-08-17T17:56:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889332+0000", 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4064000000000003e-05 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889343+0000", 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0421000000000001e-05 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889347+0000", 2024-08-17T17:56:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.437e-06 2024-08-17T17:56:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889400+0000", 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2908999999999998e-05 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.889407+0000", 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4699999999999999e-06 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.844104+0000", 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95469699200000002 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.844212+0000", 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010814099999999999 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.844438+0000", 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022628700000000001 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.850944+0000", 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0065063159999999998 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859413+0000", 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084690310000000001 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859485+0000", 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1106999999999995e-05 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859529+0000", 2024-08-17T17:56:03.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4258999999999998e-05 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859561+0000", 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2450999999999998e-05 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:37728 14.7 14:e2415c7f:::benchmark_data_smithi142_428506_object37727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:25.966162+0000", 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 217.69968624500001, 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89347881799999995, 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 37728 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966162+0000", 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966162+0000", 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966161+0000", 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966189+0000", 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8507e-05 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966202+0000", 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.27e-05 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966208+0000", 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0519999999999998e-06 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966285+0000", 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6763999999999999e-05 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.966295+0000", 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8349999999999999e-06 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.845491+0000", 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87919598899999996 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.845605+0000", 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011431500000000001 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.845748+0000", 2024-08-17T17:56:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014322400000000001 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.850954+0000", 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052055670000000004 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859445+0000", 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084909760000000008 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859594+0000", 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014955400000000001 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859623+0000", 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8486000000000001e-05 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859641+0000", 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8479e-05 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:37744 14.7 14:e20989ba:::benchmark_data_smithi142_428506_object37743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:25.976994+0000", 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 217.688854405, 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.882770677, 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 37744 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.976994+0000", 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.976994+0000", 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.976992+0000", 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.977048+0000", 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5970000000000001e-05 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.977060+0000", 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1962e-05 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.977066+0000", 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3040000000000004e-06 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.977118+0000", 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2188999999999998e-05 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:25.977127+0000", 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.166e-06 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.846650+0000", 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86952326400000002 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.846746+0000", 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5315999999999993e-05 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.846842+0000", 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6946000000000003e-05 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.850961+0000", 2024-08-17T17:56:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004118521 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859673+0000", 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087124660000000003 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859719+0000", 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5349000000000002e-05 2024-08-17T17:56:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859743+0000", 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3896999999999998e-05 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:26.859765+0000", 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2141e-05 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:03.850 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:03 smithi152 ceph-mon[93456]: osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310] boot 2024-08-17T17:56:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:03 smithi152 ceph-mon[93456]: osdmap e511: 8 total, 8 up, 6 in 2024-08-17T17:56:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:03 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:56:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:03 smithi142 ceph-mon[103614]: osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310] boot 2024-08-17T17:56:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:03 smithi142 ceph-mon[103614]: osdmap e511: 8 total, 8 up, 6 in 2024-08-17T17:56:04.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:03 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:56:04.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:03 smithi142 ceph-mon[106729]: osd.7 [v2:172.21.15.152:6824/4168801310,v1:172.21.15.152:6825/4168801310] boot 2024-08-17T17:56:04.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:03 smithi142 ceph-mon[106729]: osdmap e511: 8 total, 8 up, 6 in 2024-08-17T17:56:04.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:03 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:56:04.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:56:04.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:04 smithi152 ceph-mon[93456]: pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:04.827 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:04 smithi142 ceph-mon[103614]: pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:05.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:04 smithi142 ceph-mon[106729]: pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:05.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:56:06.158 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:06.907 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:06.907 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:56:06.907 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:56:06.907 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:06.907 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:06 smithi152 ceph-mon[93456]: pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:07.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:06 smithi142 ceph-mon[103614]: pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:07.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:06 smithi142 ceph-mon[106729]: pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:08.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:56:08.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:08 smithi152 ceph-mon[93456]: pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:09.020 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:09.020 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:56:09.020 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:56:09.020 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:09.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:08 smithi142 ceph-mon[103614]: pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:09.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:08 smithi142 ceph-mon[106729]: pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:09.615 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:09.637 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-17T17:56:09.637 INFO:tasks.thrashosds.thrasher:Adding osd 7 2024-08-17T17:56:09.637 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 7 2024-08-17T17:56:09.904 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:10.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:56:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:10.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:10 smithi152 ceph-mon[93456]: pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:11.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:10 smithi142 ceph-mon[103614]: pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:11.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:10 smithi142 ceph-mon[106729]: pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:11.468 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:11.468 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:56:11.468 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:56:11.468 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:56:11.468 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:11.714 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:11.714 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:12.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:12.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:56:12.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:56:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:56:11] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.51.0" 2024-08-17T17:56:12.366 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:12.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:56:12.127+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e511 definitely_dead 0 2024-08-17T17:56:12.682 INFO:teuthology.orchestra.run.smithi142.stderr:marked in osd.7. 2024-08-17T17:56:12.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/378984389' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-08-17T17:56:12.989 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:12 smithi142 ceph-mon[106729]: pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:12.989 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/378984389' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-08-17T17:56:12.989 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:12 smithi142 ceph-mon[103614]: pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/378984389' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-08-17T17:56:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:12 smithi152 ceph-mon[93456]: pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:13.436 INFO:tasks.thrashosds.thrasher:Added osd 7 2024-08-17T17:56:13.436 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-17T17:56:13.436 INFO:tasks.thrashosds.thrasher:Adding osd 2 2024-08-17T17:56:13.436 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 2 2024-08-17T17:56:13.692 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/378984389' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-08-17T17:56:13.693 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:13 smithi142 ceph-mon[106729]: osdmap e512: 8 total, 8 up, 7 in 2024-08-17T17:56:13.700 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:13.820 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/378984389' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-08-17T17:56:13.820 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:13 smithi152 ceph-mon[93456]: osdmap e512: 8 total, 8 up, 7 in 2024-08-17T17:56:13.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/378984389' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-08-17T17:56:13.951 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:13 smithi142 ceph-mon[103614]: osdmap e512: 8 total, 8 up, 7 in 2024-08-17T17:56:14.059 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:14.059 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:56:14.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:56:14.059 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:56:14.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.059 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253451 13.f 13.53bf25f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.632726+0000", 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.41951401199998, 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99935936599999997, 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253451 2024-08-17T17:56:14.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632726+0000", 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632726+0000", 2024-08-17T17:56:14.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632724+0000", 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632731+0000", 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6189999999999996e-06 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632742+0000", 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1131999999999999e-05 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.632749+0000", 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1040000000000001e-06 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635110+0000", 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023606259999999998 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635117+0000", 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1760000000000004e-06 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632064+0000", 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.996947162 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632085+0000", 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1166e-05 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253457 13.f 13.93d985af (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.634981+0000", 2024-08-17T17:56:14.066 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.417258699, 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99713874400000002, 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253457 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634981+0000", 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634981+0000", 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634980+0000", 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.634986+0000", 2024-08-17T17:56:14.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7799999999999997e-06 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635000+0000", 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4596000000000001e-05 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635006+0000", 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.037e-06 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635157+0000", 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015080399999999999 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635162+0000", 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.882e-06 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632112+0000", 2024-08-17T17:56:14.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99694965400000002 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632120+0000", 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4770000000000003e-06 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.073 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253461 13.1 13.5a2c6081 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.635015+0000", 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.41722515399999, 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99467669400000003, 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.074 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253461 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635015+0000", 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635015+0000", 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635014+0000", 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635017+0000", 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8760000000000001e-06 2024-08-17T17:56:14.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635023+0000", 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1149999999999999e-06 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.635027+0000", 2024-08-17T17:56:14.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7859999999999998e-06 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637454+0000", 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002426896 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637461+0000", 2024-08-17T17:56:14.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.629679+0000", 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99221840800000005 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.629692+0000", 2024-08-17T17:56:14.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2121e-05 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253465 13.f 13.5c3917bf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.636214+0000", 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.41602590500003, 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99594324999999995, 2024-08-17T17:56:14.081 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253465 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636214+0000", 2024-08-17T17:56:14.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636214+0000", 2024-08-17T17:56:14.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636213+0000", 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636218+0000", 2024-08-17T17:56:14.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0590000000000002e-06 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636227+0000", 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8389999999999996e-06 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.636233+0000", 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3959999999999997e-06 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637290+0000", 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010574200000000001 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.637297+0000", 2024-08-17T17:56:14.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5599999999999999e-06 2024-08-17T17:56:14.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632147+0000", 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99485042400000001 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632157+0000", 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0331e-05 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253474 13.f 13.befb60df (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.649934+0000", 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.40230633099998, 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98225535799999997, 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.089 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253474 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649934+0000", 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649934+0000", 2024-08-17T17:56:14.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649932+0000", 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649938+0000", 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5609999999999998e-06 2024-08-17T17:56:14.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649947+0000", 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3719999999999994e-06 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.649954+0000", 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7950000000000002e-06 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.650215+0000", 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026070100000000001 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.650220+0000", 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.327e-06 2024-08-17T17:56:14.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632180+0000", 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98196058399999997 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632189+0000", 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5909999999999996e-06 2024-08-17T17:56:14.094 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253475 13.7 13.5eda9997 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.651001+0000", 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.40123899399998, 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97973126799999999, 2024-08-17T17:56:14.095 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253475 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651001+0000", 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651001+0000", 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651000+0000", 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651005+0000", 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5979999999999999e-06 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651014+0000", 2024-08-17T17:56:14.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6319999999999997e-06 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651019+0000", 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8499999999999999e-06 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651133+0000", 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113795 2024-08-17T17:56:14.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.651141+0000", 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9410000000000007e-06 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630715+0000", 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97957360199999999 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630732+0000", 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7581e-05 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.101 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253477 13.7 13.4a9300f7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.652914+0000", 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.39932645800002, 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97785777200000001, 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.102 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253477 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652914+0000", 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652914+0000", 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652913+0000", 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652921+0000", 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2290000000000003e-06 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652936+0000", 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4642e-05 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652942+0000", 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2759999999999997e-06 2024-08-17T17:56:14.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652982+0000", 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9894000000000001e-05 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.652996+0000", 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4136e-05 2024-08-17T17:56:14.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630762+0000", 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97776666999999995 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.630771+0000", 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8599999999999999e-06 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48005.0:253479 13.f 13.4d8bcbdf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:51:19.655935+0000", 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 294.39630503699999, 2024-08-17T17:56:14.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97635279200000002, 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48005", 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/2501418713", 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 253479 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.110 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655935+0000", 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655935+0000", 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655934+0000", 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655938+0000", 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.25e-06 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655948+0000", 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0138e-05 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655955+0000", 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2009999999999996e-06 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655989+0000", 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4171999999999997e-05 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:19.655994+0000", 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5609999999999998e-06 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632274+0000", 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97627963500000003 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:51:20.632288+0000", 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3783e-05 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:17921 14.2 14:4ae35ce4:::benchmark_data_smithi142_428506_object17920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:12.298240+0000", 2024-08-17T17:56:14.116 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 241.75399991200001, 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.103513878, 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17921 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298240+0000", 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298240+0000", 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298239+0000", 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298294+0000", 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5248999999999999e-05 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298304+0000", 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5349999999999993e-06 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298314+0000", 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0397000000000001e-05 2024-08-17T17:56:14.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298356+0000", 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1950000000000003e-05 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298435+0000", 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8924999999999994e-05 2024-08-17T17:56:14.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:56:14.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.298560+0000", 2024-08-17T17:56:14.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012495400000000001 2024-08-17T17:56:14.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.299215+0000", 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000654629 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.302264+0000", 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030490209999999998 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.401671+0000", 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.099407521999999998 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.401716+0000", 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4910999999999997e-05 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:12.401754+0000", 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7790000000000002e-05 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31202 14.4 14:2ed62245:::benchmark_data_smithi142_428506_object31201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:56:14.126 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.294152+0000", 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75808766700001, 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16861922200000001, 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31202 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.127 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294152+0000", 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294152+0000", 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294151+0000", 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294299+0000", 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148536 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294311+0000", 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1907e-05 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294317+0000", 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5160000000000002e-06 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294364+0000", 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.71e-05 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294501+0000", 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000137026 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.294644+0000", 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143466 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297581+0000", 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002937055 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445885+0000", 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14830384499999999 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.462644+0000", 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016758975999999998 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.462732+0000", 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8158999999999994e-05 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.462772+0000", 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9403999999999997e-05 2024-08-17T17:56:14.134 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31205 14.d 14:bb2a3e9e:::benchmark_data_smithi142_428506_object31204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.296230+0000", 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75601033800001, 2024-08-17T17:56:14.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16761810299999999, 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31205 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296230+0000", 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296230+0000", 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296228+0000", 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296275+0000", 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7293000000000003e-05 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296287+0000", 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1579e-05 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.139 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296292+0000", 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1279999999999999e-06 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296329+0000", 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7184999999999997e-05 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296406+0000", 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6779000000000005e-05 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4,5", 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.296487+0000", 2024-08-17T17:56:14.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0684000000000002e-05 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297726+0000", 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012392690000000001 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445865+0000", 2024-08-17T17:56:14.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14813903 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.463722+0000", 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017856674999999999 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.463811+0000", 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9608e-05 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.463848+0000", 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6455999999999998e-05 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.144 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48311.0:31206 14.e e469/460 14:7831df26:::benchmark_data_smithi142_428506_object31205:head v 469'1965, mlcod=469'1963)", 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75441089200001, 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14908843099999999, 2024-08-17T17:56:14.145 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297828+0000", 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297847+0000", 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8865e-05 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297863+0000", 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5965999999999999e-05 2024-08-17T17:56:14.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297867+0000", 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.391e-06 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297906+0000", 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8538999999999998e-05 2024-08-17T17:56:14.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297921+0000", 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4949999999999999e-05 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.150 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446876+0000", 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14895483500000001 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446918+0000", 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1888000000000003e-05 2024-08-17T17:56:14.151 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48311.0:31200 14.6 e469/466 14:61488a53:::benchmark_data_smithi142_428506_object31199:head v 469'1914, mlcod=464'650)", 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.297830+0000", 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.754410221, 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14994484399999999, 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.152 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297830+0000", 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297830+0000", 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297829+0000", 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297864+0000", 2024-08-17T17:56:14.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5240999999999997e-05 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297875+0000", 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1144e-05 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297881+0000", 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3340000000000001e-06 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297957+0000", 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6713999999999998e-05 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.297975+0000", 2024-08-17T17:56:14.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8131999999999999e-05 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447739+0000", 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14976406 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447775+0000", 2024-08-17T17:56:14.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5163999999999997e-05 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.48311.0:31205 14.d e469/460 ondisk, result = 0)", 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75411066300001, 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14773847200000001, 2024-08-17T17:56:14.158 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:56:14.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298129+0000", 2024-08-17T17:56:14.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:14.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298134+0000", 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.78e-06 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298143+0000", 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2259999999999994e-06 2024-08-17T17:56:14.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.298147+0000", 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4700000000000004e-06 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445838+0000", 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.147690721 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445850+0000", 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1626e-05 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445868+0000", 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8291000000000002e-05 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.164 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.48311.0:31202 14.4 e469/460 ondisk, result = 0)", 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.299312+0000", 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75292803599999, 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.146575443, 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299312+0000", 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299312+0000", 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299311+0000", 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.299316+0000", 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1400000000000002e-06 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445780+0000", 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14646446699999999 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445789+0000", 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5399999999999996e-06 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445872+0000", 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.331e-05 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445879+0000", 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445887+0000", 2024-08-17T17:56:14.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1159999999999999e-06 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(14.6 469/466 [PushOp(14:63d9078e:::benchmark_data_smithi142_428506_object11236:head, version: 464'656, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:63d9078e:::benchmark_data_smithi142_428506_object11236:head@464'656, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.300794+0000", 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75144636900001, 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.146983531, 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:14.169 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300794+0000", 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300794+0000", 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300793+0000", 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.300863+0000", 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0047000000000002e-05 2024-08-17T17:56:14.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445713+0000", 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14485007699999999 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445757+0000", 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4274000000000003e-05 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446965+0000", 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012074659999999999 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.446972+0000", 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.058e-06 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447777+0000", 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00080517399999999997 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.48311.0:31212 14.e e469/460 14:7651fe2b:::benchmark_data_smithi142_428506_object31211:head v 469'1966, mlcod=469'1963)", 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.75000963400001, 2024-08-17T17:56:14.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14557045499999999, 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302230+0000", 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.302249+0000", 2024-08-17T17:56:14.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9341999999999999e-05 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445736+0000", 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.143487059 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445747+0000", 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0898e-05 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445911+0000", 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000164339 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.445935+0000", 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3632e-05 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447779+0000", 2024-08-17T17:56:14.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018443400000000001 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.447801+0000", 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1501e-05 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31956 14.2 14:4c283338:::benchmark_data_smithi142_428506_object31955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.914275+0000", 2024-08-17T17:56:14.179 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.137964713, 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26473845299999998, 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31956 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914275+0000", 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914275+0000", 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914274+0000", 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2024-08-17T17:56:14.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914342+0000", 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8796999999999999e-05 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914355+0000", 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.243e-05 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914361+0000", 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5370000000000003e-06 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914419+0000", 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7663e-05 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914529+0000", 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109883 2024-08-17T17:56:14.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.914647+0000", 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011777699999999999 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.916187+0000", 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015400419999999999 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:14.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.917003+0000", 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00081600499999999996 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.178920+0000", 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26191723099999997 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.178974+0000", 2024-08-17T17:56:14.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3569999999999997e-05 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179014+0000", 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0258e-05 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:31964 14.2 14:432e571c:::benchmark_data_smithi142_428506_object31963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2024-08-17T17:56:14.186 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:52:21.920722+0000", 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 232.13151835100001, 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25839342100000001, 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 31964 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920722+0000", 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920722+0000", 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920721+0000", 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920766+0000", 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5485999999999998e-05 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920777+0000", 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.04e-05 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920782+0000", 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5099999999999998e-06 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920826+0000", 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3488000000000001e-05 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.920939+0000", 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112961 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.921063+0000", 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012485900000000001 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.922398+0000", 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001334996 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:21.922459+0000", 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0294999999999998e-05 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179046+0000", 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25658730200000002 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179084+0000", 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8390999999999999e-05 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:52:22.179115+0000", 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0679000000000002e-05 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.48311.0:216588 14.2 14:471c98fa:::benchmark_data_smithi142_428506_object97888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:54:56.777719+0000", 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 77.274521410000006, 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.104456514, 2024-08-17T17:56:14.193 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.48311", 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.142:0/985443584", 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 216588 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.777719+0000", 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.777719+0000", 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.777718+0000", 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.777723+0000", 2024-08-17T17:56:14.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7450000000000001e-06 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.777734+0000", 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1056000000000001e-05 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.777740+0000", 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5709999999999997e-06 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.779964+0000", 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022245580000000002 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.795346+0000", 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015381430999999999 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.796541+0000", 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011954820000000001 2024-08-17T17:56:14.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.799795+0000", 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032538440000000001 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.799800+0000", 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2299999999999999e-06 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:14.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.882019+0000", 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082218081999999998 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.882137+0000", 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011863500000000001 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:54:56.882175+0000", 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7936999999999999e-05 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.199 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:14.200 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:14.200 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:14 smithi142 ceph-mon[106729]: osdmap e513: 8 total, 8 up, 7 in 2024-08-17T17:56:14.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:14 smithi142 ceph-mon[106729]: pgmap v2023: 73 pgs: 7 peering, 66 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:14.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:14 smithi142 ceph-mon[103614]: osdmap e513: 8 total, 8 up, 7 in 2024-08-17T17:56:14.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:14 smithi142 ceph-mon[103614]: pgmap v2023: 73 pgs: 7 peering, 66 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:15.042 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:14 smithi152 ceph-mon[93456]: osdmap e513: 8 total, 8 up, 7 in 2024-08-17T17:56:15.042 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:14 smithi152 ceph-mon[93456]: pgmap v2023: 73 pgs: 7 peering, 66 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail 2024-08-17T17:56:15.068 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:56:15.326 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:15.699 INFO:teuthology.orchestra.run.smithi142.stderr:marked in osd.2. 2024-08-17T17:56:16.055 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:15 smithi142 ceph-mon[106729]: osdmap e514: 8 total, 8 up, 7 in 2024-08-17T17:56:16.056 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:15 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2078504158' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-08-17T17:56:16.056 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:15 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:56:15.635+0000 7f3d6af2c640 -1 mon.a@0(leader).osd e514 definitely_dead 0 2024-08-17T17:56:16.056 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:15 smithi142 ceph-mon[103614]: osdmap e514: 8 total, 8 up, 7 in 2024-08-17T17:56:16.056 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:15 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2078504158' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-08-17T17:56:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:15 smithi152 ceph-mon[93456]: osdmap e514: 8 total, 8 up, 7 in 2024-08-17T17:56:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:15 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2078504158' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-08-17T17:56:16.391 INFO:tasks.thrashosds.thrasher:Added osd 2 2024-08-17T17:56:16.391 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-08-17T17:56:16.646 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:16.899 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2078504158' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-08-17T17:56:16.899 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:16 smithi152 ceph-mon[93456]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:56:16.899 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:16 smithi152 ceph-mon[93456]: pgmap v2026: 73 pgs: 1 active+undersized+remapped, 7 peering, 65 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:16.899 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:16 smithi152 ceph-mon[93456]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:56:17.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2078504158' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-08-17T17:56:17.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[106729]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:56:17.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[106729]: pgmap v2026: 73 pgs: 1 active+undersized+remapped, 7 peering, 65 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:17.075 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[106729]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:56:17.075 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2078504158' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-08-17T17:56:17.075 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[103614]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:56:17.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[103614]: pgmap v2026: 73 pgs: 1 active+undersized+remapped, 7 peering, 65 active+clean; 582 KiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:17.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:16 smithi142 ceph-mon[103614]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:56:17.127 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:17.127 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:56:17.127 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2024-08-17T17:56:17.127 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:17.834 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:56:18.087 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:18.691 INFO:teuthology.orchestra.run.smithi142.stdout:mon.a: {} 2024-08-17T17:56:18.691 INFO:teuthology.orchestra.run.smithi142.stdout:mon.b: {} 2024-08-17T17:56:18.692 INFO:teuthology.orchestra.run.smithi142.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:56:18.692 INFO:teuthology.orchestra.run.smithi142.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:56:18.692 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:18 smithi142 ceph-mon[106729]: pgmap v2028: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:18.692 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:18 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/598738145' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:56:18.692 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:18 smithi142 ceph-mon[103614]: pgmap v2028: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:18.693 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:18 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/598738145' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:56:18.697 INFO:teuthology.orchestra.run.smithi142.stdout:mon.c: {} 2024-08-17T17:56:18.697 INFO:teuthology.orchestra.run.smithi142.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:56:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:18 smithi152 ceph-mon[93456]: pgmap v2028: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:18.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:18 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/598738145' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:56:19.464 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:56:19.688 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2024-08-17T17:56:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2024-08-17T17:56:19.688 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2024-08-17T17:56:19.689 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:19.720 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:20.289 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:56:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:56:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:20.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:56:20.574 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:56:20.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:20 smithi142 ceph-mon[106729]: pgmap v2029: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:20.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:20 smithi142 ceph-mon[103614]: pgmap v2029: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:20.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:20 smithi152 ceph-mon[93456]: pgmap v2029: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:56:21.886 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:56:22.056 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:21 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3921762780' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:56:22.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:21 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3921762780' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:56:22.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:21 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3921762780' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:56:22.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:56:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:56:21] "GET /metrics HTTP/1.1" 200 34169 "" "Prometheus/2.51.0" 2024-08-17T17:56:22.442 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-08-17T17:56:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2024-08-17T17:56:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2024-08-17T17:56:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(2.3 513/512 [PushOp(2:c4c92e5a:::rbd_trash:head, version: 63'1, data_included: [], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:c4c92e5a:::rbd_trash:head@63'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.706196+0000", 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7234522309999996, 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096058899999999997, 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706196+0000", 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706196+0000", 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706195+0000", 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706213+0000", 2024-08-17T17:56:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8088e-05 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706225+0000", 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.17e-05 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706235+0000", 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8859999999999999e-06 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.706286+0000", 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0943999999999999e-05 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.707001+0000", 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00071548399999999996 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.707156+0000", 2024-08-17T17:56:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015491699999999999 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 513/512 [PushOp(3:2f55791f:::606.00000000:head, version: 71'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:2f55791f:::606.00000000:head@71'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.708764+0000", 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7208837169999995, 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032015100000000002, 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708764+0000", 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708764+0000", 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708763+0000", 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708779+0000", 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5539000000000001e-05 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708795+0000", 2024-08-17T17:56:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6175999999999999e-05 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708806+0000", 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0560000000000001e-05 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708928+0000", 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121947 2024-08-17T17:56:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.708935+0000", 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4449999999999999e-06 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709084+0000", 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000149173 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.7 513/512 [PushOp(3:e0b41d06:::609.00000000:head, version: 71'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:e0b41d06:::609.00000000:head@71'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.709052+0000", 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7205954190000003, 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029272899999999998, 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709052+0000", 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709052+0000", 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709052+0000", 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709070+0000", 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7861000000000001e-05 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709085+0000", 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5381000000000001e-05 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709095+0000", 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7350000000000008e-06 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709131+0000", 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5998000000000003e-05 2024-08-17T17:56:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709135+0000", 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0040000000000001e-06 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709345+0000", 2024-08-17T17:56:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000210437 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 513/512 [PushOp(3:287034c4:::1.00000000.inode:head, version: 71'4, data_included: [0~570], data_size: 570, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:287034c4:::1.00000000.inode:head@71'4, size: 570, copy_subset: [0~570], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:570, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.709785+0000", 2024-08-17T17:56:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7198624339999995, 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027743700000000001, 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709785+0000", 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709785+0000", 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709785+0000", 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709796+0000", 2024-08-17T17:56:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1194e-05 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709812+0000", 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6231000000000001e-05 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709822+0000", 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8369999999999992e-06 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709892+0000", 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9559000000000003e-05 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.709898+0000", 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9290000000000003e-06 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.710063+0000", 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016532100000000001 2024-08-17T17:56:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 513/512 [PushOp(3:292cf221:::200.00000000:head, version: 85'21, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:292cf221:::200.00000000:head@85'21, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.710970+0000", 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7186774949999997, 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025420599999999998, 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.710970+0000", 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.710970+0000", 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.710970+0000", 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.710982+0000", 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2115e-05 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.710999+0000", 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7615999999999999e-05 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711009+0000", 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1940000000000004e-06 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711067+0000", 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8638999999999999e-05 2024-08-17T17:56:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711075+0000", 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3540000000000002e-06 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711225+0000", 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014997799999999999 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 513/512 [PushOp(3:2e2fa760:::200.00000001:head, version: 85'22, data_included: [0~4084], data_size: 4084, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:2e2fa760:::200.00000001:head@85'22, size: 4084, copy_subset: [0~4084], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4084, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.711845+0000", 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7178024010000001, 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00023302699999999999, 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711845+0000", 2024-08-17T17:56:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711845+0000", 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711845+0000", 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711871+0000", 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5738e-05 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711890+0000", 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8896e-05 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711900+0000", 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0773e-05 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711956+0000", 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5806000000000001e-05 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.711961+0000", 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.361e-06 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.712079+0000", 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117073 2024-08-17T17:56:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.d 513/512 [PushOp(3:b50e409b:::mds_snaptable:head, version: 71'1, data_included: [0~70], data_size: 70, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:b50e409b:::mds_snaptable:head@71'1, size: 70, copy_subset: [0~70], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:70, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.713560+0000", 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7160879819999995, 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041207700000000001, 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713560+0000", 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713560+0000", 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713559+0000", 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713568+0000", 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6659999999999997e-06 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713580+0000", 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2374e-05 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713586+0000", 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0990000000000004e-06 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713643+0000", 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6981999999999998e-05 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713651+0000", 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9349999999999994e-06 2024-08-17T17:56:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.713972+0000", 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000320594 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.f 513/512 [PushOp(3:f1f69726:::400.00000000:head, version: 71'1, data_included: [0~22], data_size: 22, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:f1f69726:::400.00000000:head@71'1, size: 22, copy_subset: [0~22], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:22, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.715485+0000", 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7141630830000008, 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000274362, 2024-08-17T17:56:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715485+0000", 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715485+0000", 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715484+0000", 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715494+0000", 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2250000000000006e-06 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715504+0000", 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0659000000000001e-05 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715510+0000", 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9950000000000002e-06 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715545+0000", 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4612e-05 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715550+0000", 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7639999999999996e-06 2024-08-17T17:56:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.715759+0000", 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000209438 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.13 513/512 [PushOp(3:cd5a64a3:::100.00000000:head, version: 71'1, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 10, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:cd5a64a3:::100.00000000:head@71'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.717118+0000", 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7125294189999991, 2024-08-17T17:56:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026915899999999999, 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717118+0000", 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717118+0000", 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717118+0000", 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717132+0000", 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3546e-05 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717143+0000", 2024-08-17T17:56:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1922e-05 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717149+0000", 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9669999999999998e-06 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717191+0000", 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1291999999999997e-05 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717195+0000", 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0409999999999999e-06 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.717388+0000", 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000192867 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.13 513/512 [PushOp(3:c91d4a1d:::mds0_inotable:head, version: 71'2, data_included: [0~34], data_size: 34, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:c91d4a1d:::mds0_inotable:head@71'2, size: 34, copy_subset: [0~34], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:34, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.718047+0000", 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7116008249999997, 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027490799999999998, 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718047+0000", 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718047+0000", 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718047+0000", 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718055+0000", 2024-08-17T17:56:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3659999999999992e-06 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718067+0000", 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2531000000000001e-05 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718073+0000", 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9349999999999999e-06 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718117+0000", 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3284000000000001e-05 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718120+0000", 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8179999999999997e-06 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.718322+0000", 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020147900000000001 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.e 513/512 [PushOp(3:717a0223:::608.00000000:head, version: 71'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:717a0223:::608.00000000:head@71'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.719747+0000", 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7099005110000007, 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000373113, 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719747+0000", 2024-08-17T17:56:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719747+0000", 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719747+0000", 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719756+0000", 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0180000000000007e-06 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719768+0000", 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2607e-05 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719776+0000", 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719829+0000", 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3242999999999998e-05 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.719837+0000", 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5000000000000002e-06 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.720120+0000", 2024-08-17T17:56:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00028381499999999998 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.15 513/512 [PushOp(3:a93a17c2:::604.00000000:head, version: 71'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:a93a17c2:::604.00000000:head@71'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.721518+0000", 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7081300590000001, 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025408800000000002, 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721518+0000", 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721518+0000", 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721517+0000", 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721525+0000", 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8320000000000006e-06 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721537+0000", 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1938e-05 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721543+0000", 2024-08-17T17:56:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5419999999999996e-06 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721579+0000", 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6134000000000003e-05 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721584+0000", 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5639999999999996e-06 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.721772+0000", 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018746 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.15 513/512 [PushOp(3:aa448500:::500.00000000:head, version: 71'3, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:aa448500:::500.00000000:head@71'3, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:13.722281+0000", 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 8.7073669880000004, 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027287999999999998, 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722281+0000", 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722281+0000", 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722280+0000", 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722289+0000", 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1729999999999995e-06 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722300+0000", 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1596999999999999e-05 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722308+0000", 2024-08-17T17:56:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6629999999999993e-06 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722342+0000", 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3776999999999999e-05 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722346+0000", 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6800000000000001e-06 2024-08-17T17:56:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:13.722554+0000", 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020743199999999999 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(1.0 514/513 [PushOp(1:1b0e96ae:devicehealth::main.db-journal.0000000000000000:head, version: 22'116, data_included: [0~262688], data_size: 262688, omap_header_size: 0, omap_entries_size: 0, attrset_size: 9, recovery_info: ObjectRecoveryInfo(1:1b0e96ae:devicehealth::main.db-journal.0000000000000000:head@22'116, size: 262688, copy_subset: [0~262688], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:262688, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:14.706523+0000", 2024-08-17T17:56:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.7231245069999996, 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00075629099999999997, 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706523+0000", 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706523+0000", 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706523+0000", 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706825+0000", 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00030227100000000001 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706844+0000", 2024-08-17T17:56:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8836000000000001e-05 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706874+0000", 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0620000000000002e-05 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706912+0000", 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8062999999999998e-05 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.706919+0000", 2024-08-17T17:56:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1290000000000001e-06 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.707280+0000", 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036104600000000001 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(1.0 514/513 [PushOp(1:30306672:devicehealth::main.db.0000000000000000:head, version: 85'149, data_included: [0~327680], data_size: 327680, omap_header_size: 0, omap_entries_size: 0, attrset_size: 10, recovery_info: ObjectRecoveryInfo(1:30306672:devicehealth::main.db.0000000000000000:head@85'149, size: 327680, copy_subset: [0~327680], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:327680, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:14.708433+0000", 2024-08-17T17:56:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.7212145400000001, 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00059160699999999996, 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708433+0000", 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708433+0000", 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708433+0000", 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708747+0000", 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000314235 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708759+0000", 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1768e-05 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708765+0000", 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0850000000000002e-06 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708798+0000", 2024-08-17T17:56:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3127000000000003e-05 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.708802+0000", 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6320000000000001e-06 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:14.709025+0000", 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000223186 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPushReply(3.13 516/515 [PushReplyOp(3:cd5a64a3:::100.00000000:head)])", 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:16.714311+0000", 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 5.7153365879999996, 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032711200000000001, 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714311+0000", 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714311+0000", 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714311+0000", 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714318+0000", 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4699999999999996e-06 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714332+0000", 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3443e-05 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714340+0000", 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5860000000000005e-06 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714399+0000", 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9219999999999999e-05 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.714638+0000", 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00023907600000000001 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPushReply(3.13 516/515 [PushReplyOp(3:c91d4a1d:::mds0_inotable:head)])", 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2024-08-17T17:56:16.716515+0000", 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 5.7131329900000001, 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000159054, 2024-08-17T17:56:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716515+0000", 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716515+0000", 2024-08-17T17:56:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716515+0000", 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716518+0000", 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.473e-06 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716526+0000", 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8730000000000007e-06 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716532+0000", 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7339999999999996e-06 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716560+0000", 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7928000000000001e-05 2024-08-17T17:56:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2024-08-17T17:56:16.716674+0000", 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114385 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-08-17T17:56:22.515 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-08-17T17:56:22.546 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 1 2024-08-17T17:56:22.814 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:23.087 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3921762780' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:56:23.087 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:22 smithi142 ceph-mon[106729]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:56:23.087 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:22 smithi142 ceph-mon[106729]: pgmap v2031: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 774 B/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:56:23.087 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3921762780' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:56:23.088 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:22 smithi142 ceph-mon[103614]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:56:23.088 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:22 smithi142 ceph-mon[103614]: pgmap v2031: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 774 B/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:56:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3921762780' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:56:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:22 smithi152 ceph-mon[93456]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:56:23.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:22 smithi152 ceph-mon[93456]: pgmap v2031: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 774 B/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:56:24.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:24 smithi142 ceph-mon[103614]: pgmap v2032: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 668 B/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:56:24.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:24 smithi142 ceph-mon[106729]: pgmap v2032: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 668 B/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:56:25.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:24 smithi152 ceph-mon[93456]: pgmap v2032: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 668 B/s, 1 keys/s, 1 objects/s recovering 2024-08-17T17:56:25.684 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-08-17T17:56:26.069 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:25 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3913565708' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:56:26.070 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:25 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3913565708' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:56:26.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:25 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3913565708' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:56:26.507 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:56:26.775 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3913565708' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:56:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[106729]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:56:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:26.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[106729]: pgmap v2034: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 1 keys/s, 0 objects/s recovering 2024-08-17T17:56:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3913565708' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:56:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[103614]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:56:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:26.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:26 smithi142 ceph-mon[103614]: pgmap v2034: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 1 keys/s, 0 objects/s recovering 2024-08-17T17:56:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:26 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3913565708' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:56:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:26 smithi152 ceph-mon[93456]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:56:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:27.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:26 smithi152 ceph-mon[93456]: pgmap v2034: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 1 keys/s, 0 objects/s recovering 2024-08-17T17:56:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:27 smithi142 ceph-mon[106729]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:56:27.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:27 smithi142 ceph-mon[103614]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:56:27.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:27 smithi152 ceph-mon[93456]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:56:28.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:28.699 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:56:29.059 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:28 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/934285429' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:29.059 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:28 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:29.059 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:28 smithi142 ceph-mon[106729]: pgmap v2036: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:29.059 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:28 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/934285429' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:29.060 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:28 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:29.060 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:28 smithi142 ceph-mon[103614]: pgmap v2036: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:29.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:28 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/934285429' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:29.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:28 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:28 smithi152 ceph-mon[93456]: pgmap v2036: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:29.384 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 1 1 2024-08-17T17:56:29.638 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:30.021 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:29 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:56:30.022 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:29 smithi142 ceph-mon[106729]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:56:30.022 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:29 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:56:30.022 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:29 smithi142 ceph-mon[103614]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:56:30.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:29 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:56:30.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:29 smithi152 ceph-mon[93456]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:56:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:56:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:56:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:30.848 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:30 smithi152 ceph-mon[93456]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:56:30.848 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:30 smithi152 ceph-mon[93456]: pgmap v2039: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:30.859 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:30 smithi142 ceph-mon[106729]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:56:30.859 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:30 smithi142 ceph-mon[106729]: pgmap v2039: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:30.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:30 smithi142 ceph-mon[103614]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:56:30.860 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:30 smithi142 ceph-mon[103614]: pgmap v2039: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:31.715 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-08-17T17:56:32.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:31 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3528152362' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:32.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:31 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:32.076 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:31 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3528152362' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:32.076 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:31 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:32.076 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:56:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:56:31] "GET /metrics HTTP/1.1" 200 34184 "" "Prometheus/2.51.0" 2024-08-17T17:56:32.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:31 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3528152362' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:32.076 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:31 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:56:32.408 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:56:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:32 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:56:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:32 smithi152 ceph-mon[93456]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:56:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:32 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:56:32.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:32 smithi152 ceph-mon[93456]: pgmap v2041: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:32.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:56:32.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[106729]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:56:32.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:56:32.995 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[106729]: pgmap v2041: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:32.996 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:56:32.996 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[103614]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:56:32.996 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:56:32.996 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:32 smithi142 ceph-mon[103614]: pgmap v2041: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:33.433 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:33.852 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:33 smithi142 ceph-mon[106729]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:56:33.852 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:33 smithi142 ceph-mon[103614]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:56:34.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:33 smithi152 ceph-mon[93456]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:56:34.994 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:34 smithi142 ceph-mon[106729]: pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:34.994 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:34 smithi142 ceph-mon[103614]: pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:35.005 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:34 smithi152 ceph-mon[93456]: pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:35.730 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:56:35.743 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:35 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2028515588' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:56:35.743 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:35 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2028515588' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:56:36.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:35 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2028515588' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:56:36.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 1 2024-08-17T17:56:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2028515588' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:56:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[106729]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:56:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:36.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[106729]: pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2028515588' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:56:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[103614]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:56:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:36.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:36 smithi142 ceph-mon[103614]: pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:36 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2028515588' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:56:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:36 smithi152 ceph-mon[93456]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:56:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:36 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:37.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:36 smithi152 ceph-mon[93456]: pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:37.352 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:38.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:38 smithi142 ceph-mon[106729]: pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:38.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:38 smithi142 ceph-mon[103614]: pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:39.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:38 smithi152 ceph-mon[93456]: pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:40.554 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:56:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:56:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:56:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:40 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1802974066' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:56:40.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1802974066' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:40.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:40 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1802974066' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:56:40.996 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-08-17T17:56:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[106729]: pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1802974066' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:56:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[106729]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:56:41.772 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:41.773 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[103614]: pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:41.773 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1802974066' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:56:41.773 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[103614]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:56:41.773 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:41 smithi152 ceph-mon[93456]: pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:41 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1802974066' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:56:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:41 smithi152 ceph-mon[93456]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:56:41.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:41 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:41.837 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:56:42.098 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:42.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:56:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:56:41] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-17T17:56:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:43 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:43.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:43 smithi142 ceph-mon[106729]: pgmap v2049: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:43.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:43 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:43.373 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:43 smithi142 ceph-mon[103614]: pgmap v2049: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:43.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:43 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:56:43.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:43 smithi152 ceph-mon[93456]: pgmap v2049: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:44.249 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:56:44.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:44 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/758547029' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:44.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:44 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:44.598 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:44 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/758547029' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:44.599 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:44 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:44.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:44 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/758547029' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:44.599 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:44 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:44.926 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 3 1 2024-08-17T17:56:45.182 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:45.569 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:45 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:56:45.569 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:45 smithi142 ceph-mon[103614]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:56:45.569 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:45 smithi142 ceph-mon[103614]: pgmap v2051: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:45.570 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:45 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:56:45.570 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:45 smithi142 ceph-mon[106729]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:56:45.570 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:45 smithi142 ceph-mon[106729]: pgmap v2051: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:45 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:56:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:45 smithi152 ceph-mon[93456]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:56:45.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:45 smithi152 ceph-mon[93456]: pgmap v2051: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:46.738 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:46 smithi142 ceph-mon[106729]: pgmap v2052: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:46.738 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:46 smithi142 ceph-mon[103614]: pgmap v2052: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:47.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:46 smithi152 ceph-mon[93456]: pgmap v2052: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:47.696 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-08-17T17:56:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:47 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2757240314' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:48.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:47 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:48.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:47 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2757240314' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:48.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:47 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:48.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:47 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2757240314' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:48.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:47 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:56:48.611 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:56:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:48 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:56:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:48 smithi142 ceph-mon[103614]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:56:48.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:48 smithi142 ceph-mon[103614]: pgmap v2054: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:48 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:56:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:48 smithi142 ceph-mon[106729]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:56:48.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:48 smithi142 ceph-mon[106729]: pgmap v2054: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:48.877 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:48 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:56:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:48 smithi152 ceph-mon[93456]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:56:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:48 smithi152 ceph-mon[93456]: pgmap v2054: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:49 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-08-17T17:56:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:49 smithi152 ceph-mon[93456]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:56:50.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:49 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-08-17T17:56:50.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:49 smithi142 ceph-mon[103614]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:56:50.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:49 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-08-17T17:56:50.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:49 smithi142 ceph-mon[106729]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:56:50.572 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:56:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:56:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:50 smithi152 ceph-mon[93456]: pgmap v2056: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:51.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:50 smithi142 ceph-mon[103614]: pgmap v2056: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:51.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:50 smithi142 ceph-mon[106729]: pgmap v2056: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:51.716 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:56:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:51 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2899590657' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:52.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:51 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:52.100 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:51 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2899590657' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:52.100 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:51 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:52.100 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:56:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:56:51] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-17T17:56:52.100 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:51 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2899590657' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:52.100 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:51 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:52.516 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 4 1 2024-08-17T17:56:52.778 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:52.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:52 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:56:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:52 smithi142 ceph-mon[106729]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:56:52.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:52 smithi142 ceph-mon[106729]: pgmap v2058: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:52 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:56:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:52 smithi142 ceph-mon[103614]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:56:52.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:52 smithi142 ceph-mon[103614]: pgmap v2058: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:52 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:56:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:52 smithi152 ceph-mon[93456]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:56:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:52 smithi152 ceph-mon[93456]: pgmap v2058: 73 pgs: 8 peering, 65 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:54 smithi152 ceph-mon[93456]: pgmap v2059: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:54 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 8 pgs peering) 2024-08-17T17:56:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:54 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:56:55.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:54 smithi142 ceph-mon[106729]: pgmap v2059: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:55.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:54 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 8 pgs peering) 2024-08-17T17:56:55.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:54 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:56:55.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:54 smithi142 ceph-mon[103614]: pgmap v2059: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:54 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 8 pgs peering) 2024-08-17T17:56:55.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:54 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:56:55.687 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-08-17T17:56:56.017 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2665032678' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:56.017 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:55 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:56.017 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2665032678' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:56.017 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:55 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:56.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2665032678' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:56.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:55 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:56:56.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:56:56.758 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:56:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[106729]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:56:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[106729]: pgmap v2061: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:56:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[103614]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:56:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:56 smithi142 ceph-mon[103614]: pgmap v2061: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:56 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:56:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:56 smithi152 ceph-mon[93456]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:56:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:56 smithi152 ceph-mon[93456]: pgmap v2061: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:58.697 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:56:59.016 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:58 smithi142 ceph-mon[103614]: pgmap v2062: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:59.016 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/117109645' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:56:59.017 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:58 smithi142 ceph-mon[106729]: pgmap v2062: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:59.017 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/117109645' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:56:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:58 smithi152 ceph-mon[93456]: pgmap v2062: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:56:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/117109645' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:56:59.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-17T17:56:59.757 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:59 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/117109645' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:56:59.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:56:59 smithi142 ceph-mon[103614]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:56:59.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:59 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/117109645' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:56:59.874 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:56:59 smithi142 ceph-mon[106729]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:57:00.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:59 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/117109645' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:57:00.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:56:59 smithi152 ceph-mon[93456]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:57:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:57:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:57:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:00.842 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:00 smithi152 ceph-mon[93456]: pgmap v2064: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:01.000 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:00 smithi142 ceph-mon[106729]: pgmap v2064: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:01.000 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:00 smithi142 ceph-mon[103614]: pgmap v2064: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:01.772 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-17T17:57:02.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:01 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1412989246' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:57:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:01 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:57:02.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:57:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:57:01] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.51.0" 2024-08-17T17:57:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:01 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1412989246' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:57:02.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:01 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:57:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:01 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1412989246' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:57:02.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:01 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:57:02.567 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:57:02.830 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:57:02.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[106729]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:57:02.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:02.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[106729]: pgmap v2066: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:02.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:57:02.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[103614]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:57:02.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:02.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:02 smithi142 ceph-mon[103614]: pgmap v2066: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:02 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:57:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:02 smithi152 ceph-mon[93456]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:57:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:02 smithi152 ceph-mon[93456]: pgmap v2066: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:04 smithi152 ceph-mon[93456]: pgmap v2067: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:04 smithi142 ceph-mon[103614]: pgmap v2067: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:04 smithi142 ceph-mon[106729]: pgmap v2067: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:05.690 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:57:06.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:05 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2081918074' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:06.025 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:05 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:06.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:05 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2081918074' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:06.026 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:05 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:06.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:05 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2081918074' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:06.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:05 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:06.438 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-17T17:57:06.692 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:06 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:57:06.699 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:06.985 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:06 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:57:06.985 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:06 smithi142 ceph-mon[106729]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:57:06.985 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:06 smithi142 ceph-mon[106729]: pgmap v2069: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:06.985 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:06 smithi142 ceph-mon[103614]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:57:06.985 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:06 smithi142 ceph-mon[103614]: pgmap v2069: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:06 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:57:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:06 smithi152 ceph-mon[93456]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:57:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:06 smithi152 ceph-mon[93456]: pgmap v2069: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:08.712 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-17T17:57:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:08 smithi142 ceph-mon[106729]: pgmap v2070: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:08 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3039594468' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:08.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:08 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:08 smithi142 ceph-mon[103614]: pgmap v2070: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:08 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3039594468' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:08.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:08 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:08 smithi152 ceph-mon[93456]: pgmap v2070: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:08 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3039594468' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:09.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:08 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:57:09.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:57:09.748 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:09.871 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:09 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:57:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:09 smithi142 ceph-mon[106729]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:57:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:09 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:57:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:09 smithi142 ceph-mon[103614]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:57:10.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:09 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:57:10.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:09 smithi152 ceph-mon[93456]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:57:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:57:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:57:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:11.064 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:10 smithi142 ceph-mon[106729]: pgmap v2072: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:11.065 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:10 smithi142 ceph-mon[103614]: pgmap v2072: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:11.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:10 smithi152 ceph-mon[93456]: pgmap v2072: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:12.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:11 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:12.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:11 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:12.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:11 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:12.123 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:57:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:57:11] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-17T17:57:12.719 INFO:teuthology.orchestra.run.smithi142.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:57:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:12 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/932850538' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:57:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:12 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]} : dispatch 2024-08-17T17:57:13.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:12 smithi152 ceph-mon[93456]: pgmap v2073: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/932850538' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:13.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:57:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]} : dispatch 2024-08-17T17:57:13.123 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[106729]: pgmap v2073: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/932850538' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:57:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]} : dispatch 2024-08-17T17:57:13.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:12 smithi142 ceph-mon[103614]: pgmap v2073: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:13.636 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 7 1 2024-08-17T17:57:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/932850538' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:57:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-08-17T17:57:13.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-08-17T17:57:13.873 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[106729]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:57:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/932850538' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:57:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-08-17T17:57:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-08-17T17:57:13.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:13 smithi142 ceph-mon[103614]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:57:13.898 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:13 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/932850538' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:57:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-08-17T17:57:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:13 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-08-17T17:57:14.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:13 smithi152 ceph-mon[93456]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:57:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:14 smithi152 ceph-mon[93456]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:57:15.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:14 smithi152 ceph-mon[93456]: pgmap v2076: 73 pgs: 1 activating, 1 peering, 71 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:14 smithi142 ceph-mon[106729]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:57:15.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:14 smithi142 ceph-mon[106729]: pgmap v2076: 73 pgs: 1 activating, 1 peering, 71 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:14 smithi142 ceph-mon[103614]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:57:15.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:14 smithi142 ceph-mon[103614]: pgmap v2076: 73 pgs: 1 activating, 1 peering, 71 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:16.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:15 smithi152 ceph-mon[93456]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:57:16.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:15 smithi142 ceph-mon[106729]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:57:16.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:15 smithi142 ceph-mon[103614]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:57:16.744 INFO:teuthology.orchestra.run.smithi142.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-08-17T17:57:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:16 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/587046139' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:16 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:17.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:16 smithi152 ceph-mon[93456]: pgmap v2077: 73 pgs: 1 activating, 1 peering, 71 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:16 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/587046139' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:16 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:17.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:16 smithi142 ceph-mon[106729]: pgmap v2077: 73 pgs: 1 activating, 1 peering, 71 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:16 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/587046139' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:17.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:16 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:57:17.123 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:16 smithi142 ceph-mon[103614]: pgmap v2077: 73 pgs: 1 activating, 1 peering, 71 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:17.528 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-08-17T17:57:17.528 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-08-17T17:57:17.528 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-08-17T17:57:17.529 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-08-17T17:57:17.529 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:57:17.529 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:57:17.790 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:17 smithi142 ceph-mon[106729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:57:17.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:17 smithi142 ceph-mon[106729]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:57:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:17 smithi142 ceph-mon[103614]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:57:17.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:17 smithi142 ceph-mon[103614]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:57:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:17 smithi152 ceph-mon[93456]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:57:18.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:17 smithi152 ceph-mon[93456]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:57:18.971 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:18 smithi142 ceph-mon[106729]: pgmap v2079: 73 pgs: 1 activating, 72 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:18.971 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:18 smithi142 ceph-mon[103614]: pgmap v2079: 73 pgs: 1 activating, 72 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:19.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:18 smithi152 ceph-mon[93456]: pgmap v2079: 73 pgs: 1 activating, 72 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:19.777 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:57:19.777 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:19 smithi142 ceph-mon[106729]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:57:19.778 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:19 smithi142 ceph-mon[106729]: Cluster is now healthy 2024-08-17T17:57:19.778 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":537,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T17:57:16.729767+0000","last_up_change":"2024-08-17T17:56:02.679700+0000","last_in_change":"2024-08-17T17:56:15.697262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"489","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:57:27.258162+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:57:27.611866+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":518,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6803","nonce":608035512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6805","nonce":608035512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6809","nonce":608035512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6807","nonce":608035512}]},"public_addr":"172.21.15.142:6803/608035512","cluster_addr":"172.21.15.142:6805/608035512","heartbeat_back_addr":"172.21.15.142:6809/608035512","heartbeat_front_addr":"172.21.15.142:6807/608035512","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":100,"up_from":106,"up_thru":522,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2324159993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2324159993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2324159993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2324159993}]},"public_addr":"172.21.15.142:6811/2324159993","cluster_addr":"172.21.15.142:6813/2324159993","heartbeat_back_addr":"172.21.15.142:6817/2324159993","heartbeat_front_addr":"172.21.15.142:6815/2324159993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":535,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6819","nonce":366314937}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6821","nonce":366314937}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6825","nonce":366314937}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6823","nonce":366314937}]},"public_addr":"172.21.15.142:6819/366314937","cluster_addr":"172.21.15.142:6821/366314937","heartbeat_back_addr":"172.21.15.142:6825/366314937","heartbeat_front_addr":"172.21.15.142:6823/366314937","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":142,"up_from":146,"up_thru":527,"down_at":143,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6827","nonce":195335216}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6829","nonce":195335216}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6833","nonce":195335216}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6831","nonce":195335216}]},"public_addr":"172.21.15.142:6827/195335216","cluster_addr":"172.21.15.142:6829/195335216","heartbeat_back_addr":"172.21.15.142:6833/195335216","heartbeat_front_addr":"172.21.15.142:6831/195335216","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":520,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":190,"up_from":199,"up_thru":515,"down_at":191,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6809","nonce":395680087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6811","nonce":395680087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6815","nonce":395680087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6813","nonce":395680087}]},"public_addr":"172.21.15.152:6809/395680087","cluster_addr":"172.21.15.152:6811/395680087","heartbeat_back_addr":"172.21.15.152:6815/395680087","heartbeat_front_addr":"172.21.15.152:6813/395680087","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":443,"up_from":453,"up_thru":535,"down_at":444,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6817","nonce":779491873}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6819","nonce":779491873}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6823","nonce":779491873}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6821","nonce":779491873}]},"public_addr":"172.21.15.152:6817/779491873","cluster_addr":"172.21.15.152:6819/779491873","heartbeat_back_addr":"172.21.15.152:6823/779491873","heartbeat_front_addr":"172.21.15.152:6821/779491873","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":270,"last_clean_end":435,"up_from":511,"up_thru":515,"down_at":436,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6825","nonce":4168801310}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6827","nonce":4168801310}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6831","nonce":4168801310}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6829","nonce":4168801310}]},"public_addr":"172.21.15.152:6825/4168801310","cluster_addr":"172.21.15.152:6827/4168801310","heartbeat_back_addr":"172.21.15.152:6831/4168801310","heartbeat_front_addr":"172.21.15.152:6829/4168801310","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":100},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":142},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":190},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":443},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":435}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.142:6801/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1253216666":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/3034915416":"2024-08-18T17:03:11.046184+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.152:0/681595170":"2024-08-18T17:02:18.823112+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6800/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/267249921":"2024-08-18T17:03:11.046184+0000","172.21.15.152:6834/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3319043742":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1554300251":"2024-08-18T17:00:33.779952+0000","172.21.15.152:0/3235892463":"2024-08-18T17:02:18.823112+0000","172.21.15.152:0/3133806592":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2472242802":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/639679378":"2024-08-18T17:00:33.779952+0000","172.21.15.152:6835/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/1828093647":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/2627951813":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/3716608685":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.152:0/3126412235":"2024-08-18T17:02:18.823112+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:57:19.779 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:19 smithi142 ceph-mon[103614]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:57:19.779 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:19 smithi142 ceph-mon[103614]: Cluster is now healthy 2024-08-17T17:57:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:19 smithi152 ceph-mon[93456]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:57:20.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:19 smithi152 ceph-mon[93456]: Cluster is now healthy 2024-08-17T17:57:20.482 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:57:20.482 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:57:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:57:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:57:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:20.741 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:21.018 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:20 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/405555027' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:57:21.018 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:20 smithi142 ceph-mon[106729]: pgmap v2080: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:21.018 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:20 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/405555027' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:57:21.018 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:20 smithi142 ceph-mon[103614]: pgmap v2080: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:20 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/405555027' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:57:21.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:20 smithi152 ceph-mon[93456]: pgmap v2080: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:21.945 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:57:21 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:57:21] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-17T17:57:22.665 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:57:22.665 INFO:teuthology.orchestra.run.smithi142.stdout:{"epoch":537,"fsid":"bcf463ee-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:50:52.038786+0000","modified":"2024-08-17T17:57:16.729767+0000","last_up_change":"2024-08-17T17:56:02.679700+0000","last_in_change":"2024-08-17T17:56:15.697262+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:53:47.946475+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"489","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:56:11.984400+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:57:27.258162+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:57:27.611866+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4a1ac03b-9520-4665-ba56-380cd889d3f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":518,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6802","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6803","nonce":608035512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6804","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6805","nonce":608035512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6808","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6809","nonce":608035512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6806","nonce":608035512},{"type":"v1","addr":"172.21.15.142:6807","nonce":608035512}]},"public_addr":"172.21.15.142:6803/608035512","cluster_addr":"172.21.15.142:6805/608035512","heartbeat_back_addr":"172.21.15.142:6809/608035512","heartbeat_front_addr":"172.21.15.142:6807/608035512","state":["exists","up"]},{"osd":1,"uuid":"fa145811-9dfa-468b-a005-75eb2042368d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":100,"up_from":106,"up_thru":522,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6810","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6811","nonce":2324159993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6812","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6813","nonce":2324159993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6816","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6817","nonce":2324159993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6814","nonce":2324159993},{"type":"v1","addr":"172.21.15.142:6815","nonce":2324159993}]},"public_addr":"172.21.15.142:6811/2324159993","cluster_addr":"172.21.15.142:6813/2324159993","heartbeat_back_addr":"172.21.15.142:6817/2324159993","heartbeat_front_addr":"172.21.15.142:6815/2324159993","state":["exists","up"]},{"osd":2,"uuid":"4d5884fb-4ca9-45c8-93f4-f0cbaeddbb63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":535,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6818","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6819","nonce":366314937}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6820","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6821","nonce":366314937}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6824","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6825","nonce":366314937}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6822","nonce":366314937},{"type":"v1","addr":"172.21.15.142:6823","nonce":366314937}]},"public_addr":"172.21.15.142:6819/366314937","cluster_addr":"172.21.15.142:6821/366314937","heartbeat_back_addr":"172.21.15.142:6825/366314937","heartbeat_front_addr":"172.21.15.142:6823/366314937","state":["exists","up"]},{"osd":3,"uuid":"f9b26ffd-2955-4af8-90c8-5473d3c78fa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":142,"up_from":146,"up_thru":527,"down_at":143,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6826","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6827","nonce":195335216}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6828","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6829","nonce":195335216}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6832","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6833","nonce":195335216}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.142:6830","nonce":195335216},{"type":"v1","addr":"172.21.15.142:6831","nonce":195335216}]},"public_addr":"172.21.15.142:6827/195335216","cluster_addr":"172.21.15.142:6829/195335216","heartbeat_back_addr":"172.21.15.142:6833/195335216","heartbeat_front_addr":"172.21.15.142:6831/195335216","state":["exists","up"]},{"osd":4,"uuid":"76eecfff-f5c2-4180-a0c5-26b83e8359e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":520,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6801","nonce":3554198822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6803","nonce":3554198822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6807","nonce":3554198822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3554198822},{"type":"v1","addr":"172.21.15.152:6805","nonce":3554198822}]},"public_addr":"172.21.15.152:6801/3554198822","cluster_addr":"172.21.15.152:6803/3554198822","heartbeat_back_addr":"172.21.15.152:6807/3554198822","heartbeat_front_addr":"172.21.15.152:6805/3554198822","state":["exists","up"]},{"osd":5,"uuid":"f36620dd-2b34-4322-92c9-59ff43a6799e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":190,"up_from":199,"up_thru":515,"down_at":191,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6809","nonce":395680087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6811","nonce":395680087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6815","nonce":395680087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":395680087},{"type":"v1","addr":"172.21.15.152:6813","nonce":395680087}]},"public_addr":"172.21.15.152:6809/395680087","cluster_addr":"172.21.15.152:6811/395680087","heartbeat_back_addr":"172.21.15.152:6815/395680087","heartbeat_front_addr":"172.21.15.152:6813/395680087","state":["exists","up"]},{"osd":6,"uuid":"4dc7a5d8-e2a2-4687-a4d8-53799e862d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":443,"up_from":453,"up_thru":535,"down_at":444,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6817","nonce":779491873}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6819","nonce":779491873}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6823","nonce":779491873}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":779491873},{"type":"v1","addr":"172.21.15.152:6821","nonce":779491873}]},"public_addr":"172.21.15.152:6817/779491873","cluster_addr":"172.21.15.152:6819/779491873","heartbeat_back_addr":"172.21.15.152:6823/779491873","heartbeat_front_addr":"172.21.15.152:6821/779491873","state":["exists","up"]},{"osd":7,"uuid":"3a1c6335-e51f-45be-afe7-fd487ca70a0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":270,"last_clean_end":435,"up_from":511,"up_thru":515,"down_at":436,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6825","nonce":4168801310}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6827","nonce":4168801310}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6831","nonce":4168801310}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":4168801310},{"type":"v1","addr":"172.21.15.152:6829","nonce":4168801310}]},"public_addr":"172.21.15.152:6825/4168801310","cluster_addr":"172.21.15.152:6827/4168801310","heartbeat_back_addr":"172.21.15.152:6831/4168801310","heartbeat_front_addr":"172.21.15.152:6829/4168801310","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:52:56.493031+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:19.576680+0000","dead_epoch":100},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:53:44.199902+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:08.890414+0000","dead_epoch":142},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:30.224079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:54:53.193699+0000","dead_epoch":190},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:17.120552+0000","dead_epoch":443},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:55:40.288056+0000","dead_epoch":435}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/3235892463":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6801/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/292376296":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6800/3742822359":"2024-08-18T17:03:11.046184+0000","172.21.15.152:0/3126412235":"2024-08-18T17:02:18.823112+0000","172.21.15.142:0/2627951813":"2024-08-18T17:00:33.779952+0000","172.21.15.142:6801/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6800/3686065451":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3686891819":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/283129895":"2024-08-18T16:51:41.645069+0000","172.21.15.142:6801/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/913350931":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/267249921":"2024-08-18T17:03:11.046184+0000","172.21.15.152:6834/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.152:0/681595170":"2024-08-18T17:02:18.823112+0000","172.21.15.142:0/2588146503":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6800/3356817710":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/2543002123":"2024-08-18T16:51:20.134164+0000","172.21.15.142:0/3034915416":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/298990150":"2024-08-18T16:51:41.645069+0000","172.21.15.152:0/3133806592":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6800/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/2472242802":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/639679378":"2024-08-18T17:00:33.779952+0000","172.21.15.152:6835/3003700763":"2024-08-18T17:02:18.823112+0000","172.21.15.142:6801/508280165":"2024-08-18T16:51:41.645069+0000","172.21.15.142:0/1828093647":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/528641382":"2024-08-18T16:51:20.134164+0000","172.21.15.142:6800/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:6801/744771467":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/1253216666":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/1554300251":"2024-08-18T17:00:33.779952+0000","172.21.15.142:0/3319043742":"2024-08-18T17:03:11.046184+0000","172.21.15.142:0/54516382":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/3367745640":"2024-08-18T16:56:12.454665+0000","172.21.15.142:0/3716608685":"2024-08-18T17:03:11.046184+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:57:23.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:22 smithi142 ceph-mon[103614]: pgmap v2081: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:23.062 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:22 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/2338749989' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:57:23.062 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:22 smithi142 ceph-mon[106729]: pgmap v2081: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:23.063 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:22 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/2338749989' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:57:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:22 smithi152 ceph-mon[93456]: pgmap v2081: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:23.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:22 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/2338749989' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:57:23.491 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:57:23.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:57:23.492 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:57:23.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:57:23.493 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:57:23.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:57:23.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:57:23.494 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:57:24.060 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.061 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.061 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.062 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.063 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.065 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.069 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.102 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:24.803 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:24 smithi142 ceph-mon[106729]: pgmap v2082: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:24.803 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:24 smithi142 ceph-mon[103614]: pgmap v2082: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:25.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:24 smithi152 ceph-mon[93456]: pgmap v2082: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:26.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:26 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:26.865 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:26 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:26.865 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:26 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:27 smithi142 ceph-mon[106729]: pgmap v2083: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:27.622 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:27 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:27.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:27 smithi142 ceph-mon[103614]: pgmap v2083: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:27.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:27 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:27 smithi152 ceph-mon[93456]: pgmap v2083: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:27.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:27 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:28.939 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:28 smithi142 ceph-mon[106729]: pgmap v2084: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:28.939 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:28 smithi142 ceph-mon[103614]: pgmap v2084: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:29.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:28 smithi152 ceph-mon[93456]: pgmap v2084: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:30.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:57:30 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:57:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:30.748 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:30 smithi142 ceph-mon[106729]: pgmap v2085: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:30.748 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:30 smithi142 ceph-mon[103614]: pgmap v2085: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:31.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:30 smithi152 ceph-mon[93456]: pgmap v2085: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:31.263 INFO:teuthology.orchestra.run.smithi142.stdout:455266533974 2024-08-17T17:57:31.264 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:57:31.872 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:57:31 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:57:31] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.51.0" 2024-08-17T17:57:33.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:32 smithi152 ceph-mon[93456]: pgmap v2086: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:33.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:32 smithi142 ceph-mon[106729]: pgmap v2086: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:33.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:32 smithi142 ceph-mon[103614]: pgmap v2086: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:34.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:34 smithi142 ceph-mon[106729]: pgmap v2087: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:34.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:34 smithi142 ceph-mon[103614]: pgmap v2087: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:35.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:34 smithi152 ceph-mon[93456]: pgmap v2087: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:35.364 INFO:teuthology.orchestra.run.smithi142.stdout:854698492338 2024-08-17T17:57:35.380 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:57:36.814 INFO:teuthology.orchestra.run.smithi142.stdout:515396076083 2024-08-17T17:57:36.815 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:57:37.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:36 smithi152 ceph-mon[93456]: pgmap v2088: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:37.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:36 smithi142 ceph-mon[106729]: pgmap v2088: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:37.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:36 smithi142 ceph-mon[103614]: pgmap v2088: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:38.927 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:38 smithi142 ceph-mon[106729]: pgmap v2089: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:38.927 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:38 smithi142 ceph-mon[103614]: pgmap v2089: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:39.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:38 smithi152 ceph-mon[93456]: pgmap v2089: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:39.082 INFO:teuthology.orchestra.run.smithi142.stdout:627065225759 2024-08-17T17:57:39.082 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:57:40.253 INFO:teuthology.orchestra.run.smithi142.stdout:133143986930 2024-08-17T17:57:40.253 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:57:40.485 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:40 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:57:40.485 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:40 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:57:40.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:40 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:57:40.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:57:40 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:57:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:40.933 INFO:teuthology.orchestra.run.smithi142.stdout:2194728288277 2024-08-17T17:57:40.933 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:57:41.426 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:41 smithi142 ceph-mon[106729]: pgmap v2090: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:41.426 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:41 smithi142 ceph-mon[103614]: pgmap v2090: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:41.427 INFO:teuthology.orchestra.run.smithi142.stdout:382252089953 2024-08-17T17:57:41.427 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:57:41.531 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:41 smithi152 ceph-mon[93456]: pgmap v2090: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:41.939 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:41.939 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:41.940 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:42.067 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:57:41 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:57:41] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.51.0" 2024-08-17T17:57:42.342 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:42 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:42.343 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:42 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:42.365 INFO:teuthology.orchestra.run.smithi142.stdout:1945620185166 2024-08-17T17:57:42.365 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:57:42.441 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:42.531 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:42 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:43.287 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:43.372 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:43 smithi142 ceph-mon[103614]: pgmap v2091: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:43.373 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:43 smithi142 ceph-mon[106729]: pgmap v2091: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:43.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:43 smithi152 ceph-mon[93456]: pgmap v2091: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:43.960 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:45.622 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:45 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:45.623 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:45 smithi142 ceph-mon[103614]: pgmap v2092: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:45 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:45.623 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:45 smithi142 ceph-mon[106729]: pgmap v2092: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:45 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:57:45.823 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:45 smithi152 ceph-mon[93456]: pgmap v2092: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:46.952 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:46 smithi142 ceph-mon[103614]: pgmap v2093: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:46.952 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:46 smithi142 ceph-mon[106729]: pgmap v2093: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:47.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:46 smithi152 ceph-mon[93456]: pgmap v2093: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:48.090 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:48.870 INFO:teuthology.orchestra.run.smithi142.stdout:627065225761 2024-08-17T17:57:48.871 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:48 smithi142 ceph-mon[103614]: pgmap v2094: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:48.871 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:48 smithi142 ceph-mon[106729]: pgmap v2094: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:49.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:48 smithi152 ceph-mon[93456]: pgmap v2094: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:49.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:49 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/330671567' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:57:49.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:49 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/330671567' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:57:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:49 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/330671567' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:57:50.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:57:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:57:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:50.999 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:50 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1765339915' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:57:50.999 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:50 smithi142 ceph-mon[106729]: pgmap v2095: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:50.999 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:50 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1765339915' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:57:50.999 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:50 smithi142 ceph-mon[103614]: pgmap v2095: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:51.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:50 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1765339915' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:57:51.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:50 smithi152 ceph-mon[93456]: pgmap v2095: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:51.725 INFO:teuthology.orchestra.run.smithi142.stdout:455266533978 2024-08-17T17:57:52.122 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:57:51 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:57:51] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.51.0" 2024-08-17T17:57:52.851 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:52 smithi142 ceph-mon[103614]: pgmap v2096: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:52.852 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:52 smithi142 ceph-mon[106729]: pgmap v2096: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:52.883 INFO:tasks.cephadm.ceph_manager.ceph:need seq 455266533974 got 455266533978 for osd.1 2024-08-17T17:57:52.883 DEBUG:teuthology.parallel:result is None 2024-08-17T17:57:53.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:52 smithi152 ceph-mon[93456]: pgmap v2096: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:53.357 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:53.507 INFO:teuthology.orchestra.run.smithi142.stdout:382252089956 2024-08-17T17:57:53.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:53 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3150587191' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:57:53.825 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:53 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3150587191' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:57:54.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:53 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3150587191' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:57:54.251 INFO:teuthology.orchestra.run.smithi142.stdout:133143986933 2024-08-17T17:57:54.823 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:54 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1646355309' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:57:54.823 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:54 smithi142 ceph-mon[103614]: pgmap v2097: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:54.823 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:54 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1646355309' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:57:54.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:54 smithi142 ceph-mon[106729]: pgmap v2097: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:54 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1646355309' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:57:55.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:54 smithi152 ceph-mon[93456]: pgmap v2097: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:55.539 INFO:teuthology.orchestra.run.smithi142.stdout:854698492342 2024-08-17T17:57:55.801 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:55 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1187076628' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:57:55.801 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:55 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1187076628' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:57:55.802 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089953 got 382252089956 for osd.0 2024-08-17T17:57:55.803 DEBUG:teuthology.parallel:result is None 2024-08-17T17:57:55.876 INFO:tasks.cephadm.ceph_manager.ceph:need seq 627065225759 got 627065225761 for osd.3 2024-08-17T17:57:55.876 DEBUG:teuthology.parallel:result is None 2024-08-17T17:57:56.074 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:55 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1187076628' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:57:56.646 INFO:tasks.cephadm.ceph_manager.ceph:need seq 854698492338 got 854698492342 for osd.5 2024-08-17T17:57:56.646 DEBUG:teuthology.parallel:result is None 2024-08-17T17:57:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:56 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:56.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:56 smithi142 ceph-mon[106729]: pgmap v2098: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:56 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:56.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:56 smithi142 ceph-mon[103614]: pgmap v2098: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:57.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:56 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:57.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:56 smithi152 ceph-mon[93456]: pgmap v2098: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:57:57.628 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986930 got 133143986933 for osd.4 2024-08-17T17:57:57.628 DEBUG:teuthology.parallel:result is None 2024-08-17T17:57:58.579 INFO:teuthology.orchestra.run.smithi142.stdout:1945620185169 2024-08-17T17:57:58.705 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:58 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1713416237' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:57:58.706 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:58 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1713416237' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:57:59.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:58 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1713416237' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:58:00.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:57:59 smithi152 ceph-mon[93456]: pgmap v2099: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:00.115 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:57:59 smithi142 ceph-mon[106729]: pgmap v2099: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:00.115 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:57:59 smithi142 ceph-mon[103614]: pgmap v2099: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:00.143 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1945620185166 got 1945620185169 for osd.6 2024-08-17T17:58:00.143 DEBUG:teuthology.parallel:result is None 2024-08-17T17:58:00.492 INFO:teuthology.orchestra.run.smithi142.stdout:515396076088 2024-08-17T17:58:00.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:00.824 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:00 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/3718221899' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:58:00.824 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:00 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/3718221899' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:58:00.857 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:00 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/3718221899' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:58:02.070 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:58:01] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.51.0" 2024-08-17T17:58:02.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-mon[106729]: pgmap v2100: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:02.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.070 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-mon[103614]: pgmap v2100: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:02.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.071 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:01 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:01 smithi152 ceph-mon[93456]: pgmap v2100: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:01 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.094 INFO:tasks.cephadm.ceph_manager.ceph:need seq 515396076083 got 515396076088 for osd.2 2024-08-17T17:58:02.095 DEBUG:teuthology.parallel:result is None 2024-08-17T17:58:02.812 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[106729]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[106729]: pgmap v2101: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[103614]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:02.814 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:02 smithi142 ceph-mon[103614]: pgmap v2101: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:58:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:58:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:02 smithi152 ceph-mon[93456]: from='mgr.34107 172.21.15.142:0/3319668344' entity='mgr.y' 2024-08-17T17:58:03.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:02 smithi152 ceph-mon[93456]: pgmap v2101: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:03.236 INFO:teuthology.orchestra.run.smithi142.stdout:2194728288282 2024-08-17T17:58:03.922 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:03 smithi142 ceph-mon[106729]: from='client.? 172.21.15.142:0/1755496264' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:58:03.923 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:03 smithi142 ceph-mon[103614]: from='client.? 172.21.15.142:0/1755496264' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:58:03.947 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2194728288277 got 2194728288282 for osd.7 2024-08-17T17:58:03.948 DEBUG:teuthology.parallel:result is None 2024-08-17T17:58:03.948 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:58:03.948 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:58:04.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:03 smithi152 ceph-mon[93456]: from='client.? 172.21.15.142:0/1755496264' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:58:04.204 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:05.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:04 smithi152 ceph-mon[93456]: pgmap v2102: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:05.122 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:04 smithi142 ceph-mon[106729]: pgmap v2102: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:05.122 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:04 smithi142 ceph-mon[103614]: pgmap v2102: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:06.308 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:58:06.309 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:58:07.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:06 smithi142 ceph-mon[106729]: from='client.48803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:07.072 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:06 smithi142 ceph-mon[106729]: pgmap v2103: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:07.072 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:06 smithi142 ceph-mon[103614]: from='client.48803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:07.073 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:06 smithi142 ceph-mon[103614]: pgmap v2103: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:06 smithi152 ceph-mon[93456]: from='client.48803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:07.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:06 smithi152 ceph-mon[93456]: pgmap v2103: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:07.101 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":2102,"stamp":"2024-08-17T17:58:04.626528+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":4152094,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8623392,"kb_used_data":56900,"kb_used_omap":60,"kb_used_meta":8566339,"kb_avail":741337824,"statfs":{"total":767960285184,"available":759129931776,"internally_reserved":0,"allocated":58265600,"data_stored":35670199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":61862,"internal_metadata":8771931738},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003362"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":740,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742660+0000","last_change":"2024-08-17T17:56:48.923231+0000","last_active":"2024-08-17T17:57:16.742660+0000","last_peered":"2024-08-17T17:57:16.742660+0000","last_clean":"2024-08-17T17:57:16.742660+0000","last_became_active":"2024-08-17T17:56:48.922801+0000","last_became_peered":"2024-08-17T17:56:48.922801+0000","last_unstale":"2024-08-17T17:57:16.742660+0000","last_undegraded":"2024-08-17T17:57:16.742660+0000","last_fullsized":"2024-08-17T17:57:16.742660+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:31.893589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":56,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436894+0000","last_change":"2024-08-17T17:56:16.716895+0000","last_active":"2024-08-17T17:57:14.436894+0000","last_peered":"2024-08-17T17:57:14.436894+0000","last_clean":"2024-08-17T17:57:14.436894+0000","last_became_active":"2024-08-17T17:56:16.707886+0000","last_became_peered":"2024-08-17T17:56:16.707886+0000","last_unstale":"2024-08-17T17:57:14.436894+0000","last_undegraded":"2024-08-17T17:57:14.436894+0000","last_fullsized":"2024-08-17T17:57:14.436894+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:17:08.074085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":467,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409205+0000","last_change":"2024-08-17T17:56:13.701524+0000","last_active":"2024-08-17T17:57:14.409205+0000","last_peered":"2024-08-17T17:57:14.409205+0000","last_clean":"2024-08-17T17:57:14.409205+0000","last_became_active":"2024-08-17T17:56:13.701199+0000","last_became_peered":"2024-08-17T17:56:13.701199+0000","last_unstale":"2024-08-17T17:57:14.409205+0000","last_undegraded":"2024-08-17T17:57:14.409205+0000","last_fullsized":"2024-08-17T17:57:14.409205+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:18.981975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":75,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983065+0000","last_change":"2024-08-17T17:56:26.698788+0000","last_active":"2024-08-17T17:57:16.983065+0000","last_peered":"2024-08-17T17:57:16.983065+0000","last_clean":"2024-08-17T17:57:16.983065+0000","last_became_active":"2024-08-17T17:56:26.698492+0000","last_became_peered":"2024-08-17T17:56:26.698492+0000","last_unstale":"2024-08-17T17:57:16.983065+0000","last_undegraded":"2024-08-17T17:57:16.983065+0000","last_fullsized":"2024-08-17T17:57:16.983065+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:46:37.215212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":821,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983455+0000","last_change":"2024-08-17T17:56:26.697110+0000","last_active":"2024-08-17T17:57:16.983455+0000","last_peered":"2024-08-17T17:57:16.983455+0000","last_clean":"2024-08-17T17:57:16.983455+0000","last_became_active":"2024-08-17T17:56:26.696806+0000","last_became_peered":"2024-08-17T17:56:26.696806+0000","last_unstale":"2024-08-17T17:57:16.983455+0000","last_undegraded":"2024-08-17T17:57:16.983455+0000","last_fullsized":"2024-08-17T17:57:16.983455+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:11.561891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436738+0000","last_change":"2024-08-17T17:56:13.697570+0000","last_active":"2024-08-17T17:57:14.436738+0000","last_peered":"2024-08-17T17:57:14.436738+0000","last_clean":"2024-08-17T17:57:14.436738+0000","last_became_active":"2024-08-17T17:56:13.697201+0000","last_became_peered":"2024-08-17T17:56:13.697201+0000","last_unstale":"2024-08-17T17:57:14.436738+0000","last_undegraded":"2024-08-17T17:57:14.436738+0000","last_fullsized":"2024-08-17T17:57:14.436738+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:51.586418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":910,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.743136+0000","last_change":"2024-08-17T17:56:13.703302+0000","last_active":"2024-08-17T17:57:16.743136+0000","last_peered":"2024-08-17T17:57:16.743136+0000","last_clean":"2024-08-17T17:57:16.743136+0000","last_became_active":"2024-08-17T17:56:13.702945+0000","last_became_peered":"2024-08-17T17:56:13.702945+0000","last_unstale":"2024-08-17T17:57:16.743136+0000","last_undegraded":"2024-08-17T17:57:16.743136+0000","last_fullsized":"2024-08-17T17:57:16.743136+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:35.455764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":571,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724027+0000","last_change":"2024-08-17T17:55:43.577289+0000","last_active":"2024-08-17T17:57:13.724027+0000","last_peered":"2024-08-17T17:57:13.724027+0000","last_clean":"2024-08-17T17:57:13.724027+0000","last_became_active":"2024-08-17T17:55:43.557987+0000","last_became_peered":"2024-08-17T17:55:43.557987+0000","last_unstale":"2024-08-17T17:57:13.724027+0000","last_undegraded":"2024-08-17T17:57:13.724027+0000","last_fullsized":"2024-08-17T17:57:13.724027+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.922781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437286+0000","last_change":"2024-08-17T17:56:13.716268+0000","last_active":"2024-08-17T17:57:14.437286+0000","last_peered":"2024-08-17T17:57:14.437286+0000","last_clean":"2024-08-17T17:57:14.437286+0000","last_became_active":"2024-08-17T17:56:13.704484+0000","last_became_peered":"2024-08-17T17:56:13.704484+0000","last_unstale":"2024-08-17T17:57:14.437286+0000","last_undegraded":"2024-08-17T17:57:14.437286+0000","last_fullsized":"2024-08-17T17:57:14.437286+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:30.830639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":450,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.408845+0000","last_change":"2024-08-17T17:56:13.697698+0000","last_active":"2024-08-17T17:57:14.408845+0000","last_peered":"2024-08-17T17:57:14.408845+0000","last_clean":"2024-08-17T17:57:14.408845+0000","last_became_active":"2024-08-17T17:56:13.697368+0000","last_became_peered":"2024-08-17T17:56:13.697368+0000","last_unstale":"2024-08-17T17:57:14.408845+0000","last_undegraded":"2024-08-17T17:57:14.408845+0000","last_fullsized":"2024-08-17T17:57:14.408845+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:35.736513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437251+0000","last_change":"2024-08-17T17:56:13.720636+0000","last_active":"2024-08-17T17:57:14.437251+0000","last_peered":"2024-08-17T17:57:14.437251+0000","last_clean":"2024-08-17T17:57:14.437251+0000","last_became_active":"2024-08-17T17:56:13.705228+0000","last_became_peered":"2024-08-17T17:56:13.705228+0000","last_unstale":"2024-08-17T17:57:14.437251+0000","last_undegraded":"2024-08-17T17:57:14.437251+0000","last_fullsized":"2024-08-17T17:57:14.437251+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:17.022691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722003+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:57:13.722003+0000","last_peered":"2024-08-17T17:57:13.722003+0000","last_clean":"2024-08-17T17:57:13.722003+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:57:13.722003+0000","last_undegraded":"2024-08-17T17:57:13.722003+0000","last_fullsized":"2024-08-17T17:57:13.722003+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437176+0000","last_change":"2024-08-17T17:56:13.714434+0000","last_active":"2024-08-17T17:57:14.437176+0000","last_peered":"2024-08-17T17:57:14.437176+0000","last_clean":"2024-08-17T17:57:14.437176+0000","last_became_active":"2024-08-17T17:56:13.704336+0000","last_became_peered":"2024-08-17T17:56:13.704336+0000","last_unstale":"2024-08-17T17:57:14.437176+0000","last_undegraded":"2024-08-17T17:57:14.437176+0000","last_fullsized":"2024-08-17T17:57:14.437176+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:21.850091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.438107+0000","last_change":"2024-08-17T17:57:14.437762+0000","last_active":"2024-08-17T17:57:14.438107+0000","last_peered":"2024-08-17T17:57:14.438107+0000","last_clean":"2024-08-17T17:57:14.438107+0000","last_became_active":"2024-08-17T17:57:14.437448+0000","last_became_peered":"2024-08-17T17:57:14.437448+0000","last_unstale":"2024-08-17T17:57:14.438107+0000","last_undegraded":"2024-08-17T17:57:14.438107+0000","last_fullsized":"2024-08-17T17:57:14.438107+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:28.583831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":721,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409185+0000","last_change":"2024-08-17T17:55:24.727133+0000","last_active":"2024-08-17T17:57:14.409185+0000","last_peered":"2024-08-17T17:57:14.409185+0000","last_clean":"2024-08-17T17:57:14.409185+0000","last_became_active":"2024-08-17T17:55:24.726891+0000","last_became_peered":"2024-08-17T17:55:24.726891+0000","last_unstale":"2024-08-17T17:57:14.409185+0000","last_undegraded":"2024-08-17T17:57:14.409185+0000","last_fullsized":"2024-08-17T17:57:14.409185+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:36.257834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983520+0000","last_change":"2024-08-17T17:55:43.562214+0000","last_active":"2024-08-17T17:57:16.983520+0000","last_peered":"2024-08-17T17:57:16.983520+0000","last_clean":"2024-08-17T17:57:16.983520+0000","last_became_active":"2024-08-17T17:55:43.561906+0000","last_became_peered":"2024-08-17T17:55:43.561906+0000","last_unstale":"2024-08-17T17:57:16.983520+0000","last_undegraded":"2024-08-17T17:57:16.983520+0000","last_fullsized":"2024-08-17T17:57:16.983520+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:19:25.898467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":742,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742618+0000","last_change":"2024-08-17T17:56:48.703755+0000","last_active":"2024-08-17T17:57:16.742618+0000","last_peered":"2024-08-17T17:57:16.742618+0000","last_clean":"2024-08-17T17:57:16.742618+0000","last_became_active":"2024-08-17T17:56:48.703449+0000","last_became_peered":"2024-08-17T17:56:48.703449+0000","last_unstale":"2024-08-17T17:57:16.742618+0000","last_undegraded":"2024-08-17T17:57:16.742618+0000","last_fullsized":"2024-08-17T17:57:16.742618+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:47.197324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722309+0000","last_change":"2024-08-17T17:51:20.661629+0000","last_active":"2024-08-17T17:57:13.722309+0000","last_peered":"2024-08-17T17:57:13.722309+0000","last_clean":"2024-08-17T17:57:13.722309+0000","last_became_active":"2024-08-17T17:51:20.661353+0000","last_became_peered":"2024-08-17T17:51:20.661353+0000","last_unstale":"2024-08-17T17:57:13.722309+0000","last_undegraded":"2024-08-17T17:57:13.722309+0000","last_fullsized":"2024-08-17T17:57:13.722309+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:58.549718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":488,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724036+0000","last_change":"2024-08-17T17:51:20.643281+0000","last_active":"2024-08-17T17:57:13.724036+0000","last_peered":"2024-08-17T17:57:13.724036+0000","last_clean":"2024-08-17T17:57:13.724036+0000","last_became_active":"2024-08-17T17:51:20.642865+0000","last_became_peered":"2024-08-17T17:51:20.642865+0000","last_unstale":"2024-08-17T17:57:13.724036+0000","last_undegraded":"2024-08-17T17:57:13.724036+0000","last_fullsized":"2024-08-17T17:57:13.724036+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:08.053447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":544,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722618+0000","last_change":"2024-08-17T17:56:16.708310+0000","last_active":"2024-08-17T17:57:13.722618+0000","last_peered":"2024-08-17T17:57:13.722618+0000","last_clean":"2024-08-17T17:57:13.722618+0000","last_became_active":"2024-08-17T17:56:16.707992+0000","last_became_peered":"2024-08-17T17:56:16.707992+0000","last_unstale":"2024-08-17T17:57:13.722618+0000","last_undegraded":"2024-08-17T17:57:13.722618+0000","last_fullsized":"2024-08-17T17:57:13.722618+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:36.385074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":548,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722144+0000","last_change":"2024-08-17T17:56:16.709801+0000","last_active":"2024-08-17T17:57:13.722144+0000","last_peered":"2024-08-17T17:57:13.722144+0000","last_clean":"2024-08-17T17:57:13.722144+0000","last_became_active":"2024-08-17T17:56:16.709490+0000","last_became_peered":"2024-08-17T17:56:16.709490+0000","last_unstale":"2024-08-17T17:57:13.722144+0000","last_undegraded":"2024-08-17T17:57:13.722144+0000","last_fullsized":"2024-08-17T17:57:13.722144+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:11.131476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":527,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722614+0000","last_change":"2024-08-17T17:56:29.708231+0000","last_active":"2024-08-17T17:57:13.722614+0000","last_peered":"2024-08-17T17:57:13.722614+0000","last_clean":"2024-08-17T17:57:13.722614+0000","last_became_active":"2024-08-17T17:56:29.707834+0000","last_became_peered":"2024-08-17T17:56:29.707834+0000","last_unstale":"2024-08-17T17:57:13.722614+0000","last_undegraded":"2024-08-17T17:57:13.722614+0000","last_fullsized":"2024-08-17T17:57:13.722614+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:09.036256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":902,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742564+0000","last_change":"2024-08-17T17:56:13.697218+0000","last_active":"2024-08-17T17:57:16.742564+0000","last_peered":"2024-08-17T17:57:16.742564+0000","last_clean":"2024-08-17T17:57:16.742564+0000","last_became_active":"2024-08-17T17:56:13.696863+0000","last_became_peered":"2024-08-17T17:56:13.696863+0000","last_unstale":"2024-08-17T17:57:16.742564+0000","last_undegraded":"2024-08-17T17:57:16.742564+0000","last_fullsized":"2024-08-17T17:57:16.742564+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:55.636366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409271+0000","last_change":"2024-08-17T17:55:24.713174+0000","last_active":"2024-08-17T17:57:14.409271+0000","last_peered":"2024-08-17T17:57:14.409271+0000","last_clean":"2024-08-17T17:57:14.409271+0000","last_became_active":"2024-08-17T17:55:24.710147+0000","last_became_peered":"2024-08-17T17:55:24.710147+0000","last_unstale":"2024-08-17T17:57:14.409271+0000","last_undegraded":"2024-08-17T17:57:14.409271+0000","last_fullsized":"2024-08-17T17:57:14.409271+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:31:46.255385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":497,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436802+0000","last_change":"2024-08-17T17:56:16.710678+0000","last_active":"2024-08-17T17:57:14.436802+0000","last_peered":"2024-08-17T17:57:14.436802+0000","last_clean":"2024-08-17T17:57:14.436802+0000","last_became_active":"2024-08-17T17:56:16.709581+0000","last_became_peered":"2024-08-17T17:56:16.709581+0000","last_unstale":"2024-08-17T17:57:14.436802+0000","last_undegraded":"2024-08-17T17:57:14.436802+0000","last_fullsized":"2024-08-17T17:57:14.436802+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:40:53.545956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":520,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409043+0000","last_change":"2024-08-17T17:56:16.721074+0000","last_active":"2024-08-17T17:57:14.409043+0000","last_peered":"2024-08-17T17:57:14.409043+0000","last_clean":"2024-08-17T17:57:14.409043+0000","last_became_active":"2024-08-17T17:56:16.708516+0000","last_became_peered":"2024-08-17T17:56:16.708516+0000","last_unstale":"2024-08-17T17:57:14.409043+0000","last_undegraded":"2024-08-17T17:57:14.409043+0000","last_fullsized":"2024-08-17T17:57:14.409043+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:24.661532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":691,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742581+0000","last_change":"2024-08-17T17:56:49.272754+0000","last_active":"2024-08-17T17:57:16.742581+0000","last_peered":"2024-08-17T17:57:16.742581+0000","last_clean":"2024-08-17T17:57:16.742581+0000","last_became_active":"2024-08-17T17:56:49.272489+0000","last_became_peered":"2024-08-17T17:56:49.272489+0000","last_unstale":"2024-08-17T17:57:16.742581+0000","last_undegraded":"2024-08-17T17:57:16.742581+0000","last_fullsized":"2024-08-17T17:57:16.742581+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:04.415107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":511,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724175+0000","last_change":"2024-08-17T17:56:13.697602+0000","last_active":"2024-08-17T17:57:13.724175+0000","last_peered":"2024-08-17T17:57:13.724175+0000","last_clean":"2024-08-17T17:57:13.724175+0000","last_became_active":"2024-08-17T17:56:13.697324+0000","last_became_peered":"2024-08-17T17:56:13.697324+0000","last_unstale":"2024-08-17T17:57:13.724175+0000","last_undegraded":"2024-08-17T17:57:13.724175+0000","last_fullsized":"2024-08-17T17:57:13.724175+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:50.421356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":34,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436684+0000","last_change":"2024-08-17T17:56:13.701635+0000","last_active":"2024-08-17T17:57:14.436684+0000","last_peered":"2024-08-17T17:57:14.436684+0000","last_clean":"2024-08-17T17:57:14.436684+0000","last_became_active":"2024-08-17T17:56:13.701302+0000","last_became_peered":"2024-08-17T17:56:13.701302+0000","last_unstale":"2024-08-17T17:57:14.436684+0000","last_undegraded":"2024-08-17T17:57:14.436684+0000","last_fullsized":"2024-08-17T17:57:14.436684+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:40.510305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":499,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982719+0000","last_change":"2024-08-17T17:56:26.699992+0000","last_active":"2024-08-17T17:57:16.982719+0000","last_peered":"2024-08-17T17:57:16.982719+0000","last_clean":"2024-08-17T17:57:16.982719+0000","last_became_active":"2024-08-17T17:56:26.699390+0000","last_became_peered":"2024-08-17T17:56:26.699390+0000","last_unstale":"2024-08-17T17:57:16.982719+0000","last_undegraded":"2024-08-17T17:57:16.982719+0000","last_fullsized":"2024-08-17T17:57:16.982719+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:18.107253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983689+0000","last_change":"2024-08-17T17:56:16.712316+0000","last_active":"2024-08-17T17:57:16.983689+0000","last_peered":"2024-08-17T17:57:16.983689+0000","last_clean":"2024-08-17T17:57:16.983689+0000","last_became_active":"2024-08-17T17:56:16.711972+0000","last_became_peered":"2024-08-17T17:56:16.711972+0000","last_unstale":"2024-08-17T17:57:16.983689+0000","last_undegraded":"2024-08-17T17:57:16.983689+0000","last_fullsized":"2024-08-17T17:57:16.983689+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:56.249369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":54,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982844+0000","last_change":"2024-08-17T17:56:16.712593+0000","last_active":"2024-08-17T17:57:16.982844+0000","last_peered":"2024-08-17T17:57:16.982844+0000","last_clean":"2024-08-17T17:57:16.982844+0000","last_became_active":"2024-08-17T17:56:16.711911+0000","last_became_peered":"2024-08-17T17:56:16.711911+0000","last_unstale":"2024-08-17T17:57:16.982844+0000","last_undegraded":"2024-08-17T17:57:16.982844+0000","last_fullsized":"2024-08-17T17:57:16.982844+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:20.121328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":440,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982863+0000","last_change":"2024-08-17T17:56:32.756994+0000","last_active":"2024-08-17T17:57:16.982863+0000","last_peered":"2024-08-17T17:57:16.982863+0000","last_clean":"2024-08-17T17:57:16.982863+0000","last_became_active":"2024-08-17T17:56:32.756651+0000","last_became_peered":"2024-08-17T17:56:32.756651+0000","last_unstale":"2024-08-17T17:57:16.982863+0000","last_undegraded":"2024-08-17T17:57:16.982863+0000","last_fullsized":"2024-08-17T17:57:16.982863+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:46:29.100016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983811+0000","last_change":"2024-08-17T17:57:14.409938+0000","last_active":"2024-08-17T17:57:16.983811+0000","last_peered":"2024-08-17T17:57:16.983811+0000","last_clean":"2024-08-17T17:57:16.983811+0000","last_became_active":"2024-08-17T17:57:14.409649+0000","last_became_peered":"2024-08-17T17:57:14.409649+0000","last_unstale":"2024-08-17T17:57:16.983811+0000","last_undegraded":"2024-08-17T17:57:16.983811+0000","last_fullsized":"2024-08-17T17:57:16.983811+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:40.908624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":207,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409261+0000","last_change":"2024-08-17T17:55:24.726763+0000","last_active":"2024-08-17T17:57:14.409261+0000","last_peered":"2024-08-17T17:57:14.409261+0000","last_clean":"2024-08-17T17:57:14.409261+0000","last_became_active":"2024-08-17T17:55:24.726435+0000","last_became_peered":"2024-08-17T17:55:24.726435+0000","last_unstale":"2024-08-17T17:57:14.409261+0000","last_undegraded":"2024-08-17T17:57:14.409261+0000","last_fullsized":"2024-08-17T17:57:14.409261+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:19.761449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":548,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722472+0000","last_change":"2024-08-17T17:55:43.560148+0000","last_active":"2024-08-17T17:57:13.722472+0000","last_peered":"2024-08-17T17:57:13.722472+0000","last_clean":"2024-08-17T17:57:13.722472+0000","last_became_active":"2024-08-17T17:55:43.559822+0000","last_became_peered":"2024-08-17T17:55:43.559822+0000","last_unstale":"2024-08-17T17:57:13.722472+0000","last_undegraded":"2024-08-17T17:57:13.722472+0000","last_fullsized":"2024-08-17T17:57:13.722472+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:43.317205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":452,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983204+0000","last_change":"2024-08-17T17:56:32.756805+0000","last_active":"2024-08-17T17:57:16.983204+0000","last_peered":"2024-08-17T17:57:16.983204+0000","last_clean":"2024-08-17T17:57:16.983204+0000","last_became_active":"2024-08-17T17:56:32.756517+0000","last_became_peered":"2024-08-17T17:56:32.756517+0000","last_unstale":"2024-08-17T17:57:16.983204+0000","last_undegraded":"2024-08-17T17:57:16.983204+0000","last_fullsized":"2024-08-17T17:57:16.983204+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:47.280768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":478,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409182+0000","last_change":"2024-08-17T17:56:16.707525+0000","last_active":"2024-08-17T17:57:14.409182+0000","last_peered":"2024-08-17T17:57:14.409182+0000","last_clean":"2024-08-17T17:57:14.409182+0000","last_became_active":"2024-08-17T17:56:16.707200+0000","last_became_peered":"2024-08-17T17:56:16.707200+0000","last_unstale":"2024-08-17T17:57:14.409182+0000","last_undegraded":"2024-08-17T17:57:14.409182+0000","last_fullsized":"2024-08-17T17:57:14.409182+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:31:15.030862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":582,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983449+0000","last_change":"2024-08-17T17:56:32.873034+0000","last_active":"2024-08-17T17:57:16.983449+0000","last_peered":"2024-08-17T17:57:16.983449+0000","last_clean":"2024-08-17T17:57:16.983449+0000","last_became_active":"2024-08-17T17:56:32.872680+0000","last_became_peered":"2024-08-17T17:56:32.872680+0000","last_unstale":"2024-08-17T17:57:16.983449+0000","last_undegraded":"2024-08-17T17:57:16.983449+0000","last_fullsized":"2024-08-17T17:57:16.983449+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:36.951621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":919,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983199+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:57:16.983199+0000","last_peered":"2024-08-17T17:57:16.983199+0000","last_clean":"2024-08-17T17:57:16.983199+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:57:16.983199+0000","last_undegraded":"2024-08-17T17:57:16.983199+0000","last_fullsized":"2024-08-17T17:57:16.983199+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":465,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983305+0000","last_change":"2024-08-17T17:56:33.000834+0000","last_active":"2024-08-17T17:57:16.983305+0000","last_peered":"2024-08-17T17:57:16.983305+0000","last_clean":"2024-08-17T17:57:16.983305+0000","last_became_active":"2024-08-17T17:56:33.000545+0000","last_became_peered":"2024-08-17T17:56:33.000545+0000","last_unstale":"2024-08-17T17:57:16.983305+0000","last_undegraded":"2024-08-17T17:57:16.983305+0000","last_fullsized":"2024-08-17T17:57:16.983305+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:29:39.835110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437071+0000","last_change":"2024-08-17T17:56:13.705075+0000","last_active":"2024-08-17T17:57:14.437071+0000","last_peered":"2024-08-17T17:57:14.437071+0000","last_clean":"2024-08-17T17:57:14.437071+0000","last_became_active":"2024-08-17T17:56:13.704620+0000","last_became_peered":"2024-08-17T17:56:13.704620+0000","last_unstale":"2024-08-17T17:57:14.437071+0000","last_undegraded":"2024-08-17T17:57:14.437071+0000","last_fullsized":"2024-08-17T17:57:14.437071+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:23.036364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":635,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983427+0000","last_change":"2024-08-17T17:56:26.699475+0000","last_active":"2024-08-17T17:57:16.983427+0000","last_peered":"2024-08-17T17:57:16.983427+0000","last_clean":"2024-08-17T17:57:16.983427+0000","last_became_active":"2024-08-17T17:56:26.699152+0000","last_became_peered":"2024-08-17T17:56:26.699152+0000","last_unstale":"2024-08-17T17:57:16.983427+0000","last_undegraded":"2024-08-17T17:57:16.983427+0000","last_fullsized":"2024-08-17T17:57:16.983427+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:45:49.986225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":765,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.743224+0000","last_change":"2024-08-17T17:56:48.703032+0000","last_active":"2024-08-17T17:57:16.743224+0000","last_peered":"2024-08-17T17:57:16.743224+0000","last_clean":"2024-08-17T17:57:16.743224+0000","last_became_active":"2024-08-17T17:56:48.702627+0000","last_became_peered":"2024-08-17T17:56:48.702627+0000","last_unstale":"2024-08-17T17:57:16.743224+0000","last_undegraded":"2024-08-17T17:57:16.743224+0000","last_fullsized":"2024-08-17T17:57:16.743224+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:24.296226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":928,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983773+0000","last_change":"2024-08-17T17:51:20.639615+0000","last_active":"2024-08-17T17:57:16.983773+0000","last_peered":"2024-08-17T17:57:16.983773+0000","last_clean":"2024-08-17T17:57:16.983773+0000","last_became_active":"2024-08-17T17:51:20.639330+0000","last_became_peered":"2024-08-17T17:51:20.639330+0000","last_unstale":"2024-08-17T17:57:16.983773+0000","last_undegraded":"2024-08-17T17:57:16.983773+0000","last_fullsized":"2024-08-17T17:57:16.983773+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:41.751965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":914,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742368+0000","last_change":"2024-08-17T17:56:48.703034+0000","last_active":"2024-08-17T17:57:16.742368+0000","last_peered":"2024-08-17T17:57:16.742368+0000","last_clean":"2024-08-17T17:57:16.742368+0000","last_became_active":"2024-08-17T17:56:48.702701+0000","last_became_peered":"2024-08-17T17:56:48.702701+0000","last_unstale":"2024-08-17T17:57:16.742368+0000","last_undegraded":"2024-08-17T17:57:16.742368+0000","last_fullsized":"2024-08-17T17:57:16.742368+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:34.775405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":85,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983596+0000","last_change":"2024-08-17T17:55:43.568986+0000","last_active":"2024-08-17T17:57:16.983596+0000","last_peered":"2024-08-17T17:57:16.983596+0000","last_clean":"2024-08-17T17:57:16.983596+0000","last_became_active":"2024-08-17T17:55:43.561410+0000","last_became_peered":"2024-08-17T17:55:43.561410+0000","last_unstale":"2024-08-17T17:57:16.983596+0000","last_undegraded":"2024-08-17T17:57:16.983596+0000","last_fullsized":"2024-08-17T17:57:16.983596+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:48.268060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":541,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724413+0000","last_change":"2024-08-17T17:56:16.712692+0000","last_active":"2024-08-17T17:57:13.724413+0000","last_peered":"2024-08-17T17:57:13.724413+0000","last_clean":"2024-08-17T17:57:13.724413+0000","last_became_active":"2024-08-17T17:56:16.711843+0000","last_became_peered":"2024-08-17T17:56:16.711843+0000","last_unstale":"2024-08-17T17:57:13.724413+0000","last_undegraded":"2024-08-17T17:57:13.724413+0000","last_fullsized":"2024-08-17T17:57:13.724413+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:50.840470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722077+0000","last_change":"2024-08-17T17:51:20.651231+0000","last_active":"2024-08-17T17:57:13.722077+0000","last_peered":"2024-08-17T17:57:13.722077+0000","last_clean":"2024-08-17T17:57:13.722077+0000","last_became_active":"2024-08-17T17:51:20.650932+0000","last_became_peered":"2024-08-17T17:51:20.650932+0000","last_unstale":"2024-08-17T17:57:13.722077+0000","last_undegraded":"2024-08-17T17:57:13.722077+0000","last_fullsized":"2024-08-17T17:57:13.722077+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:55.890364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722327+0000","last_change":"2024-08-17T17:51:20.654600+0000","last_active":"2024-08-17T17:57:13.722327+0000","last_peered":"2024-08-17T17:57:13.722327+0000","last_clean":"2024-08-17T17:57:13.722327+0000","last_became_active":"2024-08-17T17:51:20.653501+0000","last_became_peered":"2024-08-17T17:51:20.653501+0000","last_unstale":"2024-08-17T17:57:13.722327+0000","last_undegraded":"2024-08-17T17:57:13.722327+0000","last_fullsized":"2024-08-17T17:57:13.722327+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:31.178218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":610,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722483+0000","last_change":"2024-08-17T17:56:13.712803+0000","last_active":"2024-08-17T17:57:13.722483+0000","last_peered":"2024-08-17T17:57:13.722483+0000","last_clean":"2024-08-17T17:57:13.722483+0000","last_became_active":"2024-08-17T17:56:13.702933+0000","last_became_peered":"2024-08-17T17:56:13.702933+0000","last_unstale":"2024-08-17T17:57:13.722483+0000","last_undegraded":"2024-08-17T17:57:13.722483+0000","last_fullsized":"2024-08-17T17:57:13.722483+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:31.008117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18976,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982974+0000","last_change":"2024-08-17T17:56:29.738340+0000","last_active":"2024-08-17T17:57:16.982974+0000","last_peered":"2024-08-17T17:57:16.982974+0000","last_clean":"2024-08-17T17:57:16.982974+0000","last_became_active":"2024-08-17T17:56:29.738036+0000","last_became_peered":"2024-08-17T17:56:29.738036+0000","last_unstale":"2024-08-17T17:57:16.982974+0000","last_undegraded":"2024-08-17T17:57:16.982974+0000","last_fullsized":"2024-08-17T17:57:16.982974+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:49:43.605252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":40,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437044+0000","last_change":"2024-08-17T17:56:13.722966+0000","last_active":"2024-08-17T17:57:14.437044+0000","last_peered":"2024-08-17T17:57:14.437044+0000","last_clean":"2024-08-17T17:57:14.437044+0000","last_became_active":"2024-08-17T17:56:13.705408+0000","last_became_peered":"2024-08-17T17:56:13.705408+0000","last_unstale":"2024-08-17T17:57:14.437044+0000","last_undegraded":"2024-08-17T17:57:14.437044+0000","last_fullsized":"2024-08-17T17:57:14.437044+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:40.685691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":451,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409036+0000","last_change":"2024-08-17T17:56:13.701419+0000","last_active":"2024-08-17T17:57:14.409036+0000","last_peered":"2024-08-17T17:57:14.409036+0000","last_clean":"2024-08-17T17:57:14.409036+0000","last_became_active":"2024-08-17T17:56:13.701094+0000","last_became_peered":"2024-08-17T17:56:13.701094+0000","last_unstale":"2024-08-17T17:57:14.409036+0000","last_undegraded":"2024-08-17T17:57:14.409036+0000","last_fullsized":"2024-08-17T17:57:14.409036+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:03:36.515687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":899,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742460+0000","last_change":"2024-08-17T17:56:13.697799+0000","last_active":"2024-08-17T17:57:16.742460+0000","last_peered":"2024-08-17T17:57:16.742460+0000","last_clean":"2024-08-17T17:57:16.742460+0000","last_became_active":"2024-08-17T17:56:13.697513+0000","last_became_peered":"2024-08-17T17:56:13.697513+0000","last_unstale":"2024-08-17T17:57:16.742460+0000","last_undegraded":"2024-08-17T17:57:16.742460+0000","last_fullsized":"2024-08-17T17:57:16.742460+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:28:18.215551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":611,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983660+0000","last_change":"2024-08-17T17:56:26.700340+0000","last_active":"2024-08-17T17:57:16.983660+0000","last_peered":"2024-08-17T17:57:16.983660+0000","last_clean":"2024-08-17T17:57:16.983660+0000","last_became_active":"2024-08-17T17:56:26.699510+0000","last_became_peered":"2024-08-17T17:56:26.699510+0000","last_unstale":"2024-08-17T17:57:16.983660+0000","last_undegraded":"2024-08-17T17:57:16.983660+0000","last_fullsized":"2024-08-17T17:57:16.983660+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:23.617760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":886,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742366+0000","last_change":"2024-08-17T17:51:20.673866+0000","last_active":"2024-08-17T17:57:16.742366+0000","last_peered":"2024-08-17T17:57:16.742366+0000","last_clean":"2024-08-17T17:57:16.742366+0000","last_became_active":"2024-08-17T17:51:20.673619+0000","last_became_peered":"2024-08-17T17:51:20.673619+0000","last_unstale":"2024-08-17T17:57:16.742366+0000","last_undegraded":"2024-08-17T17:57:16.742366+0000","last_fullsized":"2024-08-17T17:57:16.742366+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:42.965391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":474,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724425+0000","last_change":"2024-08-17T17:51:20.642743+0000","last_active":"2024-08-17T17:57:13.724425+0000","last_peered":"2024-08-17T17:57:13.724425+0000","last_clean":"2024-08-17T17:57:13.724425+0000","last_became_active":"2024-08-17T17:51:20.642129+0000","last_became_peered":"2024-08-17T17:51:20.642129+0000","last_unstale":"2024-08-17T17:57:13.724425+0000","last_undegraded":"2024-08-17T17:57:13.724425+0000","last_fullsized":"2024-08-17T17:57:13.724425+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:09.593211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":804,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742776+0000","last_change":"2024-08-17T17:56:48.923288+0000","last_active":"2024-08-17T17:57:16.742776+0000","last_peered":"2024-08-17T17:57:16.742776+0000","last_clean":"2024-08-17T17:57:16.742776+0000","last_became_active":"2024-08-17T17:56:48.922827+0000","last_became_peered":"2024-08-17T17:56:48.922827+0000","last_unstale":"2024-08-17T17:57:16.742776+0000","last_undegraded":"2024-08-17T17:57:16.742776+0000","last_fullsized":"2024-08-17T17:57:16.742776+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:17:17.043229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":608,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983337+0000","last_change":"2024-08-17T17:56:26.697321+0000","last_active":"2024-08-17T17:57:16.983337+0000","last_peered":"2024-08-17T17:57:16.983337+0000","last_clean":"2024-08-17T17:57:16.983337+0000","last_became_active":"2024-08-17T17:56:26.697042+0000","last_became_peered":"2024-08-17T17:56:26.697042+0000","last_unstale":"2024-08-17T17:57:16.983337+0000","last_undegraded":"2024-08-17T17:57:16.983337+0000","last_fullsized":"2024-08-17T17:57:16.983337+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:22.182085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":914,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982681+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:57:16.982681+0000","last_peered":"2024-08-17T17:57:16.982681+0000","last_clean":"2024-08-17T17:57:16.982681+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:57:16.982681+0000","last_undegraded":"2024-08-17T17:57:16.982681+0000","last_fullsized":"2024-08-17T17:57:16.982681+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":690,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742846+0000","last_change":"2024-08-17T17:56:48.923470+0000","last_active":"2024-08-17T17:57:16.742846+0000","last_peered":"2024-08-17T17:57:16.742846+0000","last_clean":"2024-08-17T17:57:16.742846+0000","last_became_active":"2024-08-17T17:56:48.922775+0000","last_became_peered":"2024-08-17T17:56:48.922775+0000","last_unstale":"2024-08-17T17:57:16.742846+0000","last_undegraded":"2024-08-17T17:57:16.742846+0000","last_fullsized":"2024-08-17T17:57:16.742846+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:54.249979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":549,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722214+0000","last_change":"2024-08-17T17:56:16.714976+0000","last_active":"2024-08-17T17:57:13.722214+0000","last_peered":"2024-08-17T17:57:13.722214+0000","last_clean":"2024-08-17T17:57:13.722214+0000","last_became_active":"2024-08-17T17:56:16.714660+0000","last_became_peered":"2024-08-17T17:56:16.714660+0000","last_unstale":"2024-08-17T17:57:13.722214+0000","last_undegraded":"2024-08-17T17:57:13.722214+0000","last_fullsized":"2024-08-17T17:57:13.722214+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:29.904777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":54,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983339+0000","last_change":"2024-08-17T17:56:16.710915+0000","last_active":"2024-08-17T17:57:16.983339+0000","last_peered":"2024-08-17T17:57:16.983339+0000","last_clean":"2024-08-17T17:57:16.983339+0000","last_became_active":"2024-08-17T17:56:16.710641+0000","last_became_peered":"2024-08-17T17:56:16.710641+0000","last_unstale":"2024-08-17T17:57:16.983339+0000","last_undegraded":"2024-08-17T17:57:16.983339+0000","last_fullsized":"2024-08-17T17:57:16.983339+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:38.936824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":166,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982950+0000","last_change":"2024-08-17T17:56:26.700363+0000","last_active":"2024-08-17T17:57:16.982950+0000","last_peered":"2024-08-17T17:57:16.982950+0000","last_clean":"2024-08-17T17:57:16.982950+0000","last_became_active":"2024-08-17T17:56:26.700094+0000","last_became_peered":"2024-08-17T17:56:26.700094+0000","last_unstale":"2024-08-17T17:57:16.982950+0000","last_undegraded":"2024-08-17T17:57:16.982950+0000","last_fullsized":"2024-08-17T17:57:16.982950+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:29.588348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":527,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722333+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:57:13.722333+0000","last_peered":"2024-08-17T17:57:13.722333+0000","last_clean":"2024-08-17T17:57:13.722333+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:57:13.722333+0000","last_undegraded":"2024-08-17T17:57:13.722333+0000","last_fullsized":"2024-08-17T17:57:13.722333+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":195,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409032+0000","last_change":"2024-08-17T17:55:24.726566+0000","last_active":"2024-08-17T17:57:14.409032+0000","last_peered":"2024-08-17T17:57:14.409032+0000","last_clean":"2024-08-17T17:57:14.409032+0000","last_became_active":"2024-08-17T17:55:24.720789+0000","last_became_peered":"2024-08-17T17:55:24.720789+0000","last_unstale":"2024-08-17T17:57:14.409032+0000","last_undegraded":"2024-08-17T17:57:14.409032+0000","last_fullsized":"2024-08-17T17:57:14.409032+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:47.386540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":547,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722076+0000","last_change":"2024-08-17T17:55:43.563475+0000","last_active":"2024-08-17T17:57:13.722076+0000","last_peered":"2024-08-17T17:57:13.722076+0000","last_clean":"2024-08-17T17:57:13.722076+0000","last_became_active":"2024-08-17T17:55:43.563172+0000","last_became_peered":"2024-08-17T17:55:43.563172+0000","last_unstale":"2024-08-17T17:57:13.722076+0000","last_undegraded":"2024-08-17T17:57:13.722076+0000","last_fullsized":"2024-08-17T17:57:13.722076+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:38:49.608894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":178,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409303+0000","last_change":"2024-08-17T17:55:24.731028+0000","last_active":"2024-08-17T17:57:14.409303+0000","last_peered":"2024-08-17T17:57:14.409303+0000","last_clean":"2024-08-17T17:57:14.409303+0000","last_became_active":"2024-08-17T17:55:24.718573+0000","last_became_peered":"2024-08-17T17:55:24.718573+0000","last_unstale":"2024-08-17T17:57:14.409303+0000","last_undegraded":"2024-08-17T17:57:14.409303+0000","last_fullsized":"2024-08-17T17:57:14.409303+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:05.843256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":743,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742734+0000","last_change":"2024-08-17T17:56:48.703219+0000","last_active":"2024-08-17T17:57:16.742734+0000","last_peered":"2024-08-17T17:57:16.742734+0000","last_clean":"2024-08-17T17:57:16.742734+0000","last_became_active":"2024-08-17T17:56:48.702897+0000","last_became_peered":"2024-08-17T17:56:48.702897+0000","last_unstale":"2024-08-17T17:57:16.742734+0000","last_undegraded":"2024-08-17T17:57:16.742734+0000","last_fullsized":"2024-08-17T17:57:16.742734+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:50.715495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":717,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.743117+0000","last_change":"2024-08-17T17:56:48.704289+0000","last_active":"2024-08-17T17:57:16.743117+0000","last_peered":"2024-08-17T17:57:16.743117+0000","last_clean":"2024-08-17T17:57:16.743117+0000","last_became_active":"2024-08-17T17:56:48.703857+0000","last_became_peered":"2024-08-17T17:56:48.703857+0000","last_unstale":"2024-08-17T17:57:16.743117+0000","last_undegraded":"2024-08-17T17:57:16.743117+0000","last_fullsized":"2024-08-17T17:57:16.743117+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:16.889242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.721954+0000","last_change":"2024-08-17T17:51:20.676098+0000","last_active":"2024-08-17T17:57:13.721954+0000","last_peered":"2024-08-17T17:57:13.721954+0000","last_clean":"2024-08-17T17:57:13.721954+0000","last_became_active":"2024-08-17T17:51:20.675786+0000","last_became_peered":"2024-08-17T17:51:20.675786+0000","last_unstale":"2024-08-17T17:57:13.721954+0000","last_undegraded":"2024-08-17T17:57:13.721954+0000","last_fullsized":"2024-08-17T17:57:13.721954+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:19.690923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":90,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983831+0000","last_change":"2024-08-17T17:56:26.700463+0000","last_active":"2024-08-17T17:57:16.983831+0000","last_peered":"2024-08-17T17:57:16.983831+0000","last_clean":"2024-08-17T17:57:16.983831+0000","last_became_active":"2024-08-17T17:56:26.699579+0000","last_became_peered":"2024-08-17T17:56:26.699579+0000","last_unstale":"2024-08-17T17:57:16.983831+0000","last_undegraded":"2024-08-17T17:57:16.983831+0000","last_fullsized":"2024-08-17T17:57:16.983831+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:02.259413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":19518,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44218,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":511,"seq":2194728288282,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":368140,"kb_used_data":7492,"kb_used_omap":15,"kb_used_meta":360624,"kb_avail":93377012,"statfs":{"total":95995035648,"available":95618060288,"internally_reserved":0,"allocated":7671808,"data_stored":4831000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15573,"internal_metadata":369279787},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":453,"seq":1945620185171,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":386184,"kb_used_data":7484,"kb_used_omap":13,"kb_used_meta":378674,"kb_avail":93358968,"statfs":{"total":95995035648,"available":95599583232,"internally_reserved":0,"allocated":7663616,"data_stored":4830857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13340,"internal_metadata":387763172},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492344,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":975796,"kb_used_data":6888,"kb_used_omap":7,"kb_used_meta":968888,"kb_avail":92769356,"statfs":{"total":95995035648,"available":94995820544,"internally_reserved":0,"allocated":7053312,"data_stored":4235273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":992141645},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986936,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2092036,"kb_used_data":6904,"kb_used_omap":1,"kb_used_meta":2085118,"kb_avail":91653116,"statfs":{"total":95995035648,"available":93852790784,"internally_reserved":0,"allocated":7069696,"data_stored":4240297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":2135161176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225764,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1354032,"kb_used_data":6884,"kb_used_omap":4,"kb_used_meta":1347131,"kb_avail":92391120,"statfs":{"total":95995035648,"available":94608506880,"internally_reserved":0,"allocated":7049216,"data_stored":4235501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1379462592},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396076089,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":701432,"kb_used_data":6892,"kb_used_omap":6,"kb_used_meta":694521,"kb_avail":93043720,"statfs":{"total":95995035648,"available":95276769280,"internally_reserved":0,"allocated":7057408,"data_stored":4236312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":711189947},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533982,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516528,"kb_used_data":6884,"kb_used_omap":5,"kb_used_meta":1509626,"kb_avail":92228624,"statfs":{"total":95995035648,"available":94442110976,"internally_reserved":0,"allocated":7049216,"data_stored":4234299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1545857264},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089958,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1229244,"kb_used_data":7472,"kb_used_omap":5,"kb_used_meta":1221754,"kb_avail":92515908,"statfs":{"total":95995035648,"available":94736289792,"internally_reserved":0,"allocated":7651328,"data_stored":4826660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1251076155},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11636,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12704,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:58:07.105 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:58:07.361 INFO:teuthology.orchestra.run.smithi142.stderr:Inferring config /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:08.868 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:08 smithi142 ceph-mon[106729]: pgmap v2104: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:08.868 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:08 smithi142 ceph-mon[103614]: pgmap v2104: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:09.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:08 smithi152 ceph-mon[93456]: pgmap v2104: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:58:09.261 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-08-17T17:58:09.262 INFO:teuthology.orchestra.run.smithi142.stderr:dumped all 2024-08-17T17:58:09.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:09 smithi142 ceph-mon[103614]: from='client.48809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:09.872 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:09 smithi142 ceph-mon[106729]: from='client.48809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:09.939 INFO:teuthology.orchestra.run.smithi142.stdout:{"pg_ready":true,"pg_map":{"version":2104,"stamp":"2024-08-17T17:58:08.627670+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":4152094,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8623392,"kb_used_data":56900,"kb_used_omap":60,"kb_used_meta":8566339,"kb_avail":741337824,"statfs":{"total":767960285184,"available":759129931776,"internally_reserved":0,"allocated":58265600,"data_stored":35670199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":61862,"internal_metadata":8771931738},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003223"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":740,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742660+0000","last_change":"2024-08-17T17:56:48.923231+0000","last_active":"2024-08-17T17:57:16.742660+0000","last_peered":"2024-08-17T17:57:16.742660+0000","last_clean":"2024-08-17T17:57:16.742660+0000","last_became_active":"2024-08-17T17:56:48.922801+0000","last_became_peered":"2024-08-17T17:56:48.922801+0000","last_unstale":"2024-08-17T17:57:16.742660+0000","last_undegraded":"2024-08-17T17:57:16.742660+0000","last_fullsized":"2024-08-17T17:57:16.742660+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:31.893589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":56,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436894+0000","last_change":"2024-08-17T17:56:16.716895+0000","last_active":"2024-08-17T17:57:14.436894+0000","last_peered":"2024-08-17T17:57:14.436894+0000","last_clean":"2024-08-17T17:57:14.436894+0000","last_became_active":"2024-08-17T17:56:16.707886+0000","last_became_peered":"2024-08-17T17:56:16.707886+0000","last_unstale":"2024-08-17T17:57:14.436894+0000","last_undegraded":"2024-08-17T17:57:14.436894+0000","last_fullsized":"2024-08-17T17:57:14.436894+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:17:08.074085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":467,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409205+0000","last_change":"2024-08-17T17:56:13.701524+0000","last_active":"2024-08-17T17:57:14.409205+0000","last_peered":"2024-08-17T17:57:14.409205+0000","last_clean":"2024-08-17T17:57:14.409205+0000","last_became_active":"2024-08-17T17:56:13.701199+0000","last_became_peered":"2024-08-17T17:56:13.701199+0000","last_unstale":"2024-08-17T17:57:14.409205+0000","last_undegraded":"2024-08-17T17:57:14.409205+0000","last_fullsized":"2024-08-17T17:57:14.409205+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:18.981975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":76,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983111+0000","last_change":"2024-08-17T17:56:26.698788+0000","last_active":"2024-08-17T17:57:16.983111+0000","last_peered":"2024-08-17T17:57:16.983111+0000","last_clean":"2024-08-17T17:57:16.983111+0000","last_became_active":"2024-08-17T17:56:26.698492+0000","last_became_peered":"2024-08-17T17:56:26.698492+0000","last_unstale":"2024-08-17T17:57:16.983111+0000","last_undegraded":"2024-08-17T17:57:16.983111+0000","last_fullsized":"2024-08-17T17:57:16.983111+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:46:37.215212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":821,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983455+0000","last_change":"2024-08-17T17:56:26.697110+0000","last_active":"2024-08-17T17:57:16.983455+0000","last_peered":"2024-08-17T17:57:16.983455+0000","last_clean":"2024-08-17T17:57:16.983455+0000","last_became_active":"2024-08-17T17:56:26.696806+0000","last_became_peered":"2024-08-17T17:56:26.696806+0000","last_unstale":"2024-08-17T17:57:16.983455+0000","last_undegraded":"2024-08-17T17:57:16.983455+0000","last_fullsized":"2024-08-17T17:57:16.983455+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:11.561891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436738+0000","last_change":"2024-08-17T17:56:13.697570+0000","last_active":"2024-08-17T17:57:14.436738+0000","last_peered":"2024-08-17T17:57:14.436738+0000","last_clean":"2024-08-17T17:57:14.436738+0000","last_became_active":"2024-08-17T17:56:13.697201+0000","last_became_peered":"2024-08-17T17:56:13.697201+0000","last_unstale":"2024-08-17T17:57:14.436738+0000","last_undegraded":"2024-08-17T17:57:14.436738+0000","last_fullsized":"2024-08-17T17:57:14.436738+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:51.586418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":910,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.743136+0000","last_change":"2024-08-17T17:56:13.703302+0000","last_active":"2024-08-17T17:57:16.743136+0000","last_peered":"2024-08-17T17:57:16.743136+0000","last_clean":"2024-08-17T17:57:16.743136+0000","last_became_active":"2024-08-17T17:56:13.702945+0000","last_became_peered":"2024-08-17T17:56:13.702945+0000","last_unstale":"2024-08-17T17:57:16.743136+0000","last_undegraded":"2024-08-17T17:57:16.743136+0000","last_fullsized":"2024-08-17T17:57:16.743136+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:35.455764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":571,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724027+0000","last_change":"2024-08-17T17:55:43.577289+0000","last_active":"2024-08-17T17:57:13.724027+0000","last_peered":"2024-08-17T17:57:13.724027+0000","last_clean":"2024-08-17T17:57:13.724027+0000","last_became_active":"2024-08-17T17:55:43.557987+0000","last_became_peered":"2024-08-17T17:55:43.557987+0000","last_unstale":"2024-08-17T17:57:13.724027+0000","last_undegraded":"2024-08-17T17:57:13.724027+0000","last_fullsized":"2024-08-17T17:57:13.724027+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:38:15.922781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437286+0000","last_change":"2024-08-17T17:56:13.716268+0000","last_active":"2024-08-17T17:57:14.437286+0000","last_peered":"2024-08-17T17:57:14.437286+0000","last_clean":"2024-08-17T17:57:14.437286+0000","last_became_active":"2024-08-17T17:56:13.704484+0000","last_became_peered":"2024-08-17T17:56:13.704484+0000","last_unstale":"2024-08-17T17:57:14.437286+0000","last_undegraded":"2024-08-17T17:57:14.437286+0000","last_fullsized":"2024-08-17T17:57:14.437286+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:30.830639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":450,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.408845+0000","last_change":"2024-08-17T17:56:13.697698+0000","last_active":"2024-08-17T17:57:14.408845+0000","last_peered":"2024-08-17T17:57:14.408845+0000","last_clean":"2024-08-17T17:57:14.408845+0000","last_became_active":"2024-08-17T17:56:13.697368+0000","last_became_peered":"2024-08-17T17:56:13.697368+0000","last_unstale":"2024-08-17T17:57:14.408845+0000","last_undegraded":"2024-08-17T17:57:14.408845+0000","last_fullsized":"2024-08-17T17:57:14.408845+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:03:35.736513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437251+0000","last_change":"2024-08-17T17:56:13.720636+0000","last_active":"2024-08-17T17:57:14.437251+0000","last_peered":"2024-08-17T17:57:14.437251+0000","last_clean":"2024-08-17T17:57:14.437251+0000","last_became_active":"2024-08-17T17:56:13.705228+0000","last_became_peered":"2024-08-17T17:56:13.705228+0000","last_unstale":"2024-08-17T17:57:14.437251+0000","last_undegraded":"2024-08-17T17:57:14.437251+0000","last_fullsized":"2024-08-17T17:57:14.437251+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:17.022691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722003+0000","last_change":"2024-08-17T17:12:40.334891+0000","last_active":"2024-08-17T17:57:13.722003+0000","last_peered":"2024-08-17T17:57:13.722003+0000","last_clean":"2024-08-17T17:57:13.722003+0000","last_became_active":"2024-08-17T17:12:40.334552+0000","last_became_peered":"2024-08-17T17:12:40.334552+0000","last_unstale":"2024-08-17T17:57:13.722003+0000","last_undegraded":"2024-08-17T17:57:13.722003+0000","last_fullsized":"2024-08-17T17:57:13.722003+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:15.366614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":38,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437176+0000","last_change":"2024-08-17T17:56:13.714434+0000","last_active":"2024-08-17T17:57:14.437176+0000","last_peered":"2024-08-17T17:57:14.437176+0000","last_clean":"2024-08-17T17:57:14.437176+0000","last_became_active":"2024-08-17T17:56:13.704336+0000","last_became_peered":"2024-08-17T17:56:13.704336+0000","last_unstale":"2024-08-17T17:57:14.437176+0000","last_undegraded":"2024-08-17T17:57:14.437176+0000","last_fullsized":"2024-08-17T17:57:14.437176+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:21.850091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.438107+0000","last_change":"2024-08-17T17:57:14.437762+0000","last_active":"2024-08-17T17:57:14.438107+0000","last_peered":"2024-08-17T17:57:14.438107+0000","last_clean":"2024-08-17T17:57:14.438107+0000","last_became_active":"2024-08-17T17:57:14.437448+0000","last_became_peered":"2024-08-17T17:57:14.437448+0000","last_unstale":"2024-08-17T17:57:14.438107+0000","last_undegraded":"2024-08-17T17:57:14.438107+0000","last_fullsized":"2024-08-17T17:57:14.438107+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:21:28.583831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":721,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409185+0000","last_change":"2024-08-17T17:55:24.727133+0000","last_active":"2024-08-17T17:57:14.409185+0000","last_peered":"2024-08-17T17:57:14.409185+0000","last_clean":"2024-08-17T17:57:14.409185+0000","last_became_active":"2024-08-17T17:55:24.726891+0000","last_became_peered":"2024-08-17T17:55:24.726891+0000","last_unstale":"2024-08-17T17:57:14.409185+0000","last_undegraded":"2024-08-17T17:57:14.409185+0000","last_fullsized":"2024-08-17T17:57:14.409185+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:36.257834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983520+0000","last_change":"2024-08-17T17:55:43.562214+0000","last_active":"2024-08-17T17:57:16.983520+0000","last_peered":"2024-08-17T17:57:16.983520+0000","last_clean":"2024-08-17T17:57:16.983520+0000","last_became_active":"2024-08-17T17:55:43.561906+0000","last_became_peered":"2024-08-17T17:55:43.561906+0000","last_unstale":"2024-08-17T17:57:16.983520+0000","last_undegraded":"2024-08-17T17:57:16.983520+0000","last_fullsized":"2024-08-17T17:57:16.983520+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:19:25.898467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":742,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742618+0000","last_change":"2024-08-17T17:56:48.703755+0000","last_active":"2024-08-17T17:57:16.742618+0000","last_peered":"2024-08-17T17:57:16.742618+0000","last_clean":"2024-08-17T17:57:16.742618+0000","last_became_active":"2024-08-17T17:56:48.703449+0000","last_became_peered":"2024-08-17T17:56:48.703449+0000","last_unstale":"2024-08-17T17:57:16.742618+0000","last_undegraded":"2024-08-17T17:57:16.742618+0000","last_fullsized":"2024-08-17T17:57:16.742618+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:47.197324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722309+0000","last_change":"2024-08-17T17:51:20.661629+0000","last_active":"2024-08-17T17:57:13.722309+0000","last_peered":"2024-08-17T17:57:13.722309+0000","last_clean":"2024-08-17T17:57:13.722309+0000","last_became_active":"2024-08-17T17:51:20.661353+0000","last_became_peered":"2024-08-17T17:51:20.661353+0000","last_unstale":"2024-08-17T17:57:13.722309+0000","last_undegraded":"2024-08-17T17:57:13.722309+0000","last_fullsized":"2024-08-17T17:57:13.722309+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:58.549718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":488,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724036+0000","last_change":"2024-08-17T17:51:20.643281+0000","last_active":"2024-08-17T17:57:13.724036+0000","last_peered":"2024-08-17T17:57:13.724036+0000","last_clean":"2024-08-17T17:57:13.724036+0000","last_became_active":"2024-08-17T17:51:20.642865+0000","last_became_peered":"2024-08-17T17:51:20.642865+0000","last_unstale":"2024-08-17T17:57:13.724036+0000","last_undegraded":"2024-08-17T17:57:13.724036+0000","last_fullsized":"2024-08-17T17:57:13.724036+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:08.053447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":544,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722618+0000","last_change":"2024-08-17T17:56:16.708310+0000","last_active":"2024-08-17T17:57:13.722618+0000","last_peered":"2024-08-17T17:57:13.722618+0000","last_clean":"2024-08-17T17:57:13.722618+0000","last_became_active":"2024-08-17T17:56:16.707992+0000","last_became_peered":"2024-08-17T17:56:16.707992+0000","last_unstale":"2024-08-17T17:57:13.722618+0000","last_undegraded":"2024-08-17T17:57:13.722618+0000","last_fullsized":"2024-08-17T17:57:13.722618+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:36.385074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":548,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722144+0000","last_change":"2024-08-17T17:56:16.709801+0000","last_active":"2024-08-17T17:57:13.722144+0000","last_peered":"2024-08-17T17:57:13.722144+0000","last_clean":"2024-08-17T17:57:13.722144+0000","last_became_active":"2024-08-17T17:56:16.709490+0000","last_became_peered":"2024-08-17T17:56:16.709490+0000","last_unstale":"2024-08-17T17:57:13.722144+0000","last_undegraded":"2024-08-17T17:57:13.722144+0000","last_fullsized":"2024-08-17T17:57:13.722144+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:11.131476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":527,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722614+0000","last_change":"2024-08-17T17:56:29.708231+0000","last_active":"2024-08-17T17:57:13.722614+0000","last_peered":"2024-08-17T17:57:13.722614+0000","last_clean":"2024-08-17T17:57:13.722614+0000","last_became_active":"2024-08-17T17:56:29.707834+0000","last_became_peered":"2024-08-17T17:56:29.707834+0000","last_unstale":"2024-08-17T17:57:13.722614+0000","last_undegraded":"2024-08-17T17:57:13.722614+0000","last_fullsized":"2024-08-17T17:57:13.722614+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:09.036256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":902,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742564+0000","last_change":"2024-08-17T17:56:13.697218+0000","last_active":"2024-08-17T17:57:16.742564+0000","last_peered":"2024-08-17T17:57:16.742564+0000","last_clean":"2024-08-17T17:57:16.742564+0000","last_became_active":"2024-08-17T17:56:13.696863+0000","last_became_peered":"2024-08-17T17:56:13.696863+0000","last_unstale":"2024-08-17T17:57:16.742564+0000","last_undegraded":"2024-08-17T17:57:16.742564+0000","last_fullsized":"2024-08-17T17:57:16.742564+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:55.636366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409271+0000","last_change":"2024-08-17T17:55:24.713174+0000","last_active":"2024-08-17T17:57:14.409271+0000","last_peered":"2024-08-17T17:57:14.409271+0000","last_clean":"2024-08-17T17:57:14.409271+0000","last_became_active":"2024-08-17T17:55:24.710147+0000","last_became_peered":"2024-08-17T17:55:24.710147+0000","last_unstale":"2024-08-17T17:57:14.409271+0000","last_undegraded":"2024-08-17T17:57:14.409271+0000","last_fullsized":"2024-08-17T17:57:14.409271+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:31:46.255385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":497,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436802+0000","last_change":"2024-08-17T17:56:16.710678+0000","last_active":"2024-08-17T17:57:14.436802+0000","last_peered":"2024-08-17T17:57:14.436802+0000","last_clean":"2024-08-17T17:57:14.436802+0000","last_became_active":"2024-08-17T17:56:16.709581+0000","last_became_peered":"2024-08-17T17:56:16.709581+0000","last_unstale":"2024-08-17T17:57:14.436802+0000","last_undegraded":"2024-08-17T17:57:14.436802+0000","last_fullsized":"2024-08-17T17:57:14.436802+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:48.187990+0000","last_clean_scrub_stamp":"2024-08-17T16:53:48.187990+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:40:53.545956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":520,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409043+0000","last_change":"2024-08-17T17:56:16.721074+0000","last_active":"2024-08-17T17:57:14.409043+0000","last_peered":"2024-08-17T17:57:14.409043+0000","last_clean":"2024-08-17T17:57:14.409043+0000","last_became_active":"2024-08-17T17:56:16.708516+0000","last_became_peered":"2024-08-17T17:56:16.708516+0000","last_unstale":"2024-08-17T17:57:14.409043+0000","last_undegraded":"2024-08-17T17:57:14.409043+0000","last_fullsized":"2024-08-17T17:57:14.409043+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:32:24.661532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067987099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":691,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742581+0000","last_change":"2024-08-17T17:56:49.272754+0000","last_active":"2024-08-17T17:57:16.742581+0000","last_peered":"2024-08-17T17:57:16.742581+0000","last_clean":"2024-08-17T17:57:16.742581+0000","last_became_active":"2024-08-17T17:56:49.272489+0000","last_became_peered":"2024-08-17T17:56:49.272489+0000","last_unstale":"2024-08-17T17:57:16.742581+0000","last_undegraded":"2024-08-17T17:57:16.742581+0000","last_fullsized":"2024-08-17T17:57:16.742581+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:04.415107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":511,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724175+0000","last_change":"2024-08-17T17:56:13.697602+0000","last_active":"2024-08-17T17:57:13.724175+0000","last_peered":"2024-08-17T17:57:13.724175+0000","last_clean":"2024-08-17T17:57:13.724175+0000","last_became_active":"2024-08-17T17:56:13.697324+0000","last_became_peered":"2024-08-17T17:56:13.697324+0000","last_unstale":"2024-08-17T17:57:13.724175+0000","last_undegraded":"2024-08-17T17:57:13.724175+0000","last_fullsized":"2024-08-17T17:57:13.724175+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:50.421356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":34,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.436684+0000","last_change":"2024-08-17T17:56:13.701635+0000","last_active":"2024-08-17T17:57:14.436684+0000","last_peered":"2024-08-17T17:57:14.436684+0000","last_clean":"2024-08-17T17:57:14.436684+0000","last_became_active":"2024-08-17T17:56:13.701302+0000","last_became_peered":"2024-08-17T17:56:13.701302+0000","last_unstale":"2024-08-17T17:57:14.436684+0000","last_undegraded":"2024-08-17T17:57:14.436684+0000","last_fullsized":"2024-08-17T17:57:14.436684+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:40.510305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046263899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":499,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982719+0000","last_change":"2024-08-17T17:56:26.699992+0000","last_active":"2024-08-17T17:57:16.982719+0000","last_peered":"2024-08-17T17:57:16.982719+0000","last_clean":"2024-08-17T17:57:16.982719+0000","last_became_active":"2024-08-17T17:56:26.699390+0000","last_became_peered":"2024-08-17T17:56:26.699390+0000","last_unstale":"2024-08-17T17:57:16.982719+0000","last_undegraded":"2024-08-17T17:57:16.982719+0000","last_fullsized":"2024-08-17T17:57:16.982719+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:18.107253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983689+0000","last_change":"2024-08-17T17:56:16.712316+0000","last_active":"2024-08-17T17:57:16.983689+0000","last_peered":"2024-08-17T17:57:16.983689+0000","last_clean":"2024-08-17T17:57:16.983689+0000","last_became_active":"2024-08-17T17:56:16.711972+0000","last_became_peered":"2024-08-17T17:56:16.711972+0000","last_unstale":"2024-08-17T17:57:16.983689+0000","last_undegraded":"2024-08-17T17:57:16.983689+0000","last_fullsized":"2024-08-17T17:57:16.983689+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:56.249369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":54,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982844+0000","last_change":"2024-08-17T17:56:16.712593+0000","last_active":"2024-08-17T17:57:16.982844+0000","last_peered":"2024-08-17T17:57:16.982844+0000","last_clean":"2024-08-17T17:57:16.982844+0000","last_became_active":"2024-08-17T17:56:16.711911+0000","last_became_peered":"2024-08-17T17:56:16.711911+0000","last_unstale":"2024-08-17T17:57:16.982844+0000","last_undegraded":"2024-08-17T17:57:16.982844+0000","last_fullsized":"2024-08-17T17:57:16.982844+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:20.121328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029484800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":440,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982863+0000","last_change":"2024-08-17T17:56:32.756994+0000","last_active":"2024-08-17T17:57:16.982863+0000","last_peered":"2024-08-17T17:57:16.982863+0000","last_clean":"2024-08-17T17:57:16.982863+0000","last_became_active":"2024-08-17T17:56:32.756651+0000","last_became_peered":"2024-08-17T17:56:32.756651+0000","last_unstale":"2024-08-17T17:57:16.982863+0000","last_undegraded":"2024-08-17T17:57:16.982863+0000","last_fullsized":"2024-08-17T17:57:16.982863+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:46:29.100016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983811+0000","last_change":"2024-08-17T17:57:14.409938+0000","last_active":"2024-08-17T17:57:16.983811+0000","last_peered":"2024-08-17T17:57:16.983811+0000","last_clean":"2024-08-17T17:57:16.983811+0000","last_became_active":"2024-08-17T17:57:14.409649+0000","last_became_peered":"2024-08-17T17:57:14.409649+0000","last_unstale":"2024-08-17T17:57:16.983811+0000","last_undegraded":"2024-08-17T17:57:16.983811+0000","last_fullsized":"2024-08-17T17:57:16.983811+0000","mapping_epoch":535,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":536,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:40.908624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":207,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409261+0000","last_change":"2024-08-17T17:55:24.726763+0000","last_active":"2024-08-17T17:57:14.409261+0000","last_peered":"2024-08-17T17:57:14.409261+0000","last_clean":"2024-08-17T17:57:14.409261+0000","last_became_active":"2024-08-17T17:55:24.726435+0000","last_became_peered":"2024-08-17T17:55:24.726435+0000","last_unstale":"2024-08-17T17:57:14.409261+0000","last_undegraded":"2024-08-17T17:57:14.409261+0000","last_fullsized":"2024-08-17T17:57:14.409261+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:19.761449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001101751,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":548,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722472+0000","last_change":"2024-08-17T17:55:43.560148+0000","last_active":"2024-08-17T17:57:13.722472+0000","last_peered":"2024-08-17T17:57:13.722472+0000","last_clean":"2024-08-17T17:57:13.722472+0000","last_became_active":"2024-08-17T17:55:43.559822+0000","last_became_peered":"2024-08-17T17:55:43.559822+0000","last_unstale":"2024-08-17T17:57:13.722472+0000","last_undegraded":"2024-08-17T17:57:13.722472+0000","last_fullsized":"2024-08-17T17:57:13.722472+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:43.317205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":452,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983204+0000","last_change":"2024-08-17T17:56:32.756805+0000","last_active":"2024-08-17T17:57:16.983204+0000","last_peered":"2024-08-17T17:57:16.983204+0000","last_clean":"2024-08-17T17:57:16.983204+0000","last_became_active":"2024-08-17T17:56:32.756517+0000","last_became_peered":"2024-08-17T17:56:32.756517+0000","last_unstale":"2024-08-17T17:57:16.983204+0000","last_undegraded":"2024-08-17T17:57:16.983204+0000","last_fullsized":"2024-08-17T17:57:16.983204+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:47.280768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":478,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409182+0000","last_change":"2024-08-17T17:56:16.707525+0000","last_active":"2024-08-17T17:57:14.409182+0000","last_peered":"2024-08-17T17:57:14.409182+0000","last_clean":"2024-08-17T17:57:14.409182+0000","last_became_active":"2024-08-17T17:56:16.707200+0000","last_became_peered":"2024-08-17T17:56:16.707200+0000","last_unstale":"2024-08-17T17:57:14.409182+0000","last_undegraded":"2024-08-17T17:57:14.409182+0000","last_fullsized":"2024-08-17T17:57:14.409182+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:31:15.030862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":582,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983449+0000","last_change":"2024-08-17T17:56:32.873034+0000","last_active":"2024-08-17T17:57:16.983449+0000","last_peered":"2024-08-17T17:57:16.983449+0000","last_clean":"2024-08-17T17:57:16.983449+0000","last_became_active":"2024-08-17T17:56:32.872680+0000","last_became_peered":"2024-08-17T17:56:32.872680+0000","last_unstale":"2024-08-17T17:57:16.983449+0000","last_undegraded":"2024-08-17T17:57:16.983449+0000","last_fullsized":"2024-08-17T17:57:16.983449+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:36.951621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028976499999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":919,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983199+0000","last_change":"2024-08-17T17:21:31.910740+0000","last_active":"2024-08-17T17:57:16.983199+0000","last_peered":"2024-08-17T17:57:16.983199+0000","last_clean":"2024-08-17T17:57:16.983199+0000","last_became_active":"2024-08-17T17:21:31.910323+0000","last_became_peered":"2024-08-17T17:21:31.910323+0000","last_unstale":"2024-08-17T17:57:16.983199+0000","last_undegraded":"2024-08-17T17:57:16.983199+0000","last_fullsized":"2024-08-17T17:57:16.983199+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:10.476546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":465,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983305+0000","last_change":"2024-08-17T17:56:33.000834+0000","last_active":"2024-08-17T17:57:16.983305+0000","last_peered":"2024-08-17T17:57:16.983305+0000","last_clean":"2024-08-17T17:57:16.983305+0000","last_became_active":"2024-08-17T17:56:33.000545+0000","last_became_peered":"2024-08-17T17:56:33.000545+0000","last_unstale":"2024-08-17T17:57:16.983305+0000","last_undegraded":"2024-08-17T17:57:16.983305+0000","last_fullsized":"2024-08-17T17:57:16.983305+0000","mapping_epoch":522,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":523,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:29:39.835110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437071+0000","last_change":"2024-08-17T17:56:13.705075+0000","last_active":"2024-08-17T17:57:14.437071+0000","last_peered":"2024-08-17T17:57:14.437071+0000","last_clean":"2024-08-17T17:57:14.437071+0000","last_became_active":"2024-08-17T17:56:13.704620+0000","last_became_peered":"2024-08-17T17:56:13.704620+0000","last_unstale":"2024-08-17T17:57:14.437071+0000","last_undegraded":"2024-08-17T17:57:14.437071+0000","last_fullsized":"2024-08-17T17:57:14.437071+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:23.036364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036728200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":635,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983427+0000","last_change":"2024-08-17T17:56:26.699475+0000","last_active":"2024-08-17T17:57:16.983427+0000","last_peered":"2024-08-17T17:57:16.983427+0000","last_clean":"2024-08-17T17:57:16.983427+0000","last_became_active":"2024-08-17T17:56:26.699152+0000","last_became_peered":"2024-08-17T17:56:26.699152+0000","last_unstale":"2024-08-17T17:57:16.983427+0000","last_undegraded":"2024-08-17T17:57:16.983427+0000","last_fullsized":"2024-08-17T17:57:16.983427+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:45:49.986225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":765,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.743224+0000","last_change":"2024-08-17T17:56:48.703032+0000","last_active":"2024-08-17T17:57:16.743224+0000","last_peered":"2024-08-17T17:57:16.743224+0000","last_clean":"2024-08-17T17:57:16.743224+0000","last_became_active":"2024-08-17T17:56:48.702627+0000","last_became_peered":"2024-08-17T17:56:48.702627+0000","last_unstale":"2024-08-17T17:57:16.743224+0000","last_undegraded":"2024-08-17T17:57:16.743224+0000","last_fullsized":"2024-08-17T17:57:16.743224+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:24.296226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":928,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983773+0000","last_change":"2024-08-17T17:51:20.639615+0000","last_active":"2024-08-17T17:57:16.983773+0000","last_peered":"2024-08-17T17:57:16.983773+0000","last_clean":"2024-08-17T17:57:16.983773+0000","last_became_active":"2024-08-17T17:51:20.639330+0000","last_became_peered":"2024-08-17T17:51:20.639330+0000","last_unstale":"2024-08-17T17:57:16.983773+0000","last_undegraded":"2024-08-17T17:57:16.983773+0000","last_fullsized":"2024-08-17T17:57:16.983773+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:41.751965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":914,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742368+0000","last_change":"2024-08-17T17:56:48.703034+0000","last_active":"2024-08-17T17:57:16.742368+0000","last_peered":"2024-08-17T17:57:16.742368+0000","last_clean":"2024-08-17T17:57:16.742368+0000","last_became_active":"2024-08-17T17:56:48.702701+0000","last_became_peered":"2024-08-17T17:56:48.702701+0000","last_unstale":"2024-08-17T17:57:16.742368+0000","last_undegraded":"2024-08-17T17:57:16.742368+0000","last_fullsized":"2024-08-17T17:57:16.742368+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:34.775405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":85,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983596+0000","last_change":"2024-08-17T17:55:43.568986+0000","last_active":"2024-08-17T17:57:16.983596+0000","last_peered":"2024-08-17T17:57:16.983596+0000","last_clean":"2024-08-17T17:57:16.983596+0000","last_became_active":"2024-08-17T17:55:43.561410+0000","last_became_peered":"2024-08-17T17:55:43.561410+0000","last_unstale":"2024-08-17T17:57:16.983596+0000","last_undegraded":"2024-08-17T17:57:16.983596+0000","last_fullsized":"2024-08-17T17:57:16.983596+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:48.268060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":541,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724413+0000","last_change":"2024-08-17T17:56:16.712692+0000","last_active":"2024-08-17T17:57:13.724413+0000","last_peered":"2024-08-17T17:57:13.724413+0000","last_clean":"2024-08-17T17:57:13.724413+0000","last_became_active":"2024-08-17T17:56:16.711843+0000","last_became_peered":"2024-08-17T17:56:16.711843+0000","last_unstale":"2024-08-17T17:57:13.724413+0000","last_undegraded":"2024-08-17T17:57:13.724413+0000","last_fullsized":"2024-08-17T17:57:13.724413+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:56:12.034566+0000","last_clean_scrub_stamp":"2024-08-17T16:56:12.034566+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:50.840470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029532199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722077+0000","last_change":"2024-08-17T17:51:20.651231+0000","last_active":"2024-08-17T17:57:13.722077+0000","last_peered":"2024-08-17T17:57:13.722077+0000","last_clean":"2024-08-17T17:57:13.722077+0000","last_became_active":"2024-08-17T17:51:20.650932+0000","last_became_peered":"2024-08-17T17:51:20.650932+0000","last_unstale":"2024-08-17T17:57:13.722077+0000","last_undegraded":"2024-08-17T17:57:13.722077+0000","last_fullsized":"2024-08-17T17:57:13.722077+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:55.890364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722327+0000","last_change":"2024-08-17T17:51:20.654600+0000","last_active":"2024-08-17T17:57:13.722327+0000","last_peered":"2024-08-17T17:57:13.722327+0000","last_clean":"2024-08-17T17:57:13.722327+0000","last_became_active":"2024-08-17T17:51:20.653501+0000","last_became_peered":"2024-08-17T17:51:20.653501+0000","last_unstale":"2024-08-17T17:57:13.722327+0000","last_undegraded":"2024-08-17T17:57:13.722327+0000","last_fullsized":"2024-08-17T17:57:13.722327+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:25:31.178218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":610,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722483+0000","last_change":"2024-08-17T17:56:13.712803+0000","last_active":"2024-08-17T17:57:13.722483+0000","last_peered":"2024-08-17T17:57:13.722483+0000","last_clean":"2024-08-17T17:57:13.722483+0000","last_became_active":"2024-08-17T17:56:13.702933+0000","last_became_peered":"2024-08-17T17:56:13.702933+0000","last_unstale":"2024-08-17T17:57:13.722483+0000","last_undegraded":"2024-08-17T17:57:13.722483+0000","last_fullsized":"2024-08-17T17:57:13.722483+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:51:31.008117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18976,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982974+0000","last_change":"2024-08-17T17:56:29.738340+0000","last_active":"2024-08-17T17:57:16.982974+0000","last_peered":"2024-08-17T17:57:16.982974+0000","last_clean":"2024-08-17T17:57:16.982974+0000","last_became_active":"2024-08-17T17:56:29.738036+0000","last_became_peered":"2024-08-17T17:56:29.738036+0000","last_unstale":"2024-08-17T17:57:16.982974+0000","last_undegraded":"2024-08-17T17:57:16.982974+0000","last_fullsized":"2024-08-17T17:57:16.982974+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:49:43.605252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":40,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.437044+0000","last_change":"2024-08-17T17:56:13.722966+0000","last_active":"2024-08-17T17:57:14.437044+0000","last_peered":"2024-08-17T17:57:14.437044+0000","last_clean":"2024-08-17T17:57:14.437044+0000","last_became_active":"2024-08-17T17:56:13.705408+0000","last_became_peered":"2024-08-17T17:56:13.705408+0000","last_unstale":"2024-08-17T17:57:14.437044+0000","last_undegraded":"2024-08-17T17:57:14.437044+0000","last_fullsized":"2024-08-17T17:57:14.437044+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:40.685691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":451,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409036+0000","last_change":"2024-08-17T17:56:13.701419+0000","last_active":"2024-08-17T17:57:14.409036+0000","last_peered":"2024-08-17T17:57:14.409036+0000","last_clean":"2024-08-17T17:57:14.409036+0000","last_became_active":"2024-08-17T17:56:13.701094+0000","last_became_peered":"2024-08-17T17:56:13.701094+0000","last_unstale":"2024-08-17T17:57:14.409036+0000","last_undegraded":"2024-08-17T17:57:14.409036+0000","last_fullsized":"2024-08-17T17:57:14.409036+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:03:36.515687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":899,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742460+0000","last_change":"2024-08-17T17:56:13.697799+0000","last_active":"2024-08-17T17:57:16.742460+0000","last_peered":"2024-08-17T17:57:16.742460+0000","last_clean":"2024-08-17T17:57:16.742460+0000","last_became_active":"2024-08-17T17:56:13.697513+0000","last_became_peered":"2024-08-17T17:56:13.697513+0000","last_unstale":"2024-08-17T17:57:16.742460+0000","last_undegraded":"2024-08-17T17:57:16.742460+0000","last_fullsized":"2024-08-17T17:57:16.742460+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:28:18.215551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":611,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983660+0000","last_change":"2024-08-17T17:56:26.700340+0000","last_active":"2024-08-17T17:57:16.983660+0000","last_peered":"2024-08-17T17:57:16.983660+0000","last_clean":"2024-08-17T17:57:16.983660+0000","last_became_active":"2024-08-17T17:56:26.699510+0000","last_became_peered":"2024-08-17T17:56:26.699510+0000","last_unstale":"2024-08-17T17:57:16.983660+0000","last_undegraded":"2024-08-17T17:57:16.983660+0000","last_fullsized":"2024-08-17T17:57:16.983660+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:23.617760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":886,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742366+0000","last_change":"2024-08-17T17:51:20.673866+0000","last_active":"2024-08-17T17:57:16.742366+0000","last_peered":"2024-08-17T17:57:16.742366+0000","last_clean":"2024-08-17T17:57:16.742366+0000","last_became_active":"2024-08-17T17:51:20.673619+0000","last_became_peered":"2024-08-17T17:51:20.673619+0000","last_unstale":"2024-08-17T17:57:16.742366+0000","last_undegraded":"2024-08-17T17:57:16.742366+0000","last_fullsized":"2024-08-17T17:57:16.742366+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:42.965391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":474,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.724425+0000","last_change":"2024-08-17T17:51:20.642743+0000","last_active":"2024-08-17T17:57:13.724425+0000","last_peered":"2024-08-17T17:57:13.724425+0000","last_clean":"2024-08-17T17:57:13.724425+0000","last_became_active":"2024-08-17T17:51:20.642129+0000","last_became_peered":"2024-08-17T17:51:20.642129+0000","last_unstale":"2024-08-17T17:57:13.724425+0000","last_undegraded":"2024-08-17T17:57:13.724425+0000","last_fullsized":"2024-08-17T17:57:13.724425+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:09.593211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":805,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742819+0000","last_change":"2024-08-17T17:56:48.923288+0000","last_active":"2024-08-17T17:57:16.742819+0000","last_peered":"2024-08-17T17:57:16.742819+0000","last_clean":"2024-08-17T17:57:16.742819+0000","last_became_active":"2024-08-17T17:56:48.922827+0000","last_became_peered":"2024-08-17T17:56:48.922827+0000","last_unstale":"2024-08-17T17:57:16.742819+0000","last_undegraded":"2024-08-17T17:57:16.742819+0000","last_fullsized":"2024-08-17T17:57:16.742819+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:17:17.043229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":608,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983337+0000","last_change":"2024-08-17T17:56:26.697321+0000","last_active":"2024-08-17T17:57:16.983337+0000","last_peered":"2024-08-17T17:57:16.983337+0000","last_clean":"2024-08-17T17:57:16.983337+0000","last_became_active":"2024-08-17T17:56:26.697042+0000","last_became_peered":"2024-08-17T17:56:26.697042+0000","last_unstale":"2024-08-17T17:57:16.983337+0000","last_undegraded":"2024-08-17T17:57:16.983337+0000","last_fullsized":"2024-08-17T17:57:16.983337+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:22.182085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":914,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982681+0000","last_change":"2024-08-17T17:12:40.341665+0000","last_active":"2024-08-17T17:57:16.982681+0000","last_peered":"2024-08-17T17:57:16.982681+0000","last_clean":"2024-08-17T17:57:16.982681+0000","last_became_active":"2024-08-17T17:12:40.341071+0000","last_became_peered":"2024-08-17T17:12:40.341071+0000","last_unstale":"2024-08-17T17:57:16.982681+0000","last_undegraded":"2024-08-17T17:57:16.982681+0000","last_fullsized":"2024-08-17T17:57:16.982681+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:45.499044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":690,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742846+0000","last_change":"2024-08-17T17:56:48.923470+0000","last_active":"2024-08-17T17:57:16.742846+0000","last_peered":"2024-08-17T17:57:16.742846+0000","last_clean":"2024-08-17T17:57:16.742846+0000","last_became_active":"2024-08-17T17:56:48.922775+0000","last_became_peered":"2024-08-17T17:56:48.922775+0000","last_unstale":"2024-08-17T17:57:16.742846+0000","last_undegraded":"2024-08-17T17:57:16.742846+0000","last_fullsized":"2024-08-17T17:57:16.742846+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:54.249979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":549,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722214+0000","last_change":"2024-08-17T17:56:16.714976+0000","last_active":"2024-08-17T17:57:13.722214+0000","last_peered":"2024-08-17T17:57:13.722214+0000","last_clean":"2024-08-17T17:57:13.722214+0000","last_became_active":"2024-08-17T17:56:16.714660+0000","last_became_peered":"2024-08-17T17:56:16.714660+0000","last_unstale":"2024-08-17T17:57:13.722214+0000","last_undegraded":"2024-08-17T17:57:13.722214+0000","last_fullsized":"2024-08-17T17:57:13.722214+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:29.904777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":54,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983339+0000","last_change":"2024-08-17T17:56:16.710915+0000","last_active":"2024-08-17T17:57:16.983339+0000","last_peered":"2024-08-17T17:57:16.983339+0000","last_clean":"2024-08-17T17:57:16.983339+0000","last_became_active":"2024-08-17T17:56:16.710641+0000","last_became_peered":"2024-08-17T17:56:16.710641+0000","last_unstale":"2024-08-17T17:57:16.983339+0000","last_undegraded":"2024-08-17T17:57:16.983339+0000","last_fullsized":"2024-08-17T17:57:16.983339+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:38.936824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":166,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.982950+0000","last_change":"2024-08-17T17:56:26.700363+0000","last_active":"2024-08-17T17:57:16.982950+0000","last_peered":"2024-08-17T17:57:16.982950+0000","last_clean":"2024-08-17T17:57:16.982950+0000","last_became_active":"2024-08-17T17:56:26.700094+0000","last_became_peered":"2024-08-17T17:56:26.700094+0000","last_unstale":"2024-08-17T17:57:16.982950+0000","last_undegraded":"2024-08-17T17:57:16.982950+0000","last_fullsized":"2024-08-17T17:57:16.982950+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:29.588348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":527,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722333+0000","last_change":"2024-08-17T17:33:20.284218+0000","last_active":"2024-08-17T17:57:13.722333+0000","last_peered":"2024-08-17T17:57:13.722333+0000","last_clean":"2024-08-17T17:57:13.722333+0000","last_became_active":"2024-08-17T17:33:20.283922+0000","last_became_peered":"2024-08-17T17:33:20.283922+0000","last_unstale":"2024-08-17T17:57:13.722333+0000","last_undegraded":"2024-08-17T17:57:13.722333+0000","last_fullsized":"2024-08-17T17:57:13.722333+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:25.118125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":195,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409032+0000","last_change":"2024-08-17T17:55:24.726566+0000","last_active":"2024-08-17T17:57:14.409032+0000","last_peered":"2024-08-17T17:57:14.409032+0000","last_clean":"2024-08-17T17:57:14.409032+0000","last_became_active":"2024-08-17T17:55:24.720789+0000","last_became_peered":"2024-08-17T17:55:24.720789+0000","last_unstale":"2024-08-17T17:57:14.409032+0000","last_undegraded":"2024-08-17T17:57:14.409032+0000","last_fullsized":"2024-08-17T17:57:14.409032+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:47.386540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":547,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.722076+0000","last_change":"2024-08-17T17:55:43.563475+0000","last_active":"2024-08-17T17:57:13.722076+0000","last_peered":"2024-08-17T17:57:13.722076+0000","last_clean":"2024-08-17T17:57:13.722076+0000","last_became_active":"2024-08-17T17:55:43.563172+0000","last_became_peered":"2024-08-17T17:55:43.563172+0000","last_unstale":"2024-08-17T17:57:13.722076+0000","last_undegraded":"2024-08-17T17:57:13.722076+0000","last_fullsized":"2024-08-17T17:57:13.722076+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:38:49.608894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":178,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.409303+0000","last_change":"2024-08-17T17:55:24.731028+0000","last_active":"2024-08-17T17:57:14.409303+0000","last_peered":"2024-08-17T17:57:14.409303+0000","last_clean":"2024-08-17T17:57:14.409303+0000","last_became_active":"2024-08-17T17:55:24.718573+0000","last_became_peered":"2024-08-17T17:55:24.718573+0000","last_unstale":"2024-08-17T17:57:14.409303+0000","last_undegraded":"2024-08-17T17:57:14.409303+0000","last_fullsized":"2024-08-17T17:57:14.409303+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:05.843256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":743,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.742734+0000","last_change":"2024-08-17T17:56:48.703219+0000","last_active":"2024-08-17T17:57:16.742734+0000","last_peered":"2024-08-17T17:57:16.742734+0000","last_clean":"2024-08-17T17:57:16.742734+0000","last_became_active":"2024-08-17T17:56:48.702897+0000","last_became_peered":"2024-08-17T17:56:48.702897+0000","last_unstale":"2024-08-17T17:57:16.742734+0000","last_undegraded":"2024-08-17T17:57:16.742734+0000","last_fullsized":"2024-08-17T17:57:16.742734+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:50.715495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":717,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.743117+0000","last_change":"2024-08-17T17:56:48.704289+0000","last_active":"2024-08-17T17:57:16.743117+0000","last_peered":"2024-08-17T17:57:16.743117+0000","last_clean":"2024-08-17T17:57:16.743117+0000","last_became_active":"2024-08-17T17:56:48.703857+0000","last_became_peered":"2024-08-17T17:56:48.703857+0000","last_unstale":"2024-08-17T17:57:16.743117+0000","last_undegraded":"2024-08-17T17:57:16.743117+0000","last_fullsized":"2024-08-17T17:57:16.743117+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:16.889242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2024-08-17T17:57:13.721954+0000","last_change":"2024-08-17T17:51:20.676098+0000","last_active":"2024-08-17T17:57:13.721954+0000","last_peered":"2024-08-17T17:57:13.721954+0000","last_clean":"2024-08-17T17:57:13.721954+0000","last_became_active":"2024-08-17T17:51:20.675786+0000","last_became_peered":"2024-08-17T17:51:20.675786+0000","last_unstale":"2024-08-17T17:57:13.721954+0000","last_undegraded":"2024-08-17T17:57:13.721954+0000","last_fullsized":"2024-08-17T17:57:13.721954+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:28.368695+0000","last_clean_scrub_stamp":"2024-08-17T16:57:28.368695+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:19.690923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":90,"reported_epoch":537,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.983831+0000","last_change":"2024-08-17T17:56:26.700463+0000","last_active":"2024-08-17T17:57:16.983831+0000","last_peered":"2024-08-17T17:57:16.983831+0000","last_clean":"2024-08-17T17:57:16.983831+0000","last_became_active":"2024-08-17T17:56:26.699579+0000","last_became_peered":"2024-08-17T17:56:26.699579+0000","last_unstale":"2024-08-17T17:57:16.983831+0000","last_undegraded":"2024-08-17T17:57:16.983831+0000","last_fullsized":"2024-08-17T17:57:16.983831+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:57:27.362295+0000","last_clean_scrub_stamp":"2024-08-17T16:57:27.362295+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:08:02.259413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":19518,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44218,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":511,"seq":2194728288283,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":368140,"kb_used_data":7492,"kb_used_omap":15,"kb_used_meta":360624,"kb_avail":93377012,"statfs":{"total":95995035648,"available":95618060288,"internally_reserved":0,"allocated":7671808,"data_stored":4831000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15573,"internal_metadata":369279787},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":453,"seq":1945620185171,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":386184,"kb_used_data":7484,"kb_used_omap":13,"kb_used_meta":378674,"kb_avail":93358968,"statfs":{"total":95995035648,"available":95599583232,"internally_reserved":0,"allocated":7663616,"data_stored":4830857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13340,"internal_metadata":387763172},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":199,"seq":854698492345,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":975796,"kb_used_data":6888,"kb_used_omap":7,"kb_used_meta":968888,"kb_avail":92769356,"statfs":{"total":95995035648,"available":94995820544,"internally_reserved":0,"allocated":7053312,"data_stored":4235273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7859,"internal_metadata":992141645},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986937,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2092036,"kb_used_data":6904,"kb_used_omap":1,"kb_used_meta":2085118,"kb_avail":91653116,"statfs":{"total":95995035648,"available":93852790784,"internally_reserved":0,"allocated":7069696,"data_stored":4240297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":2135161176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":146,"seq":627065225765,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1354032,"kb_used_data":6884,"kb_used_omap":4,"kb_used_meta":1347131,"kb_avail":92391120,"statfs":{"total":95995035648,"available":94608506880,"internally_reserved":0,"allocated":7049216,"data_stored":4235501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4672,"internal_metadata":1379462592},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":120,"seq":515396076090,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":701432,"kb_used_data":6892,"kb_used_omap":6,"kb_used_meta":694521,"kb_avail":93043720,"statfs":{"total":95995035648,"available":95276769280,"internally_reserved":0,"allocated":7057408,"data_stored":4236312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6725,"internal_metadata":711189947},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":106,"seq":455266533982,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516528,"kb_used_data":6884,"kb_used_omap":5,"kb_used_meta":1509626,"kb_avail":92228624,"statfs":{"total":95995035648,"available":94442110976,"internally_reserved":0,"allocated":7049216,"data_stored":4234299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":1545857264},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":89,"seq":382252089959,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1229244,"kb_used_data":7472,"kb_used_omap":5,"kb_used_meta":1221754,"kb_avail":92515908,"statfs":{"total":95995035648,"available":94736289792,"internally_reserved":0,"allocated":7651328,"data_stored":4826660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6085,"internal_metadata":1251076155},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4381,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4200,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2174,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2968,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6155,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11636,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12704,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:58:09.942 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:58:09.943 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi142 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T17:58:10.119 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=84be659ccdf441c0adcacde1b0abd720 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi142 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T17:58:10.123 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-08-17T17:58:10.133 INFO:tasks.cephadm:Teardown begin 2024-08-17T17:58:10.133 DEBUG:teuthology.orchestra.run.smithi142:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:58:10.150 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:09 smithi152 ceph-mon[93456]: from='client.48809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:10.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:58:10.233 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-08-17T17:58:10.233 DEBUG:teuthology.orchestra.run.smithi142:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:58:10.248 DEBUG:teuthology.orchestra.run.smithi152:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:58:10.266 INFO:tasks.cephadm:Stopping all daemons... 2024-08-17T17:58:10.266 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-08-17T17:58:10.266 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a 2024-08-17T17:58:10.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:10.612 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:10 smithi142 systemd[1]: Stopping Ceph mon.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:58:10.610+0000 7f3d70737640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:10.872 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a[103591]: 2024-08-17T17:58:10.610+0000 7f3d70737640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-08-17T17:58:10.873 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:10 smithi142 podman[467610]: 2024-08-17 17:58:10.662291051 +0000 UTC m=+0.170694121 container died 3a2a93ab0fbb9f60b0e84ff7e882aa4b32934a372b9d9a772f45dcbc4738eb53 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:58:10.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:10] ENGINE Bus STOPPING 2024-08-17T17:58:10.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:58:10.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:10] ENGINE Bus STOPPED 2024-08-17T17:58:10.873 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:10] ENGINE Bus STARTING 2024-08-17T17:58:11.218 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:10] ENGINE Serving on http://:::9283 2024-08-17T17:58:11.218 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:10 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:10] ENGINE Bus STARTED 2024-08-17T17:58:11.478 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:11 smithi142 podman[467610]: 2024-08-17 17:58:11.217890713 +0000 UTC m=+0.726293775 container cleanup 3a2a93ab0fbb9f60b0e84ff7e882aa4b32934a372b9d9a772f45dcbc4738eb53 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:58:11.478 INFO:journalctl@ceph.mon.a.smithi142.stdout:Aug 17 17:58:11 smithi142 bash[467610]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:58:11.600 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.a.service' 2024-08-17T17:58:11.652 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:11.653 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-08-17T17:58:11.653 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-08-17T17:58:11.653 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.c 2024-08-17T17:58:11.771 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:11 smithi142 systemd[1]: Stopping Ceph mon.c for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:12.093 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: ::ffff:172.21.15.152 - - [17/Aug/2024:17:58:11] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-17T17:58:12.093 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:11] ENGINE Bus STOPPING 2024-08-17T17:58:12.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c[106690]: 2024-08-17T17:58:11.949+0000 7fa451831640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:12.094 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:11 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c[106690]: 2024-08-17T17:58:11.949+0000 7fa451831640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-08-17T17:58:12.372 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:12 smithi142 podman[467753]: 2024-08-17 17:58:12.092731375 +0000 UTC m=+0.247866198 container died e6fccaa16a72a0c24bdb8755fd86cda1f001212bbf5c03e2838dbcc5ea2ee3ac (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:58:12.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:58:12.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:12] ENGINE Bus STOPPED 2024-08-17T17:58:12.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:12] ENGINE Bus STARTING 2024-08-17T17:58:12.732 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:12] ENGINE Serving on http://:::9283 2024-08-17T17:58:12.733 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:12 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: [17/Aug/2024:17:58:12] ENGINE Bus STARTED 2024-08-17T17:58:12.733 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:12 smithi142 podman[467753]: 2024-08-17 17:58:12.497254084 +0000 UTC m=+0.652388904 container cleanup e6fccaa16a72a0c24bdb8755fd86cda1f001212bbf5c03e2838dbcc5ea2ee3ac (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T17:58:12.733 INFO:journalctl@ceph.mon.c.smithi142.stdout:Aug 17 17:58:12 smithi142 bash[467753]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:58:12.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.c.service' 2024-08-17T17:58:12.912 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:12.913 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-08-17T17:58:12.913 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-08-17T17:58:12.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.b 2024-08-17T17:58:13.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:12 smithi152 systemd[1]: Stopping Ceph mon.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:13.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:13 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b[93432]: 2024-08-17T17:58:13.506+0000 7fc3ebdea640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:13.783 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:13 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b[93432]: 2024-08-17T17:58:13.506+0000 7fc3ebdea640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-08-17T17:58:14.059 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:13 smithi152 podman[305872]: 2024-08-17 17:58:13.78301788 +0000 UTC m=+0.660062619 container died a690c34b9332eafa283ff841aa10628fe536e19f9e81d2879184cac28b61e574 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-08-17T17:58:14.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:14 smithi152 podman[305872]: 2024-08-17 17:58:14.28907064 +0000 UTC m=+1.166115420 container cleanup a690c34b9332eafa283ff841aa10628fe536e19f9e81d2879184cac28b61e574 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:58:14.323 INFO:journalctl@ceph.mon.b.smithi152.stdout:Aug 17 17:58:14 smithi152 bash[305872]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:58:14.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mon.b.service' 2024-08-17T17:58:14.744 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:14.745 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-08-17T17:58:14.745 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-08-17T17:58:14.745 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y 2024-08-17T17:58:15.089 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:14 smithi142 systemd[1]: Stopping Ceph mgr.y for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:15.372 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:15 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:58:15.086+0000 7f25fefab640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:15.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:15 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y[93143]: 2024-08-17T17:58:15.086+0000 7f25fefab640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-08-17T17:58:15.373 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:15 smithi142 podman[467896]: 2024-08-17 17:58:15.128924773 +0000 UTC m=+0.177811571 container died 323039997e228bb34547f4fda708dc39c28eb7bcd3d831b8cb6484a68167d9bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0) 2024-08-17T17:58:15.871 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:15 smithi142 podman[467896]: 2024-08-17 17:58:15.593939742 +0000 UTC m=+0.642826539 container cleanup 323039997e228bb34547f4fda708dc39c28eb7bcd3d831b8cb6484a68167d9bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD) 2024-08-17T17:58:15.871 INFO:journalctl@ceph.mgr.y.smithi142.stdout:Aug 17 17:58:15 smithi142 bash[467896]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-y 2024-08-17T17:58:16.041 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.y.service' 2024-08-17T17:58:16.076 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:16.076 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-08-17T17:58:16.076 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-08-17T17:58:16.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x 2024-08-17T17:58:16.448 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:58:16 smithi152 systemd[1]: Stopping Ceph mgr.x for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:16.726 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:58:16 smithi152 podman[306004]: 2024-08-17 17:58:16.448072375 +0000 UTC m=+0.165507761 container died bc654892e833d9709220e95046106a9fabceb4a247720599240482809b1e0403 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T17:58:17.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:58:16 smithi152 podman[306004]: 2024-08-17 17:58:16.903555308 +0000 UTC m=+0.620990692 container cleanup bc654892e833d9709220e95046106a9fabceb4a247720599240482809b1e0403 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-08-17T17:58:17.073 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Aug 17 17:58:16 smithi152 bash[306004]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:58:17.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@mgr.x.service' 2024-08-17T17:58:17.406 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:17.406 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-08-17T17:58:17.406 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-17T17:58:17.407 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.0 2024-08-17T17:58:17.742 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:17 smithi142 systemd[1]: Stopping Ceph osd.0 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:18.122 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[116420]: 2024-08-17T17:58:17.741+0000 7f65c74cc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:18.122 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[116420]: 2024-08-17T17:58:17.741+0000 7f65c74cc640 -1 osd.0 537 *** Got signal Terminated *** 2024-08-17T17:58:18.123 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:17 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0[116420]: 2024-08-17T17:58:17.741+0000 7f65c74cc640 -1 osd.0 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:58:20.573 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"bcf463ee-5cb8-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:23.786 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:23 smithi142 podman[468026]: 2024-08-17 17:58:23.447672477 +0000 UTC m=+5.841124158 container died 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-08-17T17:58:24.122 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:23 smithi142 podman[468026]: 2024-08-17 17:58:23.965632193 +0000 UTC m=+6.359083880 container cleanup 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:58:24.122 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:23 smithi142 bash[468026]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:58:24.622 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:24 smithi142 podman[468049]: 2024-08-17 17:58:24.284249446 +0000 UTC m=+0.827476564 container remove 72a9d7f0d685803ea1f9c53795c1fb2d5536cb7931057d8f06a97ca40afeeaca (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:58:25.080 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:24 smithi142 podman[468148]: 2024-08-17 17:58:24.659193968 +0000 UTC m=+0.017560294 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:58:25.080 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:24 smithi142 podman[468148]: 2024-08-17 17:58:24.785240254 +0000 UTC m=+0.143606579 container create d5ed971b99e44e88a51c1c93eff944bd519dc1031fe6fd06f59c8a7ff0bdc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:58:25.080 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:25 smithi142 podman[468148]: 2024-08-17 17:58:25.079531618 +0000 UTC m=+0.437897950 container init d5ed971b99e44e88a51c1c93eff944bd519dc1031fe6fd06f59c8a7ff0bdc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:58:25.372 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:25 smithi142 podman[468148]: 2024-08-17 17:58:25.084495506 +0000 UTC m=+0.442861831 container start d5ed971b99e44e88a51c1c93eff944bd519dc1031fe6fd06f59c8a7ff0bdc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:58:25.372 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:25 smithi142 podman[468148]: 2024-08-17 17:58:25.246273827 +0000 UTC m=+0.604640248 container attach d5ed971b99e44e88a51c1c93eff944bd519dc1031fe6fd06f59c8a7ff0bdc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, ceph=True) 2024-08-17T17:58:26.122 INFO:journalctl@ceph.osd.0.smithi142.stdout:Aug 17 17:58:25 smithi142 podman[468148]: 2024-08-17 17:58:25.858685582 +0000 UTC m=+1.217051908 container died d5ed971b99e44e88a51c1c93eff944bd519dc1031fe6fd06f59c8a7ff0bdc412 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-08-17T17:58:26.628 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.0.service' 2024-08-17T17:58:26.669 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:26.669 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-17T17:58:26.669 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-08-17T17:58:26.669 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.1 2024-08-17T17:58:26.872 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:26 smithi142 systemd[1]: Stopping Ceph osd.1 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:27.372 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[123031]: 2024-08-17T17:58:27.005+0000 7f1a766be640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:27.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[123031]: 2024-08-17T17:58:27.005+0000 7f1a766be640 -1 osd.1 537 *** Got signal Terminated *** 2024-08-17T17:58:27.373 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:27 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1[123031]: 2024-08-17T17:58:27.005+0000 7f1a766be640 -1 osd.1 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:58:32.771 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:32 smithi142 podman[468280]: 2024-08-17 17:58:32.494043503 +0000 UTC m=+5.597187167 container died d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS) 2024-08-17T17:58:33.122 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:32 smithi142 podman[468280]: 2024-08-17 17:58:32.979245039 +0000 UTC m=+6.082388763 container cleanup d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:58:33.122 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:32 smithi142 bash[468280]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:58:33.622 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:33 smithi142 podman[468300]: 2024-08-17 17:58:33.36437597 +0000 UTC m=+0.865612905 container remove d435dd533791457dfca3c5dcbbdeee7f9828cb2d905a0e3ac50f0b1ce2ba8372 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T17:58:34.043 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:33 smithi142 podman[468392]: 2024-08-17 17:58:33.654631698 +0000 UTC m=+0.017715641 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:58:34.043 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:33 smithi142 podman[468392]: 2024-08-17 17:58:33.75701947 +0000 UTC m=+0.120103445 container create e22e62d519c98e7aec852083e9e54e1af7847ce6fb389b45a07233d326eb9baf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-17T17:58:34.043 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:33 smithi142 podman[468392]: 2024-08-17 17:58:33.94263872 +0000 UTC m=+0.305722650 container init e22e62d519c98e7aec852083e9e54e1af7847ce6fb389b45a07233d326eb9baf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-08-17T17:58:34.044 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:33 smithi142 podman[468392]: 2024-08-17 17:58:33.947875886 +0000 UTC m=+0.310959819 container start e22e62d519c98e7aec852083e9e54e1af7847ce6fb389b45a07233d326eb9baf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-08-17T17:58:34.044 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:34 smithi142 podman[468392]: 2024-08-17 17:58:34.042497701 +0000 UTC m=+0.405581632 container attach e22e62d519c98e7aec852083e9e54e1af7847ce6fb389b45a07233d326eb9baf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T17:58:34.372 INFO:journalctl@ceph.osd.1.smithi142.stdout:Aug 17 17:58:34 smithi142 podman[468392]: 2024-08-17 17:58:34.138955503 +0000 UTC m=+0.502039450 container died e22e62d519c98e7aec852083e9e54e1af7847ce6fb389b45a07233d326eb9baf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True) 2024-08-17T17:58:34.829 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.1.service' 2024-08-17T17:58:34.870 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:34.870 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-08-17T17:58:34.870 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-17T17:58:34.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.2 2024-08-17T17:58:35.122 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:34 smithi142 systemd[1]: Stopping Ceph osd.2 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:35.622 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:35 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[132671]: 2024-08-17T17:58:35.251+0000 7ffbf2159640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:35.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:35 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[132671]: 2024-08-17T17:58:35.251+0000 7ffbf2159640 -1 osd.2 537 *** Got signal Terminated *** 2024-08-17T17:58:35.623 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:35 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2[132671]: 2024-08-17T17:58:35.251+0000 7ffbf2159640 -1 osd.2 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:58:39.823 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:39.515Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.142:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.142:8765: connect: connection refused" 2024-08-17T17:58:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:39.517Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.142:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.142:8765: connect: connection refused" 2024-08-17T17:58:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:39.517Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.142:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.142:8765: connect: connection refused" 2024-08-17T17:58:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:39.519Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.142:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.142:8765: connect: connection refused" 2024-08-17T17:58:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:39.519Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.142:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.142:8765: connect: connection refused" 2024-08-17T17:58:39.824 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:58:39 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:58:39.519Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.142:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.142:8765: connect: connection refused" 2024-08-17T17:58:41.041 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:40 smithi142 podman[468520]: 2024-08-17 17:58:40.758269082 +0000 UTC m=+5.657687601 container died 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:58:41.372 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:41 smithi142 podman[468520]: 2024-08-17 17:58:41.182195342 +0000 UTC m=+6.081613902 container cleanup 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe) 2024-08-17T17:58:41.372 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:41 smithi142 bash[468520]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:58:41.804 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:41 smithi142 podman[468545]: 2024-08-17 17:58:41.400169542 +0000 UTC m=+0.645315628 container remove 9fd11a07e9485102520bd15790b7b9286258bfa461338ee2a11c0db05f9a86de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:58:42.120 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:41 smithi142 podman[468638]: 2024-08-17 17:58:41.706359378 +0000 UTC m=+0.017635358 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:58:42.121 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:41 smithi142 podman[468638]: 2024-08-17 17:58:41.817789278 +0000 UTC m=+0.129065249 container create 8199c42a3430d2f98eb652dc5d8bc7a6d11fc9e7958506ba40b3c60a09162ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:58:42.121 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:42 smithi142 podman[468638]: 2024-08-17 17:58:42.028581498 +0000 UTC m=+0.339857470 container init 8199c42a3430d2f98eb652dc5d8bc7a6d11fc9e7958506ba40b3c60a09162ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:58:42.121 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:42 smithi142 podman[468638]: 2024-08-17 17:58:42.034509942 +0000 UTC m=+0.345785913 container start 8199c42a3430d2f98eb652dc5d8bc7a6d11fc9e7958506ba40b3c60a09162ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-17T17:58:42.121 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:42 smithi142 podman[468638]: 2024-08-17 17:58:42.120304615 +0000 UTC m=+0.431580590 container attach 8199c42a3430d2f98eb652dc5d8bc7a6d11fc9e7958506ba40b3c60a09162ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-17T17:58:42.372 INFO:journalctl@ceph.osd.2.smithi142.stdout:Aug 17 17:58:42 smithi142 podman[468638]: 2024-08-17 17:58:42.219717359 +0000 UTC m=+0.530993344 container died 8199c42a3430d2f98eb652dc5d8bc7a6d11fc9e7958506ba40b3c60a09162ec6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T17:58:42.877 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.2.service' 2024-08-17T17:58:42.917 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:42.917 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-17T17:58:42.918 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-17T17:58:42.918 DEBUG:teuthology.orchestra.run.smithi142:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.3 2024-08-17T17:58:43.122 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:42 smithi142 systemd[1]: Stopping Ceph osd.3 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:43.622 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:58:43.253+0000 7fda6091c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:43.622 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:58:43.253+0000 7fda6091c640 -1 osd.3 537 *** Got signal Terminated *** 2024-08-17T17:58:43.623 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:43 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:58:43.253+0000 7fda6091c640 -1 osd.3 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:58:44.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:44.048+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:44.074 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:43 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:43.815+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:44.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:44.111+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:45.159 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:45.049+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:45.160 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:44 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:44.835+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:45.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:45.158+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:46.072 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:46.018+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:46.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:45 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:45.818+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:46.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:46.169+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:47.205 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:46.969+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:47.206 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:46 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:46.843+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:47.483 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:47 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:47.204+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:47.815 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:47 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:47.482+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:48.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:47 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:47.994+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:48.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:47 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:47.814+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:48.372 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:47 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3[141126]: 2024-08-17T17:58:47.974+0000 7fda58f29640 -1 osd.3 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:22.014698+0000 front 2024-08-17T17:58:22.014824+0000 (oldest deadline 2024-08-17T17:58:47.914933+0000) 2024-08-17T17:58:48.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:48 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:48.451+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:48.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:48 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:48.222+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:49.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:48 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:48.970+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:49.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:48 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:48.795+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:49.230 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:48 smithi142 podman[468767]: 2024-08-17 17:58:48.881632365 +0000 UTC m=+5.737370172 container died e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:58:49.490 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:49 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:49.219+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:49.622 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:49 smithi142 podman[468767]: 2024-08-17 17:58:49.426190139 +0000 UTC m=+6.281927935 container cleanup e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:58:49.622 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:49 smithi142 bash[468767]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:58:49.785 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:49 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:49.489+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:50.048 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:49 smithi142 podman[468794]: 2024-08-17 17:58:49.6964317 +0000 UTC m=+0.749460860 container remove e93ead24f7ccdd3fea178d68663910da2a31a26632fc81531f15b4802eb09975 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3, ceph=True, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:58:50.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:49 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:49.970+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:50.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:49 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:49.783+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:50.341 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:50 smithi142 podman[468884]: 2024-08-17 17:58:49.950723563 +0000 UTC m=+0.017283724 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:58:50.341 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:50 smithi142 podman[468884]: 2024-08-17 17:58:50.055400198 +0000 UTC m=+0.121960395 container create 0e49818af349a32eff8423ca44e2da59982e740638ea8cda1113c7458556c73a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-08-17T17:58:50.341 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:50 smithi142 podman[468884]: 2024-08-17 17:58:50.257444968 +0000 UTC m=+0.324005134 container init 0e49818af349a32eff8423ca44e2da59982e740638ea8cda1113c7458556c73a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:58:50.341 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:50 smithi142 podman[468884]: 2024-08-17 17:58:50.263068882 +0000 UTC m=+0.329629037 container start 0e49818af349a32eff8423ca44e2da59982e740638ea8cda1113c7458556c73a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:58:50.341 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:50 smithi142 podman[468884]: 2024-08-17 17:58:50.340612738 +0000 UTC m=+0.407172895 container attach 0e49818af349a32eff8423ca44e2da59982e740638ea8cda1113c7458556c73a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:58:50.572 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:50.460+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:50.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:50.253+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:50.622 INFO:journalctl@ceph.osd.3.smithi142.stdout:Aug 17 17:58:50 smithi142 podman[468884]: 2024-08-17 17:58:50.449438113 +0000 UTC m=+0.515998275 container died 0e49818af349a32eff8423ca44e2da59982e740638ea8cda1113c7458556c73a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe) 2024-08-17T17:58:51.045 DEBUG:teuthology.orchestra.run.smithi142:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.3.service' 2024-08-17T17:58:51.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:50.996+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:51.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:50 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:50.762+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:51.083 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:51.083 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-17T17:58:51.083 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-08-17T17:58:51.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.4 2024-08-17T17:58:51.391 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:51 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:51.207+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:51.391 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:51 smithi152 systemd[1]: Stopping Ceph osd.4 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:51.391 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:51 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:51.390+0000 7f6241afe640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:51.392 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:51 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:51.390+0000 7f6241afe640 -1 osd.4 537 *** Got signal Terminated *** 2024-08-17T17:58:51.758 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:51 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:51.390+0000 7f6241afe640 -1 osd.4 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:58:51.759 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:51 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:51.475+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:52.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:52.004+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:52.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:51 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:51.757+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:52.439 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:52.175+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:52.759 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:52.438+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:53.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:52.758+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:53.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:52.957+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:53.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:52 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:52.957+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:27.011338+0000 front 2024-08-17T17:58:27.011206+0000 (oldest deadline 2024-08-17T17:58:52.911112+0000) 2024-08-17T17:58:53.486 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:53.186+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:53.487 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:53.186+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:53.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:53.729+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:53.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:53.485+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:54.236 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:53.932+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:54.237 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:53 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:53.932+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:27.011338+0000 front 2024-08-17T17:58:27.011206+0000 (oldest deadline 2024-08-17T17:58:52.911112+0000) 2024-08-17T17:58:54.489 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:54.235+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:54.490 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:54.235+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:54.763 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:54.488+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:55.072 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:54.762+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:55.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:54.945+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:55.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:54 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:54.945+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:27.011338+0000 front 2024-08-17T17:58:27.011206+0000 (oldest deadline 2024-08-17T17:58:52.911112+0000) 2024-08-17T17:58:55.572 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:55.282+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:55.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:55.282+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:55.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:55.453+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:56.072 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:55.752+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:56.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:55.936+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:19.509642+0000 front 2024-08-17T17:58:19.509555+0000 (oldest deadline 2024-08-17T17:58:43.609547+0000) 2024-08-17T17:58:56.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:55 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4[40212]: 2024-08-17T17:58:55.936+0000 7f623f118640 -1 osd.4 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:27.011338+0000 front 2024-08-17T17:58:27.011206+0000 (oldest deadline 2024-08-17T17:58:52.911112+0000) 2024-08-17T17:58:56.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:56.247+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:56.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:56.247+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:56.574 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:56.463+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:56.574 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:56.463+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:58:57.073 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:56 smithi152 podman[306139]: 2024-08-17 17:58:56.921726977 +0000 UTC m=+5.636271735 container died 7a1cab3a1049250f72a5584d8d4d175fea2cce1f854c9c04f675863c0ea2c1a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, ceph=True) 2024-08-17T17:58:57.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:56.717+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:57.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:56 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:56.717+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:58:57.499 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:57 smithi152 podman[306139]: 2024-08-17 17:58:57.365591321 +0000 UTC m=+6.080136072 container cleanup 7a1cab3a1049250f72a5584d8d4d175fea2cce1f854c9c04f675863c0ea2c1a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-08-17T17:58:57.500 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:57 smithi152 bash[306139]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4 2024-08-17T17:58:57.500 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:57 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:57.251+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:57.500 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:57 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:57.251+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:57.822 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:57 smithi152 podman[306164]: 2024-08-17 17:58:57.633705045 +0000 UTC m=+0.713051493 container remove 7a1cab3a1049250f72a5584d8d4d175fea2cce1f854c9c04f675863c0ea2c1a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T17:58:57.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:57 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:57.682+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:57.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:57 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:57.682+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:58:57.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:57 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:57.498+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:57.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:57 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:57.498+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:58:58.272 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:57 smithi152 podman[306255]: 2024-08-17 17:58:57.985044664 +0000 UTC m=+0.110577759 container create 2049a3c5afe43c691be51ffc0e0c531e75994009d9b5bfa4b34ff4450fe44a88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:58:58.272 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:57 smithi152 podman[306255]: 2024-08-17 17:58:57.89219885 +0000 UTC m=+0.017731956 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:58:58.537 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:58.271+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:58.537 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:58.271+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:58.537 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:58 smithi152 podman[306255]: 2024-08-17 17:58:58.296244318 +0000 UTC m=+0.421777427 container init 2049a3c5afe43c691be51ffc0e0c531e75994009d9b5bfa4b34ff4450fe44a88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-08-17T17:58:58.537 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:58 smithi152 podman[306255]: 2024-08-17 17:58:58.301113062 +0000 UTC m=+0.426646169 container start 2049a3c5afe43c691be51ffc0e0c531e75994009d9b5bfa4b34ff4450fe44a88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:58:58.538 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:58 smithi152 podman[306255]: 2024-08-17 17:58:58.396210024 +0000 UTC m=+0.521743128 container attach 2049a3c5afe43c691be51ffc0e0c531e75994009d9b5bfa4b34ff4450fe44a88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:58:58.538 INFO:journalctl@ceph.osd.4.smithi152.stdout:Aug 17 17:58:58 smithi152 podman[306255]: 2024-08-17 17:58:58.520093631 +0000 UTC m=+0.645626733 container died 2049a3c5afe43c691be51ffc0e0c531e75994009d9b5bfa4b34ff4450fe44a88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T17:58:58.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:58.646+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:58.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:58.646+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:58:58.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:58.536+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:58.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:58 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:58.536+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:58:59.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.4.service' 2024-08-17T17:58:59.166 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:58:59.166 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-08-17T17:58:59.166 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-17T17:58:59.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:58:59.505 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:59.253+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:58:59.506 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:58:59.253+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:58:59.506 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:59 smithi152 systemd[1]: Stopping Ceph osd.5 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:58:59.506 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:59.504+0000 7f42c8752640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:58:59.506 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:59.504+0000 7f42c8752640 -1 osd.5 537 *** Got signal Terminated *** 2024-08-17T17:58:59.506 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:59.504+0000 7f42c8752640 -1 osd.5 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:58:59.822 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:59.650+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:58:59.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:58:59.650+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:58:59.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:59.524+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:58:59.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:58:59 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:58:59.524+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:00.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:00.258+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:00.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:00.258+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:00.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:00.505+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:00.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:00.505+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:01.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:00.675+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:59:01.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:00 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:00.675+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:59:01.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:01.510+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:01.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:01.510+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:01.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:01.265+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:01.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:01.265+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:01.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:01.265+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:02.072 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:01.669+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:59:02.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:01 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:01.669+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:59:02.513 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:02.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:02.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:02.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:02.514 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:02.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:02.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:02.676+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:59:02.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:02.676+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:59:02.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:02.676+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:39.985074+0000 front 2024-08-17T17:58:39.985028+0000 (oldest deadline 2024-08-17T17:59:02.284922+0000) 2024-08-17T17:59:02.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:02.512+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:02.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:02.512+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:02.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:02 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:02.512+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:03.557 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:03.252+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:03.558 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:03.252+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:03.558 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:03.252+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:03.822 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:03.635+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:20.882356+0000 front 2024-08-17T17:58:20.882200+0000 (oldest deadline 2024-08-17T17:58:43.782050+0000) 2024-08-17T17:59:03.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:03.635+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:31.383473+0000 front 2024-08-17T17:58:31.383293+0000 (oldest deadline 2024-08-17T17:58:56.683265+0000) 2024-08-17T17:59:03.823 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5[124444]: 2024-08-17T17:59:03.635+0000 7f42c3d68640 -1 osd.5 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:39.985074+0000 front 2024-08-17T17:58:39.985028+0000 (oldest deadline 2024-08-17T17:59:02.284922+0000) 2024-08-17T17:59:03.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:03.556+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:03.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:03.556+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:03.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:03 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:03.556+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:04.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:04.222+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:04.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:04.222+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:04.512 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:04.222+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:04.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:04.511+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:04.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:04.511+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:04.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:04 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:04.511+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:05.246 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:04 smithi152 podman[306382]: 2024-08-17 17:59:04.976579789 +0000 UTC m=+5.580615157 container died 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-08-17T17:59:05.519 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:05 smithi152 podman[306382]: 2024-08-17 17:59:05.336448129 +0000 UTC m=+5.940483498 container cleanup 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:59:05.519 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:05 smithi152 bash[306382]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5 2024-08-17T17:59:05.520 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:05.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:05.520 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:05.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:05.520 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:05.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:05.826 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:05 smithi152 podman[306406]: 2024-08-17 17:59:05.596451006 +0000 UTC m=+0.618588753 container remove 2972017061c11a3b55fa7b4f541e6a0f1e486c052ef2204fb3db99d47c8a6525 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2) 2024-08-17T17:59:05.827 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:05.518+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:05.827 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:05.518+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:05.828 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:05 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:05.518+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:06.230 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:05 smithi152 podman[306497]: 2024-08-17 17:59:05.955911356 +0000 UTC m=+0.109127836 container create d8cce933a72003047ae0d070ae22820d88273514fc2b900e7d8e27bca240ccb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:59:06.231 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:05 smithi152 podman[306497]: 2024-08-17 17:59:05.864128702 +0000 UTC m=+0.017345194 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:59:06.231 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:06 smithi152 podman[306497]: 2024-08-17 17:59:06.158384619 +0000 UTC m=+0.311601097 container init d8cce933a72003047ae0d070ae22820d88273514fc2b900e7d8e27bca240ccb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, io.buildah.version=1.36.0) 2024-08-17T17:59:06.231 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:06 smithi152 podman[306497]: 2024-08-17 17:59:06.163414025 +0000 UTC m=+0.316630511 container start d8cce933a72003047ae0d070ae22820d88273514fc2b900e7d8e27bca240ccb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:59:06.488 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:06 smithi152 podman[306497]: 2024-08-17 17:59:06.26010028 +0000 UTC m=+0.413316772 container attach d8cce933a72003047ae0d070ae22820d88273514fc2b900e7d8e27bca240ccb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:59:06.489 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:06 smithi152 podman[306497]: 2024-08-17 17:59:06.340851108 +0000 UTC m=+0.494067597 container died d8cce933a72003047ae0d070ae22820d88273514fc2b900e7d8e27bca240ccb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T17:59:06.489 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:06.228+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:06.489 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:06.228+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:06.489 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:06.228+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:06.820 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:06.487+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:06.821 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:06.487+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:06.821 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:06 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:06.487+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:07.073 INFO:journalctl@ceph.osd.5.smithi152.stdout:Aug 17 17:59:07 smithi152 podman[306540]: 2024-08-17 17:59:07.045424883 +0000 UTC m=+0.695831596 container remove d8cce933a72003047ae0d070ae22820d88273514fc2b900e7d8e27bca240ccb0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T17:59:07.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.5.service' 2024-08-17T17:59:07.119 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:59:07.119 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-17T17:59:07.120 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T17:59:07.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:59:07.451 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 systemd[1]: Stopping Ceph osd.6 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:07.451 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:07.256+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:07.451 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:07.256+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:07.451 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:07.256+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:07.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:07.450+0000 7f727677a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:59:07.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:07.450+0000 7f727677a640 -1 osd.6 537 *** Got signal Terminated *** 2024-08-17T17:59:07.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:07.450+0000 7f727677a640 -1 osd.6 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:59:07.823 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:07.458+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:07.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:07.458+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:07.824 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:07 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:07.458+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:08.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:08.466+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:08.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:08.466+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:08.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:08.466+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:08.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:08.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:08.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:08.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:08.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:08 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:08.245+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:09.475 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:09.203+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:09.475 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:09.203+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:09.475 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:09.203+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:09.476 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:09.429+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:09.476 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:09.429+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:09.476 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:09 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:09.429+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:10.494 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:10.405+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:10.494 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:10.405+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:10.494 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:10.405+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:10.495 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:10.235+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:10.495 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:10.235+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:10.495 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:10 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:10.235+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:11.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:11.449+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.406828+0000 front 2024-08-17T17:58:21.406652+0000 (oldest deadline 2024-08-17T17:58:46.706683+0000) 2024-08-17T17:59:11.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:11.449+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:30.207614+0000 front 2024-08-17T17:58:30.207671+0000 (oldest deadline 2024-08-17T17:58:55.507453+0000) 2024-08-17T17:59:11.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6[280829]: 2024-08-17T17:59:11.449+0000 7f726fd8c640 -1 osd.6 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:40.009232+0000 front 2024-08-17T17:58:40.009107+0000 (oldest deadline 2024-08-17T17:59:02.308963+0000) 2024-08-17T17:59:11.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:11.285+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:11.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:11.285+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:11.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:11.285+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:11.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:11 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:11.285+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:12.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:12.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:12.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:12.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:12.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:12.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:12.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:12 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:12.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:13.159 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:12 smithi152 podman[306624]: 2024-08-17 17:59:12.874690121 +0000 UTC m=+5.532191761 container died 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4) 2024-08-17T17:59:13.517 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:13 smithi152 podman[306624]: 2024-08-17 17:59:13.248551309 +0000 UTC m=+5.906052950 container cleanup 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T17:59:13.517 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:13 smithi152 bash[306624]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6 2024-08-17T17:59:13.517 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:13 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:13.284+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:13.518 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:13 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:13.284+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:13.518 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:13 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:13.284+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:13.518 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:13 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:13.284+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:13.822 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:13 smithi152 podman[306660]: 2024-08-17 17:59:13.516761121 +0000 UTC m=+0.639947026 container remove 940c6673db4949fe1fd083d86c639b8aa607bfb67e6101a9fe725f7a29f5e1b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-17T17:59:14.162 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:13 smithi152 podman[306754]: 2024-08-17 17:59:13.77033098 +0000 UTC m=+0.018179493 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:59:14.162 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:13 smithi152 podman[306754]: 2024-08-17 17:59:13.876248066 +0000 UTC m=+0.124096569 container create b5fe13634f511a2ff3d669710ffe675a93242f942bb5cb7ac9aa03efc5480b3c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, GIT_CLEAN=True) 2024-08-17T17:59:14.162 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:14 smithi152 podman[306754]: 2024-08-17 17:59:14.08711843 +0000 UTC m=+0.334966933 container init b5fe13634f511a2ff3d669710ffe675a93242f942bb5cb7ac9aa03efc5480b3c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:59:14.163 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:14 smithi152 podman[306754]: 2024-08-17 17:59:14.091747113 +0000 UTC m=+0.339595617 container start b5fe13634f511a2ff3d669710ffe675a93242f942bb5cb7ac9aa03efc5480b3c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T17:59:14.163 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:14 smithi152 podman[306754]: 2024-08-17 17:59:14.161944612 +0000 UTC m=+0.409793116 container attach b5fe13634f511a2ff3d669710ffe675a93242f942bb5cb7ac9aa03efc5480b3c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4) 2024-08-17T17:59:14.573 INFO:journalctl@ceph.osd.6.smithi152.stdout:Aug 17 17:59:14 smithi152 podman[306754]: 2024-08-17 17:59:14.270738769 +0000 UTC m=+0.518587281 container died b5fe13634f511a2ff3d669710ffe675a93242f942bb5cb7ac9aa03efc5480b3c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-08-17T17:59:14.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:14.240+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:14.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:14.240+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:14.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:14.240+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:14.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:14 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:14.240+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:14.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T17:59:14.898 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:59:14.899 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T17:59:14.899 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:59:14.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:59:15.238 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:14 smithi152 systemd[1]: Stopping Ceph osd.7 for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:15.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.237+0000 7f38936fb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:59:15.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.237+0000 7f38936fb640 -1 osd.7 537 *** Got signal Terminated *** 2024-08-17T17:59:15.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.237+0000 7f38936fb640 -1 osd.7 537 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:59:15.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:15.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:15.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:15.574 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:15 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:15.254+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:16.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:16.283+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:16.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:16.283+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:16.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:16.283+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:16.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:16 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:16.283+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:17.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:17.241+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:17.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:17.241+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:17.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:17.241+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:17.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:17 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:17.241+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:18.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:18.242+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:18.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:18.242+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:18.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:18.242+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:18.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:18 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:18.242+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:19.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:19 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:19.202+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:19.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:19 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:19.202+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:19.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:19 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:19.202+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:19.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:19 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:19.202+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:20.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:20.189+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6806 osd.0 since back 2024-08-17T17:58:21.891823+0000 front 2024-08-17T17:58:21.891750+0000 (oldest deadline 2024-08-17T17:58:44.092233+0000) 2024-08-17T17:59:20.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:20.190+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6814 osd.1 since back 2024-08-17T17:58:29.993107+0000 front 2024-08-17T17:58:29.993198+0000 (oldest deadline 2024-08-17T17:58:52.893062+0000) 2024-08-17T17:59:20.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:20.190+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6822 osd.2 since back 2024-08-17T17:58:38.794016+0000 front 2024-08-17T17:58:38.793927+0000 (oldest deadline 2024-08-17T17:59:01.093822+0000) 2024-08-17T17:59:20.573 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:20 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7[301925]: 2024-08-17T17:59:20.190+0000 7f388ed11640 -1 osd.7 537 heartbeat_check: no reply from 172.21.15.142:6830 osd.3 since back 2024-08-17T17:58:46.795803+0000 front 2024-08-17T17:58:46.795908+0000 (oldest deadline 2024-08-17T17:59:10.895780+0000) 2024-08-17T17:59:20.909 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:20 smithi152 podman[306882]: 2024-08-17 17:59:20.640041995 +0000 UTC m=+5.509229505 container died 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux ) 2024-08-17T17:59:21.189 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306882]: 2024-08-17 17:59:21.002118151 +0000 UTC m=+5.871305661 container cleanup 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:59:21.189 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 bash[306882]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:59:21.572 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306903]: 2024-08-17 17:59:21.295732858 +0000 UTC m=+0.652788658 container remove 242b8681dd54a1c0d095e4858cf6df3b559d6178a732ea7e4571d58cd4433b2f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:59:21.882 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306998]: 2024-08-17 17:59:21.532491805 +0000 UTC m=+0.017253102 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:59:21.883 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306998]: 2024-08-17 17:59:21.663364657 +0000 UTC m=+0.148125951 container create ada450c36ac7ce718b4871d49d87b68f59ce3279e1d1a9aa1327e4d677314b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_CLEAN=True) 2024-08-17T17:59:21.883 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306998]: 2024-08-17 17:59:21.882584228 +0000 UTC m=+0.367345539 container init ada450c36ac7ce718b4871d49d87b68f59ce3279e1d1a9aa1327e4d677314b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:59:22.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306998]: 2024-08-17 17:59:21.887162357 +0000 UTC m=+0.371923653 container start ada450c36ac7ce718b4871d49d87b68f59ce3279e1d1a9aa1327e4d677314b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T17:59:22.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:21 smithi152 podman[306998]: 2024-08-17 17:59:21.957482103 +0000 UTC m=+0.442243424 container attach ada450c36ac7ce718b4871d49d87b68f59ce3279e1d1a9aa1327e4d677314b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-08-17T17:59:22.323 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:22 smithi152 podman[306998]: 2024-08-17 17:59:22.071009424 +0000 UTC m=+0.555770731 container died ada450c36ac7ce718b4871d49d87b68f59ce3279e1d1a9aa1327e4d677314b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS) 2024-08-17T17:59:22.626 INFO:journalctl@ceph.osd.7.smithi152.stdout:Aug 17 17:59:22 smithi152 podman[307040]: 2024-08-17 17:59:22.602731547 +0000 UTC m=+0.526118941 container remove ada450c36ac7ce718b4871d49d87b68f59ce3279e1d1a9aa1327e4d677314b04 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:59:22.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:59:22.669 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:59:22.669 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:59:22.670 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-08-17T17:59:22.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a 2024-08-17T17:59:23.016 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:22 smithi152 systemd[1]: Stopping Ceph prometheus.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:23.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.015Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:59:23.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.015Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:59:23.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.015Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:59:23.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.015Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:59:23.323 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.015Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.015Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.016Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.016Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.016Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.018Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.018Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a[83860]: ts=2024-08-17T17:59:23.018Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-17T17:59:23.324 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 podman[307126]: 2024-08-17 17:59:23.026353578 +0000 UTC m=+0.132364442 container died f12943011a9ff37b8e74596b61d25f20d48d3c951f352453bf6890828c0bc0e0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:23.690 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 podman[307126]: 2024-08-17 17:59:23.455698905 +0000 UTC m=+0.561709765 container cleanup f12943011a9ff37b8e74596b61d25f20d48d3c951f352453bf6890828c0bc0e0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:23.690 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Aug 17 17:59:23 smithi152 bash[307126]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:59:23.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@prometheus.a.service' 2024-08-17T17:59:23.869 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:59:23.870 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-08-17T17:59:23.870 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T17:59:25.818 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:25 smithi142 systemd[1]: Stopping Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:25.819 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:25 smithi142 ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a[91022]: ts=2024-08-17T17:59:25.795Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:59:26.122 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:25 smithi142 podman[469318]: 2024-08-17 17:59:25.84694218 +0000 UTC m=+0.125331976 container died 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:26.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:26 smithi142 podman[469318]: 2024-08-17 17:59:26.427020492 +0000 UTC m=+0.705410314 container cleanup 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:26.622 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:26 smithi142 bash[469318]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:59:26.980 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:26 smithi142 podman[469336]: 2024-08-17 17:59:26.719787153 +0000 UTC m=+0.865505810 container remove 95562b1fc088aa7a1c5c9139da1ef7b4819268ddbce5afd809c56b7b42c11e33 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:26.981 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:26 smithi142 podman[469336]: 2024-08-17 17:59:26.812933152 +0000 UTC m=+0.958651806 volume remove 055833574def27864103d652d17467b929d33fa5c80859ac89617598d51c6271 2024-08-17T17:59:27.372 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:27 smithi142 systemd[1]: Stopping Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:27.373 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:26 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:59:27.373 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:27 smithi142 systemd[1]: Stopped Ceph alertmanager.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:59:27.373 INFO:journalctl@ceph.alertmanager.a.smithi142.stdout:Aug 17 17:59:27 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 4.763s CPU time. 2024-08-17T17:59:27.732 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:27 smithi142 podman[469450]: 2024-08-17 17:59:27.444249342 +0000 UTC m=+0.119007022 container died c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:28.050 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:27 smithi142 podman[469450]: 2024-08-17 17:59:27.915050728 +0000 UTC m=+0.589808408 container cleanup c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:28.050 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:27 smithi142 bash[469450]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T17:59:28.372 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:28 smithi142 podman[469468]: 2024-08-17 17:59:28.049533351 +0000 UTC m=+0.609217643 container remove c6cd5b4c28e1fdfe58c706fc4b85693afb4da8134b20fe7e3dc9914b71827564 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:59:28.373 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:28 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:59:28.373 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:28 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T17:59:28.373 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:28 smithi142 systemd[1]: Stopped Ceph node-exporter.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:59:28.373 INFO:journalctl@ceph.node-exporter.a.smithi142.stdout:Aug 17 17:59:28 smithi142 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 10.658s CPU time. 2024-08-17T17:59:34.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T17:59:36.323 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:36 smithi152 systemd[1]: Stopping Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:36.645 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:36 smithi152 podman[307546]: 2024-08-17 17:59:36.381295902 +0000 UTC m=+0.123323668 container died d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, version=8.5, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, description=Ceph Grafana Container, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, name=ubi8, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64) 2024-08-17T17:59:36.919 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:36 smithi152 podman[307546]: 2024-08-17 17:59:36.709448775 +0000 UTC m=+0.451476541 container cleanup d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, distribution-scope=public, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, com.redhat.component=ubi8-container, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0) 2024-08-17T17:59:36.920 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:36 smithi152 bash[307546]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a 2024-08-17T17:59:37.307 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:36 smithi152 podman[307564]: 2024-08-17 17:59:36.919347392 +0000 UTC m=+0.540391271 container remove d463717746aeeb558825350ddea4ec7b41260c3ebd5f9839a2a4e7baf44f19cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-grafana-a, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, build-date=2022-04-29T11:48:11.055897, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., release=239.1651231664, distribution-scope=public, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-08-17T17:59:37.307 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:36 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:59:37.307 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:37 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-17T17:59:37.307 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:37 smithi152 systemd[1]: Stopped Ceph grafana.a for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:59:37.308 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Aug 17 17:59:37 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Consumed 38.536s CPU time. 2024-08-17T17:59:37.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:37 smithi152 systemd[1]: Stopping Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:59:37.866 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:37 smithi152 podman[307677]: 2024-08-17 17:59:37.576597993 +0000 UTC m=+0.126277762 container died 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:59:38.282 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 podman[307677]: 2024-08-17 17:59:38.080680507 +0000 UTC m=+0.630360310 container cleanup 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:59:38.282 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 bash[307677]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T17:59:38.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 podman[307694]: 2024-08-17 17:59:38.281760081 +0000 UTC m=+0.705263069 container remove 8e3769ed942e828f9c33892cad2bf7f8df1232fe9568107ecb2376056f6b7a05 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:59:38.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:59:38.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T17:59:38.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 systemd[1]: Stopped Ceph node-exporter.b for bcf463ee-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:59:38.573 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Aug 17 17:59:38 smithi152 systemd[1]: ceph-bcf463ee-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 10.360s CPU time. 2024-08-17T17:59:41.596 DEBUG:teuthology.orchestra.run.smithi142:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:59:41.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:59:41.658 INFO:tasks.cephadm:Archiving crash dumps... 2024-08-17T17:59:41.660 DEBUG:teuthology.misc:Transferring archived files from smithi142:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322/remote/smithi142/crash 2024-08-17T17:59:41.661 DEBUG:teuthology.orchestra.run.smithi142:> sudo tar c -f - -C /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/crash -- . 2024-08-17T17:59:41.709 INFO:teuthology.orchestra.run.smithi142.stderr:tar: /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T17:59:41.709 INFO:teuthology.orchestra.run.smithi142.stderr:tar: Error is not recoverable: exiting now 2024-08-17T17:59:41.711 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322/remote/smithi152/crash 2024-08-17T17:59:41.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/crash -- . 2024-08-17T17:59:41.751 INFO:teuthology.orchestra.run.smithi152.stderr:tar: /var/lib/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T17:59:41.751 INFO:teuthology.orchestra.run.smithi152.stderr:tar: Error is not recoverable: exiting now 2024-08-17T17:59:41.752 INFO:tasks.cephadm:Checking cluster log for badness... 2024-08-17T17:59:41.752 DEBUG:teuthology.orchestra.run.smithi142:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v PG_AVAILABILITY | head -n 1 2024-08-17T17:59:41.838 INFO:teuthology.orchestra.run.smithi142.stdout:2024-08-17T17:10:00.000181+0000 mon.a (mon.0) 524 : cluster [WRN] pg 7.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2024-08-17T17:59:41.839 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-08-17T17:59:41.839 INFO:tasks.cephadm:Compressing logs... 2024-08-17T17:59:41.839 DEBUG:teuthology.orchestra.run.smithi142:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T17:59:41.842 DEBUG:teuthology.orchestra.run.smithi152:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T17:59:41.878 INFO:teuthology.orchestra.run.smithi142.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose: No such file or directory -- 2024-08-17T17:59:41.878 INFO:teuthology.orchestra.run.smithi142.stderr: /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log 2024-08-17T17:59:41.878 INFO:teuthology.orchestra.run.smithi142.stderr:gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.2.log 2024-08-17T17:59:41.880 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T17:59:41.880 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.3.log 2024-08-17T17:59:41.880 INFO:teuthology.orchestra.run.smithi142.stderr:gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.1.log 2024-08-17T17:59:41.880 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.audit.log: /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mgr.y.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.0.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.c.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.a.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.c.log: /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi142.ndmoid.log 2024-08-17T17:59:41.883 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-17T17:59:41.884 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi142.ndmoid.log: 81.8% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi142.ndmoid.log.gz 2024-08-17T17:59:41.885 INFO:teuthology.orchestra.run.smithi152.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-08-17T17:59:41.885 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log 2024-08-17T17:59:41.885 INFO:teuthology.orchestra.run.smithi152.stderr:gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T17:59:41.886 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.7.log 2024-08-17T17:59:41.886 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.5.log 2024-08-17T17:59:41.888 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.4.log 2024-08-17T17:59:41.888 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mgr.x.log 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.6.log 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi152.tacamo.log 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi152.tacamo.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr: 62.5% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi152.tacamo.log.gz 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.b.log 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-17T17:59:41.893 INFO:teuthology.orchestra.run.smithi142.stderr:/var/log/ceph/cephadm.log: 93.4% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T17:59:42.046 INFO:teuthology.orchestra.run.smithi152.stderr:/var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 89.9% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T17:59:42.247 INFO:teuthology.orchestra.run.smithi152.stderr: 84.5% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T17:59:42.276 INFO:teuthology.orchestra.run.smithi152.stderr: 88.1% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T17:59:42.383 INFO:teuthology.orchestra.run.smithi152.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T17:59:42.416 INFO:teuthology.orchestra.run.smithi142.stderr: 91.1% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T17:59:42.545 INFO:teuthology.orchestra.run.smithi152.stderr: 91.2% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T17:59:42.712 INFO:teuthology.orchestra.run.smithi142.stderr: 95.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T17:59:42.788 INFO:teuthology.orchestra.run.smithi142.stderr: 91.2% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T17:59:42.899 INFO:teuthology.orchestra.run.smithi142.stderr: 94.0% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T17:59:42.999 INFO:teuthology.orchestra.run.smithi152.stderr: 90.3% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mgr.x.log.gz 2024-08-17T17:59:49.451 INFO:teuthology.orchestra.run.smithi142.stderr: 89.7% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mgr.y.log.gz 2024-08-17T17:59:57.235 INFO:teuthology.orchestra.run.smithi142.stderr: 91.6% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.c.log.gz 2024-08-17T18:00:25.623 INFO:teuthology.orchestra.run.smithi152.stderr: 92.0% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.b.log.gz 2024-08-17T18:01:01.115 INFO:teuthology.orchestra.run.smithi142.stderr: 90.1% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-mon.a.log.gz 2024-08-17T18:26:01.195 INFO:teuthology.orchestra.run.smithi142.stderr: 93.1% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.2.log.gz 2024-08-17T18:27:55.239 INFO:teuthology.orchestra.run.smithi152.stderr: 92.7% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.7.log.gz 2024-08-17T18:31:43.061 INFO:teuthology.orchestra.run.smithi152.stderr: 93.2% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.4.log.gz 2024-08-17T18:34:05.163 INFO:teuthology.orchestra.run.smithi152.stderr: 93.2% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.5.log.gz 2024-08-17T18:34:17.370 INFO:teuthology.orchestra.run.smithi152.stderr: 92.8% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.6.log.gz 2024-08-17T18:34:17.373 INFO:teuthology.orchestra.run.smithi152.stderr: 2024-08-17T18:34:17.373 INFO:teuthology.orchestra.run.smithi152.stderr:real 34m35.516s 2024-08-17T18:34:17.373 INFO:teuthology.orchestra.run.smithi152.stderr:user 10m22.073s 2024-08-17T18:34:17.373 INFO:teuthology.orchestra.run.smithi152.stderr:sys 0m34.822s 2024-08-17T18:35:01.414 INFO:teuthology.orchestra.run.smithi142.stderr: 92.8% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.0.log.gz 2024-08-17T18:36:47.292 INFO:teuthology.orchestra.run.smithi142.stderr: 93.0% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.1.log.gz 2024-08-17T18:38:17.413 INFO:teuthology.orchestra.run.smithi142.stderr: 92.2% -- replaced with /var/log/ceph/bcf463ee-5cb8-11ef-bccf-c7b262605968/ceph-osd.3.log.gz 2024-08-17T18:38:17.490 INFO:teuthology.orchestra.run.smithi142.stderr: 2024-08-17T18:38:17.490 INFO:teuthology.orchestra.run.smithi142.stderr:real 38m35.635s 2024-08-17T18:38:17.491 INFO:teuthology.orchestra.run.smithi142.stderr:user 12m58.418s 2024-08-17T18:38:17.491 INFO:teuthology.orchestra.run.smithi142.stderr:sys 0m38.974s 2024-08-17T18:38:17.492 INFO:tasks.cephadm:Archiving logs... 2024-08-17T18:38:17.494 DEBUG:teuthology.misc:Transferring archived files from smithi142:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322/remote/smithi142/log 2024-08-17T18:38:17.495 DEBUG:teuthology.orchestra.run.smithi142:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:40:24.940 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322/remote/smithi152/log 2024-08-17T18:40:24.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:41:59.239 INFO:tasks.cephadm:Removing cluster... 2024-08-17T18:41:59.240 DEBUG:teuthology.orchestra.run.smithi142:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 --force 2024-08-17T18:42:02.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bcf463ee-5cb8-11ef-bccf-c7b262605968 --force 2024-08-17T18:42:03.396 INFO:tasks.cephadm:Removing cephadm ... 2024-08-17T18:42:03.396 DEBUG:teuthology.orchestra.run.smithi142:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:42:03.427 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:42:03.443 INFO:tasks.cephadm:Teardown complete 2024-08-17T18:42:03.443 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-08-17T18:42:03.454 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-08-17T18:42:03.455 DEBUG:teuthology.orchestra.run.smithi142:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:42:03.471 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:42:03.591 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-08-17T18:42:03.601 INFO:teuthology.task.clock:Checking final clock skew... 2024-08-17T18:42:03.601 DEBUG:teuthology.orchestra.run.smithi142:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:42:03.604 DEBUG:teuthology.orchestra.run.smithi152:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:42:03.618 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:42:03.619 INFO:teuthology.orchestra.run.smithi142.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:42:03.648 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:42:03.649 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-08-17T18:42:03.649 INFO:teuthology.orchestra.run.smithi152.stdout:^* hv01.front.sepia.ceph.com 3 6 377 53 +321us[ +316us] +/- 43ms 2024-08-17T18:42:03.649 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 125 -360us[ -365us] +/- 63ms 2024-08-17T18:42:03.649 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 1 -15us[ -15us] +/- 78ms 2024-08-17T18:42:03.649 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:42:03.706 INFO:teuthology.orchestra.run.smithi142.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:42:03.706 INFO:teuthology.orchestra.run.smithi142.stdout:=============================================================================== 2024-08-17T18:42:03.706 INFO:teuthology.orchestra.run.smithi142.stdout:^* hv01.front.sepia.ceph.com 3 6 377 53 +286us[ +284us] +/- 43ms 2024-08-17T18:42:03.707 INFO:teuthology.orchestra.run.smithi142.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 52 -380us[ -380us] +/- 64ms 2024-08-17T18:42:03.707 INFO:teuthology.orchestra.run.smithi142.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 53 -81us[ -81us] +/- 77ms 2024-08-17T18:42:03.707 INFO:teuthology.orchestra.run.smithi142.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:42:03.709 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-08-17T18:42:03.718 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-08-17T18:42:03.719 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-08-17T18:42:03.728 DEBUG:teuthology.orchestra.run.smithi142:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:42:03.916 DEBUG:teuthology.orchestra.run.smithi152:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:42:04.066 DEBUG:teuthology.orchestra.run.smithi142:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T18:42:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:42:04.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T18:42:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:42:04.130 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-08-17T18:42:04.141 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-08-17T18:42:04.150 INFO:teuthology.task.internal:Duration was 7130.025332 seconds 2024-08-17T18:42:04.151 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-08-17T18:42:04.160 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-08-17T18:42:04.160 DEBUG:teuthology.orchestra.run.smithi142:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:42:04.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:42:04.213 INFO:teuthology.orchestra.run.smithi142.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:42:04.239 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:42:04.463 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-08-17T18:42:04.463 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi142.front.sepia.ceph.com 2024-08-17T18:42:04.463 DEBUG:teuthology.orchestra.run.smithi142:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:42:04.490 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi152.front.sepia.ceph.com 2024-08-17T18:42:04.490 DEBUG:teuthology.orchestra.run.smithi152:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:42:04.522 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-08-17T18:42:04.522 DEBUG:teuthology.orchestra.run.smithi142:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:42:04.533 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:42:04.589 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-08-17T18:42:04.589 DEBUG:teuthology.orchestra.run.smithi142:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:42:04.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:42:06.784 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-08-17T18:42:06.794 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-08-17T18:42:06.795 DEBUG:teuthology.orchestra.run.smithi142:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:42:06.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:42:06.868 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-08-17T18:42:06.878 DEBUG:teuthology.orchestra.run.smithi142:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:42:06.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:42:06.921 INFO:teuthology.orchestra.run.smithi142.stdout:kernel.core_pattern = core 2024-08-17T18:42:06.945 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = core 2024-08-17T18:42:06.966 DEBUG:teuthology.orchestra.run.smithi142:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:42:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:42:06.993 DEBUG:teuthology.orchestra.run.smithi152:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:42:07.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:42:07.023 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-08-17T18:42:07.035 INFO:teuthology.task.internal:Transferring archived files... 2024-08-17T18:42:07.036 DEBUG:teuthology.misc:Transferring archived files from smithi142:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322/remote/smithi142 2024-08-17T18:42:07.036 DEBUG:teuthology.orchestra.run.smithi142:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:42:07.133 DEBUG:teuthology.misc:Transferring archived files from smithi152:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858322/remote/smithi152 2024-08-17T18:42:07.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:42:07.189 INFO:teuthology.task.internal:Removing archive directory... 2024-08-17T18:42:07.190 DEBUG:teuthology.orchestra.run.smithi142:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:42:07.193 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:42:07.234 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-08-17T18:42:07.245 INFO:teuthology.task.internal:Not uploading archives. 2024-08-17T18:42:07.246 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-08-17T18:42:07.255 INFO:teuthology.task.internal:Tidying up after the test... 2024-08-17T18:42:07.255 DEBUG:teuthology.orchestra.run.smithi142:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:42:07.258 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:42:07.270 INFO:teuthology.orchestra.run.smithi142.stdout: 262165 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 18:42 /home/ubuntu/cephtest 2024-08-17T18:42:07.270 INFO:teuthology.orchestra.run.smithi142.stdout: 6553601 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 17:55 /home/ubuntu/cephtest/mnt.0 2024-08-17T18:42:07.270 INFO:teuthology.orchestra.run.smithi142.stdout: 6553603 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 17:55 /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T18:42:07.270 INFO:teuthology.orchestra.run.smithi142.stdout: 6558854 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 17:55 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-17T18:42:07.271 INFO:teuthology.orchestra.run.smithi142.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-08-17T18:42:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:42:07.277 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi142 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:42:07.284 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-08-17T18:42:07.294 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-08-17T18:42:07.304 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi142 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:42:07.308 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 7130.025332450867 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi142 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=84be659ccdf441c0adcacde1b0abd720 status: fail success: false 2024-08-17T18:42:07.308 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T18:42:07.331 INFO:teuthology.orchestra.run.smithi152.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 18:42 /home/ubuntu/cephtest 2024-08-17T18:42:07.421 INFO:teuthology.run:FAIL